diff --git a/.github/workflows/_build-wheels.yml b/.github/workflows/_build-wheels.yml
index 3505aaeb..f2941450 100644
--- a/.github/workflows/_build-wheels.yml
+++ b/.github/workflows/_build-wheels.yml
@@ -55,9 +55,11 @@ jobs:
cp -r submodules/protobuf/proto pyjelly/_proto
rm -rf tests/conformance_tests/rdf
rm -rf tests/conformance_tests/grpc
+ rm -rf tests/conformance_tests/sparql
mkdir -p tests/conformance_tests
cp -r submodules/protobuf/test/rdf tests/conformance_tests/rdf
cp -r submodules/protobuf/test/grpc tests/conformance_tests/grpc
+ cp -r submodules/protobuf/test/sparql tests/conformance_tests/sparql
- uses: astral-sh/setup-uv@v7
with:
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 22a229fa..bb37c449 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -68,9 +68,11 @@ jobs:
rm -r pyjelly/_proto
rm -r tests/conformance_tests/rdf
rm -r tests/conformance_tests/grpc
+ rm -r tests/conformance_tests/sparql
cp -r submodules/protobuf/proto pyjelly/_proto
cp -r submodules/protobuf/test/rdf tests/conformance_tests/rdf
cp -r submodules/protobuf/test/grpc tests/conformance_tests/grpc
+ cp -r submodules/protobuf/test/sparql tests/conformance_tests/sparql
- uses: astral-sh/setup-uv@v7
with:
diff --git a/README.md b/README.md
index 53c8aa1b..a517c2ab 100644
--- a/README.md
+++ b/README.md
@@ -9,6 +9,7 @@
## Features
- **Fast reading and writing** of RDF knowledge graphs in the [Jelly format](http://w3id.org/jelly).
+- Reading and writing **[SPARQL query results](https://w3id.org/jelly/pyjelly/dev/sparql)** in [Jelly-SPARQL](https://w3id.org/jelly/dev/specification/sparql), with rdflib or the generic API.
- **Standalone [generic API](https://w3id.org/jelly/pyjelly/dev/generic-sink)** with no third-party dependencies, allowing for:
- Serialization and parsing of statements to and from Jelly files.
- Parsing and serializing streams of graphs and statements.
diff --git a/benchmarks/README.md b/benchmarks/README.md
index af331887..ddf7b0f3 100644
--- a/benchmarks/README.md
+++ b/benchmarks/README.md
@@ -1,6 +1,6 @@
# Benchmarks
-Throughput and size benchmarks of Jelly-RDF in pyjelly, compared with the RDF formats of rdflib.
+Throughput and size benchmarks of Jelly-RDF and Jelly-SPARQL in pyjelly, compared with the formats of rdflib.
The method follows the [Jelly-JVM benchmarks](https://github.com/Jelly-RDF/jelly-jvm/tree/main/jmh) as closely as Python allows, so that the results can be read the same way.
@@ -109,3 +109,38 @@ Before measuring, each case writes the data, reads it back and compares the stat
- Before each iteration, the garbage collector runs, outside the timing. The data prepared before measuring is frozen with `gc.freeze()`, so the collector doesn't go through it again and again during the measurement.
- A sweep runs every case once. There are 3 sweeps. The result is the median of the sweeps. Values whose sweeps differ by more than 10% are marked with `*`.
- Averages over datasets are geometric means. A method that couldn't run on all datasets has the number of datasets it ran on next to its mean.
+
+## Jelly-SPARQL
+
+The `sparql` suite measures SPARQL query results instead of RDF statements, as in the Jelly-SPARQL benchmarks of Jelly-JVM. Pass `--suite sparql` to `run`, `case` and `size`:
+
+```bash
+uv run python -m benchmarks run write --suite sparql
+uv run python -m benchmarks run read --suite sparql
+uv run python -m benchmarks size --suite sparql
+```
+
+Everything else (processes, iterations, sweeps, cores, the round-trip check) is the same as for RDF.
+
+### Data
+
+- The 13 RiverBench datasets, read as the results of `SELECT * WHERE { ?s ?p ?o }` for triples and `SELECT * WHERE { GRAPH ?g { ?s ?p ?o } }` for quads, with `?g` unbound in the default graph. The first 100,000 rows are used.
+- Synthetic result sets, from the presets of Jelly-JVM's generator, ported in `sparql_data.py`.
+- By default, `run` and `size` use all RiverBench datasets and `realistic-mixed`, `wide-20`, `sparse-random-50`, `lit-lang`, `iri-shuffled`. Pass any presets with `--datasets`. Their names are in `PRESETS` in `sparql_data.py`.
+
+### Methods
+
+| Method | Writes with | Reads with |
+|---|---|---|
+| `rdflib-srj` | rdflib's SPARQL JSON results: `Result.serialize(format="json")` | `Result.parse(format="json")` |
+| `rdflib-srx` | rdflib's SPARQL XML results | `Result.parse(format="xml")` |
+| `rdflib-csv` | rdflib's SPARQL CSV results | `Result.parse(format="csv")` |
+| `rdflib-tsv` | - (rdflib can't write TSV results) | `Result.parse(format="tsv")` |
+| `rdflib-jelly` | Jelly-SPARQL through rdflib's plugin API: `Result.serialize(format="jelly")` | `Result.parse(format="jelly")` |
+| `generic-jelly` | Jelly-SPARQL with pyjelly's generic API, from a list of rows | `parse_jelly_sparql` of the generic API |
+
+The rdflib methods write a `Result` whose bindings are a list of dicts, made before measuring. Reading makes a whole `Result`, with its bindings. Jelly uses pyjelly's default options.
+
+Note that CSV is a lossy result set format, so it should not be compared directly with others.
+
+rdflib reads SPARQL TSV results but can't write them, so `rdflib-tsv` is only measured reading. The TSV it reads is written by a helper in `sparql_methods.py`.
diff --git a/benchmarks/__main__.py b/benchmarks/__main__.py
index c05b69e1..fb6db343 100644
--- a/benchmarks/__main__.py
+++ b/benchmarks/__main__.py
@@ -15,7 +15,7 @@
from pathlib import Path
from benchmarks import data, measure, report
-from benchmarks.methods import METHODS
+from benchmarks.suites import SUITES
RESULTS_DIR = Path(__file__).parent / "results"
@@ -24,18 +24,36 @@ def _names(value: str) -> list[str]:
return [name.strip() for name in value.split(",") if name.strip()]
+def _add_suite(parser: argparse.ArgumentParser) -> None:
+ parser.add_argument(
+ "--suite",
+ choices=list(SUITES),
+ default="rdf",
+ help=(
+ "rdf: Jelly-RDF and RDF formats, sparql: Jelly-SPARQL and SPARQL "
+ "result formats (default: %(default)s)"
+ ),
+ )
+
+
def _add_selection(parser: argparse.ArgumentParser) -> None:
+ _add_suite(parser)
parser.add_argument(
"--datasets",
type=_names,
- default=list(data.DATASETS),
- help="comma-separated RiverBench datasets (default: all 13)",
+ help=(
+ "comma-separated datasets (default: the 13 RiverBench datasets, and in "
+ "the sparql suite, also a few synthetic ones)"
+ ),
)
parser.add_argument(
"--methods",
type=_names,
- default=list(METHODS),
- help=f"comma-separated methods (default: all of {', '.join(METHODS)})",
+ help=(
+ "comma-separated methods (default: all of the suite: "
+ + "; ".join(f"{s.name}: {', '.join(s.methods)}" for s in SUITES.values())
+ + ")"
+ ),
)
parser.add_argument(
"--rows",
@@ -69,6 +87,16 @@ def _add_timing(parser: argparse.ArgumentParser) -> None:
)
+def _fill_selection(args: argparse.Namespace, parser: argparse.ArgumentParser) -> None:
+ """Fill in the suite's default datasets and methods, if none are given."""
+ suite = SUITES[args.suite]
+ args.datasets = args.datasets or list(suite.default_datasets)
+ args.methods = args.methods or list(suite.methods)
+ for name in args.methods:
+ if name not in suite.methods:
+ parser.error(f"no method {name} in the {args.suite} suite")
+
+
def _format_cpus(cpus: list[int]) -> str:
"""Format CPU cores like taskset does: "5-9,15"."""
ranges: list[list[int]] = []
@@ -112,10 +140,13 @@ def _case(args: argparse.Namespace, parser: argparse.ArgumentParser) -> None:
if sys.platform != "linux":
parser.error("--cpus works only on Linux")
os.sched_setaffinity(0, args.cpus)
+ if args.method not in SUITES[args.suite].methods:
+ parser.error(f"no method {args.method} in the {args.suite} suite")
result = measure.run_case(
args.task,
args.dataset,
args.method,
+ suite=args.suite,
rows=args.rows,
warmup=args.warmup,
iterations=args.iterations,
@@ -133,6 +164,7 @@ def _run(args: argparse.Namespace) -> None:
settings = {
name: getattr(args, name)
for name in (
+ "suite",
"task",
"datasets",
"methods",
@@ -156,9 +188,11 @@ def _run(args: argparse.Namespace) -> None:
seconds=args.time,
check=args.check,
cpu_sets=args.cpus,
+ suite=args.suite,
)
stamp = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S")
- out = args.out or RESULTS_DIR / f"{stamp}-{args.task}.json"
+ prefix = "sparql-" if args.suite == "sparql" else ""
+ out = args.out or RESULTS_DIR / f"{stamp}-{prefix}{args.task}.json"
measure.save(out, settings, results)
print(f"\nSaved to {out}\n")
print(report.throughput_table(args.task, results))
@@ -196,7 +230,8 @@ def main() -> None:
)
case.add_argument("task", choices=["write", "read"])
case.add_argument("dataset")
- case.add_argument("method", choices=list(METHODS))
+ case.add_argument("method")
+ _add_suite(case)
case.add_argument("--rows", type=int, default=data.DEFAULT_ROWS)
case.add_argument(
"--cpus", type=measure.parse_cpu_list, help="CPU cores to pin the case to"
@@ -215,6 +250,8 @@ def main() -> None:
show.add_argument("results", type=Path, nargs="+")
args = parser.parse_args()
+ if args.command in ("run", "size"):
+ _fill_selection(args, parser)
if args.command == "fetch":
data.fetch(args.datasets)
elif args.command == "case":
@@ -222,7 +259,7 @@ def main() -> None:
elif args.command == "run":
_run(args)
elif args.command == "size":
- print(report.size_tables(args.datasets, args.methods, args.rows))
+ print(report.size_tables(args.datasets, args.methods, args.rows, args.suite))
elif args.command == "report":
for path in args.results:
content = json.loads(path.read_text())
diff --git a/benchmarks/measure.py b/benchmarks/measure.py
index 6a500cd8..6ddf0081 100644
--- a/benchmarks/measure.py
+++ b/benchmarks/measure.py
@@ -30,8 +30,8 @@
from typing import Any
import pyjelly.parse.decode
-from benchmarks.data import load
-from benchmarks.methods import METHODS, NullOutput, round_trip
+from benchmarks.methods import NullOutput
+from benchmarks.suites import SUITES
from pyjelly.options import MAX_VERSION
# The folder that has the `benchmarks` package, where the cases run. With a
@@ -45,9 +45,10 @@ class CaseResult:
dataset: str
method: str
#: Statements done in one operation: every statement the writer writes, or
- #: every statement the reader found in the bytes
+ #: every statement the reader found in the bytes. In the sparql suite, cells
+ #: (rows times variables).
statements: int = 0
- #: Throughput of each iteration, in statements per second
+ #: Throughput of each iteration, in statements (or cells) per second
warmup: list[float] = field(default_factory=list)
measured: list[float] = field(default_factory=list)
#: Mean throughput of the measured iterations, None if the case didn't run
@@ -61,6 +62,8 @@ class CaseResult:
#: The CPU cores the case ran on
cpus: list[int] | None = None
sweep: int = 0
+ #: Suite of the case: "rdf" or "sparql", see suites.py
+ suite: str = "rdf"
def _iteration(
@@ -94,6 +97,7 @@ def run_case( # noqa: PLR0913
seconds: float,
check: bool = True,
profile: Path | None = None,
+ suite: str = "rdf",
) -> CaseResult:
"""
Run one case in this process. See the module docstring.
@@ -101,14 +105,19 @@ def run_case( # noqa: PLR0913
With `check=False`, the round-trip check is skipped. Do that only when the
code under test hasn't changed since the check last passed.
"""
- result = CaseResult(task, dataset, method_name, checked=check)
+ result = CaseResult(task, dataset, method_name, checked=check, suite=suite)
if hasattr(os, "sched_getaffinity"):
result.cpus = sorted(os.sched_getaffinity(0))
- method = METHODS[method_name]
- data = load(dataset, rows)
+ benchmarks = SUITES[suite]
+ round_trip = benchmarks.round_trip
+ method = benchmarks.methods[method_name]
+ data = benchmarks.load(dataset, rows)
if not method.supports(data):
result.skipped = "the format can't hold this kind of statements"
return result
+ if task == "write" and not getattr(method, "can_write", True):
+ result.skipped = "the library can't write the format"
+ return result
# Setup, not measured: make the terms (and the graph, for writers that need
# one), check that the method round-trips the data, and make the bytes to read
@@ -140,7 +149,7 @@ def operation() -> None:
result.statements = method.read(written, quads=data.quads)
def operation() -> int:
- return method.read(written, quads=data.quads)
+ return int(method.read(written, quads=data.quads))
# Everything made so far lives until the end. Freezing it keeps the garbage
# collector from going through it again and again during the measurement.
@@ -222,6 +231,7 @@ def _run_case_process( # noqa: PLR0913
seconds: float,
check: bool,
cpus: Sequence[int] | None,
+ suite: str,
) -> CaseResult:
"""Run one case in a new process, pinned to `cpus` if given."""
command = [
@@ -232,6 +242,7 @@ def _run_case_process( # noqa: PLR0913
task,
dataset,
method,
+ f"--suite={suite}",
f"--rows={rows}",
f"--warmup={warmup}",
f"--iterations={iterations}",
@@ -250,7 +261,7 @@ def _run_case_process( # noqa: PLR0913
# The JSON is the last line, in case a library printed something
last_line = process.stdout.strip().splitlines()[-1]
return CaseResult(**json.loads(last_line))
- result = CaseResult(task, dataset, method, checked=check)
+ result = CaseResult(task, dataset, method, checked=check, suite=suite)
result.skipped = f"the process failed with code {process.returncode}"
return result
@@ -311,6 +322,7 @@ def run_sweeps( # noqa: PLR0913
seconds: float,
check: bool = True,
cpu_sets: Sequence[Sequence[int]] = (),
+ suite: str = "rdf",
) -> list[CaseResult]:
"""
Run every case `sweeps` times, each time in a new process.
@@ -340,9 +352,11 @@ def run_share(sweep: int, worker: int) -> None:
seconds=seconds,
check=check,
cpus=sets[worker],
+ suite=suite,
)
result.sweep = sweep
- score = f"{result.score:,.0f} st/s" if result.score else result.skipped
+ unit = SUITES[suite].unit
+ score = f"{result.score:,.0f} {unit}/s" if result.score else result.skipped
with lock:
results.append(result)
progress = f"[{len(results)}/{total}] sweep {sweep + 1}"
diff --git a/benchmarks/report.py b/benchmarks/report.py
index 8618cfed..c3f13672 100644
--- a/benchmarks/report.py
+++ b/benchmarks/report.py
@@ -9,9 +9,8 @@
from collections.abc import Callable, Sequence
from typing import Any
-from benchmarks.data import load
from benchmarks.measure import CaseResult
-from benchmarks.methods import METHODS
+from benchmarks.suites import SUITES
# A case is marked when its sweeps differ by more than this, relative to the median
SPREAD_LIMIT = 0.10
@@ -53,16 +52,18 @@ def _geometric_means(
def throughput_table(task: str, results: Sequence[CaseResult]) -> str:
"""
- Median throughput of each case over its sweeps, in thousands of statements/s.
+ Median throughput of each case over its sweeps, in thousands per second.
A value marked with `*` has sweeps that differ by more than 10%.
"""
+ unit = "statements"
scores: dict[tuple[str, str], list[float]] = defaultdict(list)
datasets: list[str] = []
methods: list[str] = []
for result in results:
if result.task != task:
continue
+ unit = SUITES[result.suite].unit
if result.dataset not in datasets:
datasets.append(result.dataset)
if result.method not in methods:
@@ -87,7 +88,7 @@ def throughput_table(task: str, results: Sequence[CaseResult]) -> str:
rows.append(
["**geometric mean**", *_geometric_means(medians, methods, datasets, 1)]
)
- title = f"### {task.capitalize()} throughput (thousands of statements per second)"
+ title = f"### {task.capitalize()} throughput (thousands of {unit} per second)"
return f"{title}\n\n{_table(['dataset', *methods], rows)}"
@@ -113,24 +114,30 @@ def zstd_compress(data: bytes) -> bytes:
}
-def size_tables(datasets: Sequence[str], methods: Sequence[str], rows: int) -> str:
+def size_tables(
+ datasets: Sequence[str], methods: Sequence[str], rows: int, suite: str = "rdf"
+) -> str:
"""
- Return tables of serialized sizes in bytes per statement.
+ Build tables of serialized sizes in bytes per statement (or cell).
Always divided by the number of statements given to the writer, also when the
writer dropped duplicates.
"""
+ benchmarks = SUITES[suite]
+ unit = benchmarks.unit.removesuffix("s")
compressors = _compressors()
sizes: dict[str, dict[tuple[str, str], float]] = defaultdict(dict)
for dataset in datasets:
- data = load(dataset, rows)
+ data = benchmarks.load(dataset, rows)
+ # Number of statements (or cells) given to the writer
+ count = data.cells if suite == "sparql" else rows
for name in methods:
- method = METHODS[name]
- if not method.supports(data):
+ method = benchmarks.methods[name]
+ if not method.supports(data) or not getattr(method, "can_write", True):
continue
written = method.write_bytes(data)
for compression, compress in compressors.items():
- size = len(compress(written)) / rows
+ size = len(compress(written)) / count
sizes[compression][dataset, name] = size
print(f"sizes: {dataset} done", file=sys.stderr)
@@ -147,7 +154,7 @@ def size_tables(datasets: Sequence[str], methods: Sequence[str], rows: int) -> s
table_rows.append(
["**geometric mean**", *_geometric_means(values, methods, datasets, 2)]
)
- title = f"### Size, {compression} (bytes per statement)"
+ title = f"### Size, {compression} (bytes per {unit})"
tables.append(f"{title}\n\n{_table(['dataset', *methods], table_rows)}")
return "\n\n".join(tables)
diff --git a/benchmarks/sparql_data.py b/benchmarks/sparql_data.py
new file mode 100644
index 00000000..12fe63f5
--- /dev/null
+++ b/benchmarks/sparql_data.py
@@ -0,0 +1,465 @@
+"""Benchmark datasets for Jelly-SPARQL."""
+
+from __future__ import annotations
+
+import random
+from dataclasses import dataclass, field, replace
+from functools import cached_property
+from typing import Any, ClassVar
+
+from rdflib import BNode, Literal, URIRef, Variable
+from rdflib.graph import DATASET_DEFAULT_GRAPH_ID
+from rdflib.query import Result
+
+from benchmarks import data as rdf_data
+from pyjelly.integrations.generic import generic_sink as generic
+
+# --- Result set generator, ported from Jelly-JVM's SparqlDataGen ---
+
+IRI = "iri"
+BNODE = "bnode"
+PLAIN_LITERAL = "plain"
+LANG_LITERAL = "lang"
+DT_LITERAL = "typed"
+DIR_LANG_LITERAL = "dirlang"
+TRIPLE_TERM = "triple"
+
+# Placement of the unbound cells in a column: random cells, every second cell, or
+# blocks of `block_size` cells
+RANDOM = "random"
+ALTERNATING = "alternating"
+BLOCKS = "blocks"
+
+_NAMESPACE_BASE = "https://example.org/ns"
+_DATATYPE_BASE = "https://example.org/dt#"
+_LANGS = ("en", "fr", "de", "es", "pl", "nl", "en-GB", "pt-BR")
+
+
+@dataclass(frozen=True)
+class ColumnSpec:
+ """Settings of one variable of a generated result set. Defaults: IRIs, all bound."""
+
+ kind: str = IRI
+ #: Number of distinct values in the column
+ distinct_values: int = 1024
+ #: Number of IRI namespaces, datatypes, or language tags
+ namespaces: int = 1
+ #: Length of the local name of IRIs, or of the lexical form of literals
+ value_length: int = 12
+ #: Fraction of the cells that are bound
+ bound_fraction: float = 1.0
+ sparsity: str = RANDOM
+ block_size: int = 1
+ #: Mean number of consecutive rows with the same value
+ run_length: float = 1.0
+ #: Whether run lengths are random, rather than exactly `run_length`
+ jitter_runs: bool = True
+ #: Whether values are taken in order (as with ORDER BY), rather than at random
+ sorted: bool = False
+ #: Fraction of the bound cells that have another term type
+ mix_fraction: float = 0.0
+ #: First row that may have a value of the other term type
+ mix_start_row: int = 0
+
+
+@dataclass(frozen=True)
+class ResultSetSpec:
+ name: str
+ columns: tuple[ColumnSpec, ...]
+ rows: int
+ seed: int = 0
+
+ @property
+ def variables(self) -> list[str]:
+ return [f"v{i}" for i in range(len(self.columns))]
+
+
+def _pad(base: str, length: int) -> str:
+ return base if len(base) >= length else base + "x" * (length - len(base))
+
+
+def _iri(column: ColumnSpec, j: int, name: str) -> generic.IRI:
+ namespace = j % max(1, column.namespaces)
+ return generic.IRI(
+ f"{_NAMESPACE_BASE}{namespace}#{_pad(name, max(1, column.value_length))}"
+ )
+
+
+def _pool(column: ColumnSpec, kind: str) -> list[Any]:
+ """Build the distinct values of a column, as generic terms."""
+ n = max(1, column.distinct_values)
+ length = max(1, column.value_length)
+ if kind == IRI:
+ return [_iri(column, j, f"term{j}") for j in range(n)]
+ if kind == BNODE:
+ return [generic.BlankNode(_pad(f"b{j}", min(length, 24))) for j in range(n)]
+ if kind == TRIPLE_TERM:
+ return [_triple_term(column, j) for j in range(n)]
+ return _literal_pool(column, kind)
+
+
+def _literal_pool(column: ColumnSpec, kind: str) -> list[Any]:
+ n = max(1, column.distinct_values)
+ ns = max(1, column.namespaces)
+ length = max(1, column.value_length)
+ if kind == PLAIN_LITERAL:
+ return [generic.Literal(_pad(f"value {j} ", length)) for j in range(n)]
+ if kind == LANG_LITERAL:
+ langs = _LANGS[: min(ns, len(_LANGS))]
+ return [
+ generic.Literal(_pad(f"value {j} ", length), langs[j % len(langs)])
+ for j in range(n)
+ ]
+ if kind == DT_LITERAL:
+ return [
+ generic.Literal(_pad(f"{j}", length), None, f"{_DATATYPE_BASE}d{j % ns}")
+ for j in range(n)
+ ]
+ assert kind == DIR_LANG_LITERAL
+ # Here each "namespace" is one pair of a language tag and a direction
+ return [
+ generic.Literal(
+ _pad(f"value {j} ", length),
+ _LANGS[(j % ns) // 2 % len(_LANGS)],
+ None,
+ "ltr" if j % ns % 2 == 0 else "rtl",
+ )
+ for j in range(n)
+ ]
+
+
+def _triple_term(column: ColumnSpec, j: int) -> generic.Triple:
+ """Build a triple term with fresh IRIs, sometimes nested."""
+ o: Any
+ if j % 3 == 0:
+ o = _iri(column, j, f"o{j}")
+ elif j % 3 == 1:
+ o = generic.Literal(f"value {j}", "en", None, "ltr" if j % 2 else "rtl")
+ else:
+ datatype = f"{_DATATYPE_BASE}d{j % max(1, column.namespaces)}"
+ o = generic.Triple(
+ generic.BlankNode(f"b{j}"),
+ _iri(column, j, f"q{j}"),
+ generic.Literal(f"{j}", None, datatype),
+ )
+ return generic.Triple(_iri(column, j, f"s{j}"), _iri(column, j, f"p{j % 16}"), o)
+
+
+def _generate_column(column: ColumnSpec, rows: int, rnd: random.Random) -> list[Any]:
+ pool = _pool(column, column.kind)
+ other_kind = PLAIN_LITERAL if column.kind == IRI else IRI
+ other_pool = _pool(column, other_kind) if column.mix_fraction > 0 else []
+ cursor = 0
+
+ def next_value(row: int) -> Any:
+ nonlocal cursor
+ use_other = (
+ other_pool
+ and row >= column.mix_start_row
+ and rnd.random() < column.mix_fraction
+ )
+ values = other_pool if use_other else pool
+ if column.sorted:
+ value = values[cursor % len(values)]
+ cursor += 1
+ return value
+ return values[rnd.randrange(len(values))]
+
+ def run_length() -> int:
+ if column.run_length <= 1:
+ return 1
+ if not column.jitter_runs:
+ return max(1, round(column.run_length))
+ # Geometric distribution with mean run_length
+ p = 1 / column.run_length
+ n = 1
+ while rnd.random() > p and n < 4096: # noqa: PLR2004
+ n += 1
+ return n
+
+ cells: list[Any] = []
+ while len(cells) < rows:
+ row = len(cells)
+ if column.sparsity == ALTERNATING:
+ cells.append(None if row % 2 else next_value(row))
+ elif rnd.random() >= column.bound_fraction:
+ block = max(1, column.block_size) if column.sparsity == BLOCKS else 1
+ cells.extend([None] * min(block, rows - len(cells)))
+ else:
+ value = next_value(row)
+ cells.extend([value] * min(run_length(), rows - len(cells)))
+ return cells
+
+
+def generate(spec: ResultSetSpec) -> list[tuple[Any, ...]]:
+ """Generate the rows of a result set: generic terms, None if unbound."""
+ if not spec.columns:
+ return [()] * spec.rows
+ columns = [
+ _generate_column(column, spec.rows, random.Random(spec.seed * 31 + i)) # noqa: S311
+ for i, column in enumerate(spec.columns)
+ ]
+ return list(zip(*columns, strict=True))
+
+
+_N = 20_000
+
+PRESETS: dict[str, ResultSetSpec] = {
+ spec.name: spec
+ for spec in (
+ # IRIs: lookups, and prefix and name inference
+ ResultSetSpec("iri-sorted", (ColumnSpec(distinct_values=_N, sorted=True),), _N),
+ ResultSetSpec("iri-shuffled", (ColumnSpec(distinct_values=_N),), _N),
+ ResultSetSpec("iri-small-pool", (ColumnSpec(distinct_values=2048),), _N),
+ ResultSetSpec(
+ "iri-many-ns", (ColumnSpec(distinct_values=2048, namespaces=32),), _N
+ ),
+ # Repeated values: runs in the layouts
+ ResultSetSpec(
+ "runs-2",
+ (ColumnSpec(distinct_values=2048, run_length=2, sorted=True),),
+ _N,
+ ),
+ ResultSetSpec(
+ "runs-8",
+ (ColumnSpec(distinct_values=2048, run_length=8, sorted=True),),
+ _N,
+ ),
+ ResultSetSpec(
+ "alternating-values", (ColumnSpec(distinct_values=2, sorted=True),), _N
+ ),
+ # Unbound cells
+ ResultSetSpec(
+ "sparse-random-10",
+ (ColumnSpec(distinct_values=2048, bound_fraction=0.1),),
+ _N,
+ ),
+ ResultSetSpec(
+ "sparse-random-50",
+ (ColumnSpec(distinct_values=2048, bound_fraction=0.5),),
+ _N,
+ ),
+ ResultSetSpec(
+ "sparse-random-90",
+ (ColumnSpec(distinct_values=2048, bound_fraction=0.9),),
+ _N,
+ ),
+ ResultSetSpec(
+ "sparse-blocks",
+ (
+ ColumnSpec(
+ distinct_values=2048,
+ bound_fraction=0.5,
+ sparsity=BLOCKS,
+ block_size=16,
+ ),
+ ),
+ _N,
+ ),
+ ResultSetSpec(
+ "sparse-alternating",
+ (ColumnSpec(distinct_values=2048, sparsity=ALTERNATING),),
+ _N,
+ ),
+ # Literals
+ ResultSetSpec(
+ "lit-plain", (ColumnSpec(PLAIN_LITERAL, distinct_values=2048),), _N
+ ),
+ ResultSetSpec(
+ "lit-lang",
+ (ColumnSpec(LANG_LITERAL, distinct_values=2048, namespaces=8),),
+ _N,
+ ),
+ ResultSetSpec(
+ "lit-lang-one",
+ (ColumnSpec(LANG_LITERAL, distinct_values=2048, namespaces=1),),
+ _N,
+ ),
+ ResultSetSpec(
+ "lit-typed",
+ (ColumnSpec(DT_LITERAL, distinct_values=2048, namespaces=8),),
+ _N,
+ ),
+ ResultSetSpec(
+ "lit-long",
+ (ColumnSpec(PLAIN_LITERAL, distinct_values=512, value_length=4096),),
+ 2_000,
+ ),
+ ResultSetSpec("bnodes", (ColumnSpec(BNODE, distinct_values=2048),), _N),
+ # Columns that mix term types
+ ResultSetSpec(
+ "poly-half", (ColumnSpec(distinct_values=2048, mix_fraction=0.5),), _N
+ ),
+ ResultSetSpec(
+ "poly-late-switch",
+ (
+ ColumnSpec(
+ distinct_values=2048, mix_fraction=0.0005, mix_start_row=5_000
+ ),
+ ),
+ _N,
+ ),
+ # RDF 1.2 terms
+ ResultSetSpec(
+ "lit-dirlang",
+ (ColumnSpec(DIR_LANG_LITERAL, distinct_values=2048, namespaces=4),),
+ _N,
+ ),
+ ResultSetSpec(
+ "triple-terms",
+ (
+ ColumnSpec(TRIPLE_TERM, distinct_values=_N, namespaces=4),
+ ColumnSpec(distinct_values=2048),
+ ),
+ _N,
+ ),
+ # Shape of the result set
+ ResultSetSpec("wide-5", (ColumnSpec(distinct_values=2048),) * 5, _N),
+ ResultSetSpec("wide-20", (ColumnSpec(distinct_values=2048),) * 20, _N // 2),
+ ResultSetSpec("single-row", (ColumnSpec(distinct_values=8),) * 3, 1),
+ ResultSetSpec("zero-vars", (), _N),
+ # Entity, property, label, optional value, and a value of mixed types
+ ResultSetSpec(
+ "realistic-mixed",
+ (
+ ColumnSpec(
+ distinct_values=4096, run_length=3, sorted=True, value_length=8
+ ),
+ ColumnSpec(distinct_values=32, value_length=10),
+ ColumnSpec(
+ LANG_LITERAL, distinct_values=4096, namespaces=4, value_length=24
+ ),
+ ColumnSpec(PLAIN_LITERAL, distinct_values=1024, bound_fraction=0.4),
+ ColumnSpec(distinct_values=2048, mix_fraction=0.3),
+ ),
+ _N,
+ ),
+ )
+}
+
+#: Datasets used when none are given: all RiverBench datasets, and the presets
+#: closest to the results of real queries
+DEFAULT_DATASETS = (
+ *rdf_data.DATASETS,
+ "realistic-mixed",
+ "wide-20",
+ "sparse-random-50",
+ "lit-lang",
+ "iri-shuffled",
+)
+
+DATASETS = (*rdf_data.DATASETS, *PRESETS)
+
+
+def _resized(spec: ResultSetSpec, rows: int) -> ResultSetSpec:
+ """Resize a preset to `rows` rows, keeping its shape."""
+ factor = rows / spec.rows
+ return replace(
+ spec,
+ rows=rows,
+ columns=tuple(
+ replace(column, mix_start_row=int(column.mix_start_row * factor))
+ for column in spec.columns
+ ),
+ )
+
+
+# --- Results in the terms of each library ---
+
+
+def _rdflib_term(term: Any) -> Any:
+ if term is None:
+ return None
+ if isinstance(term, generic.IRI):
+ return URIRef(term._iri)
+ if isinstance(term, generic.BlankNode):
+ return BNode(term._id)
+ if isinstance(term, generic.Literal):
+ if term._direction:
+ msg = "rdflib has no literals with a base direction"
+ raise TypeError(msg)
+ # Keep the lexical form unchanged, as pyjelly's reader does
+ return Literal(
+ term._lex, lang=term._langtag, datatype=term._datatype, normalize=False
+ )
+ msg = "rdflib has no triple terms"
+ raise TypeError(msg)
+
+
+@dataclass
+class SparqlBenchData:
+ """
+ A result set: its variables, and its rows as generic terms and rdflib terms.
+
+ Throughput is measured in `cells`: rows times variables, or just rows for a
+ result set without variables.
+ """
+
+ #: Always False. The shared benchmark code reads this attribute of RDF data.
+ quads: ClassVar[bool] = False
+
+ name: str
+ variables: list[str]
+ generic: list[tuple[Any, ...]] = field(repr=False)
+ #: Rows as rdflib terms, if not converted from the generic ones
+ rdflib_rows: list[tuple[Any, ...]] | None = field(default=None, repr=False)
+
+ @property
+ def cells(self) -> int:
+ return len(self.generic) * max(1, len(self.variables))
+
+ @cached_property
+ def rdf_12(self) -> bool:
+ """Whether the results have RDF 1.2 terms, which rdflib doesn't support."""
+ return any(
+ isinstance(term, generic.Triple)
+ or (isinstance(term, generic.Literal) and term._direction)
+ for row in self.generic
+ for term in row
+ )
+
+ @cached_property
+ def rdflib(self) -> Result:
+ """The results as an rdflib Result, with a list of bindings."""
+ result = Result("SELECT")
+ variables = [Variable(name) for name in self.variables]
+ result.vars = variables
+ rows = self.rdflib_rows
+ if rows is None:
+ rows = [tuple(map(_rdflib_term, row)) for row in self.generic]
+ result.bindings = [
+ {
+ variable: term
+ for variable, term in zip(variables, row, strict=True)
+ if term is not None
+ }
+ for row in rows
+ ]
+ return result
+
+
+def load(dataset: str, rows: int = rdf_data.DEFAULT_ROWS) -> SparqlBenchData:
+ """Load the first `rows` rows of a dataset."""
+ if dataset in PRESETS:
+ spec = _resized(PRESETS[dataset], rows)
+ return SparqlBenchData(dataset, spec.variables, generate(spec))
+ statements = rdf_data.load(dataset, rows)
+ if not statements.quads:
+ return SparqlBenchData(
+ dataset,
+ ["s", "p", "o"],
+ [tuple(statement) for statement in statements.generic],
+ [tuple(statement) for statement in statements.rdflib],
+ )
+ return SparqlBenchData(
+ dataset,
+ ["s", "p", "o", "g"],
+ [
+ (s, p, o, None if g is generic.DefaultGraph else g)
+ for s, p, o, g in statements.generic
+ ],
+ [
+ (s, p, o, None if g == DATASET_DEFAULT_GRAPH_ID else g)
+ for s, p, o, g in statements.rdflib
+ ],
+ )
diff --git a/benchmarks/sparql_methods.py b/benchmarks/sparql_methods.py
new file mode 100644
index 00000000..63e8a24d
--- /dev/null
+++ b/benchmarks/sparql_methods.py
@@ -0,0 +1,226 @@
+"""
+SPARQL query result ser/des implementations for benchmarking.
+
+Each method writes the rows of a result set to a stream, and reads them back from
+bytes in memory.
+"""
+
+from __future__ import annotations
+
+import io
+from abc import ABC, abstractmethod
+from collections import Counter
+from typing import IO, Any
+from typing_extensions import override
+
+from rdflib import BNode, Literal, URIRef
+from rdflib.query import Result
+
+from benchmarks.sparql_data import SparqlBenchData
+from pyjelly.integrations.generic import generic_sink
+from pyjelly.integrations.generic import sparql as generic_sparql
+from pyjelly.options import STRING_DATATYPE_IRI
+
+
+class SparqlMethod(ABC):
+ name: str
+ #: Whether the library can write the format. If not, `write` is only a helper
+ #: that produces the input for reading, and writing isn't measured.
+ can_write: bool = True
+
+ def supports(self, data: SparqlBenchData) -> bool: # noqa: ARG002
+ return True
+
+ @abstractmethod
+ def rows(self, data: SparqlBenchData) -> list[tuple[Any, ...]]:
+ """Get the rows as this method's terms, None if unbound."""
+
+ def prepare(self, data: SparqlBenchData) -> None:
+ """Build the writer's input in advance, outside the measurement."""
+ self.rows(data)
+
+ def write_count(self, data: SparqlBenchData) -> int:
+ """Count the cells that `write` writes."""
+ return data.cells
+
+ @abstractmethod
+ def write(self, data: SparqlBenchData, out: IO[bytes]) -> None:
+ """Write the result set to `out`."""
+
+ @abstractmethod
+ def read(self, data: bytes, *, quads: bool = False) -> int:
+ """
+ Read the rows without keeping them, and return the number of cells read.
+
+ `quads` is unused. It's there to match the RDF methods.
+ """
+
+ @abstractmethod
+ def read_all(self, data: bytes) -> list[tuple[Any, ...]]:
+ """Read and return the rows, for the round-trip check."""
+
+ def write_bytes(self, data: SparqlBenchData) -> bytes:
+ out = io.BytesIO()
+ self.write(data, out)
+ return out.getvalue()
+
+
+def _cells(rows: int, variables: int) -> int:
+ return rows * max(1, variables)
+
+
+class RdflibResults(SparqlMethod):
+ """An rdflib result format, written with Result.serialize, read with parse."""
+
+ def __init__(self, name: str, rdflib_format: str) -> None:
+ self.name = name
+ self.format = rdflib_format
+
+ @override
+ def supports(self, data: SparqlBenchData) -> bool:
+ return not data.rdf_12
+
+ @override
+ def rows(self, data: SparqlBenchData) -> list[tuple[Any, ...]]:
+ result = data.rdflib
+ variables = result.vars or []
+ return [tuple(b.get(v) for v in variables) for b in result.bindings]
+
+ @override
+ def write(self, data: SparqlBenchData, out: IO[bytes]) -> None:
+ data.rdflib.serialize(destination=out, format=self.format)
+
+ def _parse(self, data: bytes) -> Result:
+ return Result.parse(io.BytesIO(data), format=self.format)
+
+ @override
+ def read(self, data: bytes, *, quads: bool = False) -> int:
+ result = self._parse(data)
+ return _cells(len(result.bindings), len(result.vars or []))
+
+ @override
+ def read_all(self, data: bytes) -> list[tuple[Any, ...]]:
+ result = self._parse(data)
+ variables = result.vars or []
+ return [tuple(b.get(v) for v in variables) for b in result.bindings]
+
+
+def _tsv_term(term: Any) -> str:
+ """Format a term for SPARQL TSV results, in N-Triples syntax."""
+ if term is None:
+ return ""
+ if isinstance(term, URIRef):
+ return f"<{term}>"
+ if isinstance(term, BNode):
+ return f"_:{term}"
+ lex = (
+ str(term)
+ .replace("\\", "\\\\")
+ .replace('"', '\\"')
+ .replace("\n", "\\n")
+ .replace("\r", "\\r")
+ .replace("\t", "\\t")
+ )
+ if term.language:
+ return f'"{lex}"@{term.language}'
+ if term.datatype is not None and str(term.datatype) != STRING_DATATYPE_IRI:
+ return f'"{lex}"^^<{term.datatype}>'
+ return f'"{lex}"'
+
+
+class RdflibTsv(RdflibResults):
+ """
+ SPARQL TSV results, read by rdflib.
+
+ rdflib has no TSV writer, so `write` is a helper written for these benchmarks.
+ """
+
+ can_write = False
+
+ def __init__(self) -> None:
+ super().__init__("rdflib-tsv", "tsv")
+
+ @override
+ def write(self, data: SparqlBenchData, out: IO[bytes]) -> None:
+ lines = ["\t".join(f"?{name}" for name in data.variables)]
+ lines += ["\t".join(map(_tsv_term, row)) for row in self.rows(data)]
+ out.write(("\n".join(lines) + "\n").encode())
+
+
+class GenericJelly(SparqlMethod):
+ """Jelly-SPARQL with pyjelly's generic API and its own terms."""
+
+ name = "generic-jelly"
+
+ @override
+ def rows(self, data: SparqlBenchData) -> list[tuple[Any, ...]]:
+ return data.generic
+
+ @override
+ def write(self, data: SparqlBenchData, out: IO[bytes]) -> None:
+ generic_sparql.serialize_jelly_sparql(data.variables, data.generic, out)
+
+ @override
+ def read(self, data: bytes, *, quads: bool = False) -> int:
+ results = generic_sparql.parse_jelly_sparql(io.BytesIO(data))
+ rows = 0
+ for _ in results:
+ rows += 1
+ return _cells(rows, len(results.variables))
+
+ @override
+ def read_all(self, data: bytes) -> list[tuple[Any, ...]]:
+ return list(generic_sparql.parse_jelly_sparql(io.BytesIO(data)))
+
+
+SPARQL_METHODS: dict[str, SparqlMethod] = {
+ method.name: method
+ for method in (
+ RdflibResults("rdflib-srj", "json"),
+ RdflibResults("rdflib-srx", "xml"),
+ RdflibResults("rdflib-csv", "csv"),
+ RdflibTsv(),
+ # Jelly through rdflib's plugin API: Result.serialize and Result.parse
+ RdflibResults("rdflib-jelly", "jelly"),
+ GenericJelly(),
+ )
+}
+
+
+def _normalize(term: Any) -> Any:
+ """Normalize a term for comparison. Blank node labels are ignored."""
+ if term is None:
+ return None
+ if isinstance(term, (BNode, generic_sink.BlankNode)):
+ return "_:"
+ if isinstance(term, (URIRef, Literal)):
+ return term.n3()
+ return str(term)
+
+
+def sparql_round_trip(
+ method: SparqlMethod, data: SparqlBenchData
+) -> tuple[bytes, list[str]]:
+ """
+ Write the result set with the method, read it back, and compare the rows.
+
+ Returns the written bytes and a list of problems (empty if none). Raises the
+ reader's exception if the method can't read its own output.
+ """
+ written = method.write_bytes(data)
+ expected = [tuple(map(_normalize, row)) for row in method.rows(data)]
+ actual = [tuple(map(_normalize, row)) for row in method.read_all(written)]
+ problems = []
+ if method.read(written) != method.write_count(data):
+ problems.append("read() counts a different number of cells than written")
+ missing = Counter(expected) - Counter(actual)
+ extra = Counter(actual) - Counter(expected)
+ if missing or extra:
+ problems.append(
+ f"{sum(missing.values())} rows missing, {sum(extra.values())} unexpected; "
+ f"e.g. missing {next(iter(missing), None)}, "
+ f"unexpected {next(iter(extra), None)}"
+ )
+ elif expected != actual:
+ problems.append("the rows are in another order")
+ return written, problems
diff --git a/benchmarks/suites.py b/benchmarks/suites.py
new file mode 100644
index 00000000..3beab54e
--- /dev/null
+++ b/benchmarks/suites.py
@@ -0,0 +1,45 @@
+"""Benchmark suites: Jelly-RDF and Jelly-SPARQL, each compared with other formats."""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Mapping
+from dataclasses import dataclass
+from typing import Any
+
+from benchmarks import data, sparql_data
+from benchmarks.methods import METHODS, round_trip
+from benchmarks.sparql_methods import SPARQL_METHODS, sparql_round_trip
+
+
+@dataclass(frozen=True)
+class Suite:
+ name: str
+ #: Unit of the throughput: statements or cells
+ unit: str
+ datasets: tuple[str, ...]
+ default_datasets: tuple[str, ...]
+ methods: Mapping[str, Any]
+ load: Callable[[str, int], Any]
+ round_trip: Callable[[Any, Any], tuple[bytes, list[str]]]
+
+
+SUITES = {
+ "rdf": Suite(
+ "rdf",
+ "statements",
+ data.DATASETS,
+ data.DATASETS,
+ METHODS,
+ data.load,
+ round_trip,
+ ),
+ "sparql": Suite(
+ "sparql",
+ "cells",
+ sparql_data.DATASETS,
+ sparql_data.DEFAULT_DATASETS,
+ SPARQL_METHODS,
+ sparql_data.load,
+ sparql_round_trip,
+ ),
+}
diff --git a/docs/index.md b/docs/index.md
index 194047f8..c9feb8b1 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -7,6 +7,7 @@
## Features
- **Fast reading and writing** of RDF knowledge graphs in the [Jelly format](http://w3id.org/jelly)
+- **[SPARQL query results](sparql.md)** in [Jelly-SPARQL]({{ proto_link("specification/sparql/") }})
- **Seamless integration with [RDFLib](https://rdflib.readthedocs.io/)**
- **Stream processing support** for large datasets or streams of many RDF graphs/datasets
@@ -15,6 +16,7 @@
- **[Getting started](getting-started.md)**
- **[Overview and supported features](overview.md)**
- **[Generic interface (usage without RDFLib)](generic-sink.md)**
+- **[SPARQL query results](sparql.md)**
- **[API reference](api.md)**
- **[Contributing](contributing/index.md)** – how to report issues, contribute code, and request features
diff --git a/docs/overview.md b/docs/overview.md
index c9e11f83..f1aa7bca 100644
--- a/docs/overview.md
+++ b/docs/overview.md
@@ -7,6 +7,7 @@ With Jelly, you can transmit both flat and structured streams of triples, quads,
**pyjelly** is a Python implementation of the Jelly protocol. It provides:
* Full support for reading and writing Jelly-encoded RDF data
+* Reading and writing SPARQL query results in [Jelly-SPARQL](sparql.md)
* Seamless integration with [RDFLib](https://rdflib.readthedocs.io/) (*"works just like Turtle"*)
* Standalone generic API with no third-party dependencies [Generic API](generic-sink.md)
* Support for all Jelly stream types
@@ -39,9 +40,13 @@ A Jelly-RDF 1.2 stream is either one graph or dataset, or a sequence of [RDF Mes
The generic API supports [RDF 1.2](https://www.w3.org/TR/rdf12-concepts/): triple terms, as a `Triple` in the object of a statement, and directional language-tagged strings, as `Literal("text", "en", direction="ltr")`. To declare the RDF version for a stream, set `rdf_version` in `StreamParameters`. RDFLib 7 doesn't support RDF 1.2 yet.
+### SPARQL query results
+
+pyjelly reads and writes the results of `SELECT` and `ASK` queries in [Jelly-SPARQL]({{ proto_link("specification/sparql/") }}), with RDFLib's `Result.parse` and `Result.serialize`, or with the generic API. See [SPARQL query results](sparql.md).
+
### Conformance to the Jelly specification
-pyjelly is continuously tested for conformance to [the Jelly specification]({{ proto_link("specification/") }}). It passes the [conformance tests of Jelly-RDF 1.2]({{ proto_link("conformance/rdf-test-cases/") }}), and the [tests of Jelly-RDF 1.1]({{ proto_link("conformance/rdf-1-1-test-cases/") }}), except for writing the `GRAPHS` physical stream type.
+pyjelly is continuously tested for conformance to [the Jelly specification]({{ proto_link("specification/") }}). It passes the [conformance tests of Jelly-RDF 1.2]({{ proto_link("conformance/rdf-test-cases/") }}), and the [tests of Jelly-RDF 1.1]({{ proto_link("conformance/rdf-1-1-test-cases/") }}), except for writing the `GRAPHS` physical stream type. It also passes the [conformance tests of Jelly-SPARQL]({{ proto_link("conformance/sparql-test-cases/") }}).
See the [conformance tests]({{ git_link("tests/conformance_tests") }}) in the pyjelly repository.
diff --git a/docs/sparql.md b/docs/sparql.md
new file mode 100644
index 00000000..d55ead53
--- /dev/null
+++ b/docs/sparql.md
@@ -0,0 +1,144 @@
+# SPARQL query results
+
+pyjelly reads and writes SPARQL query results in [Jelly-SPARQL]({{ proto_link("specification/sparql/") }}): the results of `SELECT` queries (solution sequences) and `ASK` queries (boolean results). Like Jelly-RDF, it is a compact binary format that can be written and read as a stream.
+
+The media type of Jelly-SPARQL is `application/x-jelly-sparql`, and the file extension is `.jellys`.
+
+## Usage with RDFLib
+
+With pyjelly installed, RDFLib reads and writes query results in the `jelly` format, like it does `json` or `xml`:
+
+{{ code_example('rdflib/09_sparql_results.py') }}
+
+`Result.parse` returns a `JellySparqlResult` (from `pyjelly.integrations.rdflib.sparql`), which is an RDFLib `Result` with more attributes:
+
+- `truncated` – `True` if the stream ended without indicating that the results are complete. pyjelly also issues a `JellyTruncatedResultsWarning` then. A stream cut in the middle of a frame can't be read at all, and raises `JellyConformanceError`.
+- `error` – why the results are incomplete, in a [sequence of results](#sequences-of-results). See: [incomplete results](#incomplete-results).
+- `links` – the IRIs of links that describe the results, like `head.link` in [SPARQL JSON results](https://www.w3.org/TR/sparql12-results-json/).
+
+RDFLib does not support RDF 1.2 yet, so triple terms and literals with a base direction can be read only with the [generic API](#generic-api).
+
+### Querying a SPARQL endpoint
+
+RDFLib's `SPARQLStore` can ask a SPARQL endpoint for Jelly-SPARQL, if the endpoint offers it (e.g., Apache Jena Fuseki with the [Jelly-JVM plugin](https://w3id.org/jelly/jelly-jvm/dev/getting-started-plugins/)):
+
+```python
+from rdflib import Graph
+from rdflib.plugins.stores.sparqlstore import SPARQLStore
+
+store = SPARQLStore(
+ "http://localhost:3030/dataset/sparql",
+ returnFormat="application/x-jelly-sparql",
+)
+for row in Graph(store=store).query("SELECT * WHERE { ?s ?p ?o } LIMIT 10"):
+ print(row)
+```
+
+!!! warning
+
+ Use the media type, `application/x-jelly-sparql`, as `returnFormat`, not `jelly`.
+
+### Large results
+
+By default, `Result.parse` reads all rows at once. With `lazy=True`, the rows are read as the result is iterated, like RDFLib does with the results of its own queries. The stream must then stay open until you are done with the result.
+
+When writing, the rows of an RDFLib result are written as the query makes them, so the first frames are written before the query is done. Each frame is flushed when it's written.
+
+{{ code_example('rdflib/10_sparql_results_lazy.py') }}
+
+!!! note
+
+ Iterate a lazy result with a `for` loop. `list(result)`, `len(result)` and `bool(result)` read all of its rows first, as for any RDFLib result.
+
+### Incomplete results
+
+A query can fail after some of its results were already written, for example in case of a timeout, or a broken backend. Jelly-SPARQL then ends the stream with an error, so that readers know that the results are incomplete. pyjelly does this for you: if getting a row fails while results are being written, the stream ends with the error, and the error is raised.
+
+When reading such results, pyjelly raises `JellyIncompleteResultsError` after the rows that were read. Its `error` attribute is the message of the writer, and its `result` attribute contains the results read before the error:
+
+```python
+from pyjelly.errors import JellyIncompleteResultsError
+
+try:
+ result = Result.parse(file, format="jelly")
+except JellyIncompleteResultsError as error:
+ print(f"The query failed: {error.error}")
+ partial_result = error.result
+```
+
+If you write results that you read from Jelly-SPARQL, and they were incomplete or truncated, pyjelly writes them as incomplete too.
+
+### Options
+
+To change how results are written, pass `SparqlOptions` to `serialize`:
+
+```python
+from pyjelly.options import LookupPreset, SparqlOptions
+
+options = SparqlOptions(
+ # Sizes of the lookups, see below
+ lookup_preset=LookupPreset(max_names=4096, max_prefixes=256, max_datatypes=32),
+ # How many values (rows times variables) a frame may have, at most
+ frame_values=4096,
+)
+results.serialize(destination="results.jellys", format="jelly", options=options)
+```
+
+The default lookup sizes are 8192 names, 1024 prefixes and 64 datatypes. Readers accept lookups of up to 16384 names, 4096 prefixes and 256 datatypes.
+
+With `delimited=False`, the results are written as one frame without its size, e.g., to be used in Kafka or MQTT. Then the lookups must be large enough to store all terms of the result set. Files and HTTP responses must be delimited, which is the default. To send the frames of the results as separate messages, see [writing frame by frame](#writing-frame-by-frame).
+
+### Sequences of results
+
+A Jelly-SPARQL stream can also contain a sequence of results, e.g., the answers to a stream of queries, or to a query evaluated again and again. This is the `PUNCTUATED` stream type of Jelly-SPARQL. To write and read such a stream, use the functions in `pyjelly.integrations.rdflib.sparql`:
+
+```python
+from pyjelly.integrations.rdflib.sparql import (
+ parse_jelly_sparql_sequence,
+ serialize_jelly_sparql_sequence,
+)
+
+with open("answers.jellys", "wb") as file:
+ serialize_jelly_sparql_sequence([result_1, result_2, ask_result], file)
+
+with open("answers.jellys", "rb") as file:
+ for result in parse_jelly_sparql_sequence(file):
+ print(result.type, len(result))
+```
+
+The lookups are shared by the results of the stream, so the terms they have in common are written once.
+
+An incomplete result doesn't end a sequence, as there may be more results after it. `parse_jelly_sparql_sequence` doesn't raise an error for it, but sets its `error` attribute, so check it.
+
+## Generic API
+
+The [generic API](generic-sink.md) supports all of RDF 1.2. A row is a tuple with one value per variable, or `None` if the variable is unbound:
+
+{{ code_example('generic/10_sparql_results.py') }}
+
+`parse_jelly_sparql` returns a `SparqlResultSet`. Its rows are read as you iterate it, so the input must stay open until then, and the rows can be iterated only once. If the results are incomplete, iterating the rows raises `JellyIncompleteResultsError` after the last row.
+
+`serialize_jelly_sparql_ask` writes the result of an `ASK` query. `parse_jelly_sparql_sequence` and `serialize_jelly_sparql_sequence` read and write [sequences of results](#sequences-of-results).
+
+### Writing frame by frame
+
+To write results frame by frame, e.g., to send each frame as a message over Kafka, use `SparqlStream` directly. It works with the terms of RDFLib (`pyjelly.integrations.rdflib.sparql.sparql_stream()`) or of the generic API (`pyjelly.integrations.generic.sparql.sparql_stream()`):
+
+```python
+from pyjelly.integrations.generic.sparql import sparql_stream
+
+stream = sparql_stream()
+stream.start(["person", "name"])
+for row in rows:
+ # A frame, when it is full
+ if frame := stream.row(row):
+ send(frame.SerializeToString())
+# The last frame of the results, which says they are complete
+send(stream.end().SerializeToString())
+```
+
+If the results can't be completed, end them with `stream.end("why")` instead.
+
+## Conformance
+
+pyjelly passes the [conformance tests of Jelly-SPARQL]({{ proto_link("conformance/sparql-test-cases/") }}), with the generic API. The RDFLib integration passes all tests except those that use RDF 1.2 terms.
diff --git a/examples/generic/10_sparql_results.py b/examples/generic/10_sparql_results.py
new file mode 100644
index 00000000..b500aad9
--- /dev/null
+++ b/examples/generic/10_sparql_results.py
@@ -0,0 +1,23 @@
+from pyjelly.integrations.generic.generic_sink import IRI, Literal
+from pyjelly.integrations.generic.sparql import (
+ parse_jelly_sparql,
+ serialize_jelly_sparql,
+)
+
+alice = IRI("http://example.org/alice")
+bob = IRI("http://example.org/bob")
+
+# The rows of the results: one value per variable, None if it is unbound
+rows = [
+ (alice, Literal("Alice"), bob),
+ (bob, Literal("Bob"), None),
+]
+with open("results.jellys", "wb") as file:
+ serialize_jelly_sparql(["person", "name", "knows"], rows, file)
+
+with open("results.jellys", "rb") as file:
+ results = parse_jelly_sparql(file)
+ print("Variables:", results.variables)
+ # The rows are read as they are iterated, so before the file is closed
+ for person, name, knows in results:
+ print(person, name, knows)
diff --git a/examples/rdflib/09_sparql_results.py b/examples/rdflib/09_sparql_results.py
new file mode 100644
index 00000000..1b799f5c
--- /dev/null
+++ b/examples/rdflib/09_sparql_results.py
@@ -0,0 +1,32 @@
+from rdflib import Graph
+from rdflib.query import Result
+
+graph = Graph()
+graph.parse(
+ data="""
+ @prefix foaf: .
+ foaf:name "Alice" ; foaf:knows .
+ foaf:name "Bob" .
+ """,
+ format="turtle",
+)
+
+# Run a query and write its results in Jelly-SPARQL
+results = graph.query("""
+ PREFIX foaf:
+ SELECT ?person ?name WHERE { ?person foaf:name ?name }
+""")
+results.serialize(destination="results.jellys", format="jelly")
+
+# Read them back
+with open("results.jellys", "rb") as file:
+ parsed = Result.parse(file, format="jelly")
+
+for row in parsed:
+ print(f"{row.person} is called {row.name}")
+
+# Results of ASK queries work the same way
+answer = graph.query("ASK { ?person ?p 'Alice' }")
+answer.serialize(destination="answer.jellys", format="jelly")
+with open("answer.jellys", "rb") as file:
+ print("Is anyone called Alice?", Result.parse(file, format="jelly").askAnswer)
diff --git a/examples/rdflib/10_sparql_results_lazy.py b/examples/rdflib/10_sparql_results_lazy.py
new file mode 100644
index 00000000..91d41eee
--- /dev/null
+++ b/examples/rdflib/10_sparql_results_lazy.py
@@ -0,0 +1,33 @@
+from rdflib import Literal, URIRef, Variable
+from rdflib.query import Result
+
+from pyjelly.errors import JellyIncompleteResultsError
+from pyjelly.integrations.rdflib.sparql import (
+ parse_jelly_sparql,
+ serialize_jelly_sparql,
+)
+
+# Many results, made one by one
+results = Result("SELECT")
+results.vars = [Variable("item"), Variable("number")]
+results.bindings = (
+ {
+ Variable("item"): URIRef(f"http://example.org/item/{i}"),
+ Variable("number"): Literal(i),
+ }
+ for i in range(100_000)
+)
+with open("many_results.jellys", "wb") as file:
+ serialize_jelly_sparql(results, file)
+
+# Read the rows as they are needed, not all at once
+with open("many_results.jellys", "rb") as file:
+ parsed = parse_jelly_sparql(file, lazy=True)
+ try:
+ total = sum(row.number.toPython() for row in parsed)
+ except JellyIncompleteResultsError as error:
+ # The writer said that the results are incomplete, e.g., the query failed
+ print(f"Some results are missing: {error.error}")
+ raise
+
+print(f"The sum of the numbers is {total}")
diff --git a/mkdocs.yml b/mkdocs.yml
index 53fba597..4356d7b8 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -119,6 +119,7 @@ nav:
- Getting started (RDFLib): "getting-started.md"
- Overview and supported features: "overview.md"
- Generic API (no RDFLib): "generic-sink.md"
+ - SPARQL query results: "sparql.md"
- Neo4j: "rdflib-neo4j-integration.md"
- NetworkX: "networkx-integration.md"
- API reference: "api.md"
diff --git a/pyjelly/errors.py b/pyjelly/errors.py
index 6ffd686c..b3c94923 100644
--- a/pyjelly/errors.py
+++ b/pyjelly/errors.py
@@ -8,3 +8,27 @@ class JellyAssertionError(AssertionError):
class JellyNotImplementedError(NotImplementedError):
"""Raised when a future feature is not yet implemented."""
+
+
+class JellyIncompleteResultsError(Exception):
+ """
+ Raised when the producer of SPARQL results reports that they are incomplete.
+
+ For example, the query failed midway. The rows read before the error are valid.
+ """
+
+ def __init__(self, message: str, error: str, result: object = None) -> None:
+ super().__init__(message)
+ #: Error message from the producer
+ self.error = error
+ #: Results read before the error, if the reader keeps them
+ self.result = result
+
+
+class JellyTruncatedResultsWarning(UserWarning):
+ """
+ Issued when a stream of SPARQL results ends without a trailer.
+
+ Some results may be missing, e.g., because the writer stopped or the connection
+ was lost.
+ """
diff --git a/pyjelly/integrations/generic/serialize.py b/pyjelly/integrations/generic/serialize.py
index 7a6feb38..cef90cb5 100644
--- a/pyjelly/integrations/generic/serialize.py
+++ b/pyjelly/integrations/generic/serialize.py
@@ -30,6 +30,33 @@
class GenericSinkTermEncoder(TermEncoder):
+ def same_term(self, a: object, b: object) -> bool:
+ """
+ Check whether two terms are equal, to find repeated terms.
+
+ IRIs, blank nodes and literals are compared field by field, which is
+ cheaper than calling their __eq__.
+ """
+ if a is b:
+ return True
+ cls = type(a)
+ if cls is not type(b):
+ return False
+ if cls is IRI:
+ return cast(IRI, a)._iri == cast(IRI, b)._iri
+ if cls is BlankNode:
+ return cast(BlankNode, a)._id == cast(BlankNode, b)._id
+ if cls is Literal:
+ x = cast(Literal, a)
+ y = cast(Literal, b)
+ return (
+ x._lex == y._lex
+ and x._langtag == y._langtag
+ and x._datatype == y._datatype
+ and x._direction == y._direction
+ )
+ return a == b
+
def encode_spo(self, term: object, out: EncodedTerm) -> None:
"""
Encode term based on its GenericSink object.
diff --git a/pyjelly/integrations/generic/sparql.py b/pyjelly/integrations/generic/sparql.py
new file mode 100644
index 00000000..a707918a
--- /dev/null
+++ b/pyjelly/integrations/generic/sparql.py
@@ -0,0 +1,153 @@
+"""Jelly-SPARQL support for the generic API (without rdflib)."""
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Iterator, Sequence
+from typing import IO
+
+from pyjelly.integrations.generic.decode import GenericStatementSinkAdapter
+from pyjelly.integrations.generic.generic_sink import Node
+from pyjelly.integrations.generic.serialize import GenericSinkTermEncoder
+from pyjelly.options import SparqlOptions
+from pyjelly.parse.sparql import SparqlResultSet, read_result_set, read_result_sets
+from pyjelly.serialize.sparql import FrameWriter, SparqlStream, write_solutions
+
+
+def parse_jelly_sparql(inp: IO[bytes] | bytes) -> SparqlResultSet:
+ """
+ Read SPARQL query results from Jelly-SPARQL.
+
+ Rows are read during iteration (only once), so the input must stay open
+ until then.
+
+ Args:
+ inp (IO[bytes] | bytes): the stream, delimited or not. It must have one
+ result set (the FLAT stream type, the default).
+
+ Returns:
+ SparqlResultSet: the results. Iterating it gives the rows, as tuples with
+ one value per variable, None if the variable is unbound.
+
+ Raises:
+ JellyIncompleteResultsError: if the writer reported that the results are
+ incomplete. Raised at the end of the iteration, or right away for a
+ boolean result.
+
+ """
+ return read_result_set(inp, GenericStatementSinkAdapter)
+
+
+def parse_jelly_sparql_sequence(inp: IO[bytes] | bytes) -> Iterator[SparqlResultSet]:
+ """
+ Read a sequence of SPARQL query results from one Jelly-SPARQL stream.
+
+ This reads PUNCTUATED streams, which hold any number of results. A FLAT
+ stream (the default) holds one. Rows are read during iteration. If the next
+ result set is read first, the unread rows are kept in memory.
+
+ Args:
+ inp (IO[bytes] | bytes): the stream, delimited.
+
+ Yields:
+ SparqlResultSet: the results, one per result set.
+
+ """
+ return read_result_sets(inp, GenericStatementSinkAdapter)
+
+
+def sparql_stream(options: SparqlOptions | None = None) -> SparqlStream:
+ """Create a Jelly-SPARQL writer for generic API terms."""
+ if options is None:
+ options = SparqlOptions()
+ encoder = GenericSinkTermEncoder(lookup_preset=options.lookup_preset)
+ return SparqlStream(encoder, options)
+
+
+def serialize_jelly_sparql(
+ variables: Iterable[str],
+ rows: Iterable[Sequence[Node | None]],
+ out: IO[bytes],
+ options: SparqlOptions | None = None,
+ *,
+ links: Sequence[str] = (),
+) -> None:
+ """
+ Write a solution sequence (the result of a SELECT query) in Jelly-SPARQL.
+
+ If the rows fail midway (e.g., the query fails), the stream is ended with the
+ error so readers know the results are incomplete, and the exception is
+ re-raised.
+
+ Args:
+ variables (Iterable[str]): variable names, in projection order.
+ rows (Iterable[Sequence[Node | None]]): the rows, with one value per
+ variable, None if the variable is unbound.
+ out (IO[bytes]): output stream.
+ options (SparqlOptions | None): options of the stream.
+ links (Sequence[str]): link IRIs for the results.
+
+ """
+ stream = sparql_stream(options)
+ writer = FrameWriter(out, delimited=stream.options.delimited)
+ write_solutions(stream, variables, rows, writer, links=links)
+
+
+def serialize_jelly_sparql_ask(
+ value: bool, # noqa: FBT001
+ out: IO[bytes],
+ options: SparqlOptions | None = None,
+ *,
+ links: Sequence[str] = (),
+) -> None:
+ """
+ Write a boolean result (of an ASK query) in Jelly-SPARQL.
+
+ Args:
+ value (bool): the result.
+ out (IO[bytes]): output stream.
+ options (SparqlOptions | None): options of the stream.
+ links (Sequence[str]): link IRIs for the result.
+
+ """
+ stream = sparql_stream(options)
+ FrameWriter(out, delimited=stream.options.delimited).write(
+ stream.ask(value, links=links)
+ )
+
+
+#: One result in a sequence: (variables, rows) for a SELECT query, or a bool for an
+#: ASK query
+SequenceItem = tuple[Iterable[str], Iterable[Sequence[Node | None]]] | bool
+
+
+def serialize_jelly_sparql_sequence(
+ results: Iterable[SequenceItem],
+ out: IO[bytes],
+ options: SparqlOptions | None = None,
+) -> None:
+ """
+ Write a sequence of SPARQL query results in one Jelly-SPARQL stream.
+
+ The stream is PUNCTUATED, with one result set per result.
+
+ Args:
+ results (Iterable[SequenceItem]): the results, each a (variables, rows)
+ pair for a SELECT query, or a bool for an ASK query.
+ out (IO[bytes]): output stream.
+ options (SparqlOptions | None): options of the stream, which must be
+ punctuated. Defaults to SparqlOptions(punctuated=True).
+
+ """
+ if options is None:
+ options = SparqlOptions(punctuated=True)
+ if not options.punctuated:
+ msg = "a sequence of results needs SparqlOptions(punctuated=True)"
+ raise ValueError(msg)
+ stream = sparql_stream(options)
+ writer = FrameWriter(out, delimited=True)
+ for result in results:
+ if isinstance(result, bool):
+ writer.write(stream.ask(result))
+ else:
+ variables, rows = result
+ write_solutions(stream, variables, rows, writer)
diff --git a/pyjelly/integrations/rdflib/decode.py b/pyjelly/integrations/rdflib/decode.py
index 68729c92..b7d45fc0 100644
--- a/pyjelly/integrations/rdflib/decode.py
+++ b/pyjelly/integrations/rdflib/decode.py
@@ -10,11 +10,12 @@
from collections.abc import Iterable
from itertools import chain
from typing import Any, Final
-from typing_extensions import override
+from typing_extensions import Never, override
import rdflib
from mypy_extensions import mypyc_attr
from rdflib.graph import DATASET_DEFAULT_GRAPH_ID
+from rdflib.term import Variable
from pyjelly.errors import JellyConformanceError
from pyjelly.integrations.rdflib.statements import Prefix, Quad, Triple
@@ -73,6 +74,22 @@ def literal(
def namespace_declaration(self, name: str, iri: rdflib.URIRef) -> Prefix:
return Prefix(name, iri)
+ @override
+ def directional_literal(self, lex: str, language: str, direction: str) -> Never:
+ msg = (
+ "rdflib does not support literals with a base direction (RDF 1.2). "
+ "Read the data with the generic API instead."
+ )
+ raise NotImplementedError(msg)
+
+ @override
+ def quoted_triple(self, terms: Iterable[Any]) -> Never:
+ msg = (
+ "rdflib does not support triple terms (RDF 1.2) and quoted triples "
+ "(RDF-star). Read the data with the generic API instead."
+ )
+ raise NotImplementedError(msg)
+
@mypyc_attr(allow_interpreted_subclasses=True)
class RDFLibTriplesAdapter(RDFLibAdapter):
@@ -152,3 +169,36 @@ def triple(self, terms: Iterable[Any]) -> Quad:
@override
def graph_end(self) -> None:
self._graph_id = None
+
+
+class BindingRows:
+ """
+ Row factory that turns the cells of Jelly-SPARQL frames into rdflib bindings.
+
+ A binding is a dict from each bound Variable to its value, as in rdflib's query
+ results. Equal names share one Variable object (see variables()).
+ """
+
+ def __init__(self) -> None:
+ self.names: list[str] = []
+ self.vars: list[Variable] = []
+
+ def variables(self, names: list[str]) -> list[Variable]:
+ """Get the Variable objects for these names."""
+ if names != self.names:
+ self.names = names
+ self.vars = [Variable(name) for name in names]
+ return self.vars
+
+ def __call__(
+ self, names: list[str], cells: list[list[Any]], rows: int
+ ) -> list[Any]:
+ bindings: list[dict[Variable, Any]] = [{} for _ in range(rows)]
+ variables = self.variables(names)
+ for j in range(len(cells)):
+ variable = variables[j]
+ # Iterating a list is cheaper than indexing it
+ for r, value in enumerate(cells[j]):
+ if value is not None:
+ bindings[r][variable] = value
+ return bindings
diff --git a/pyjelly/integrations/rdflib/encode.py b/pyjelly/integrations/rdflib/encode.py
index 7ac4d5f5..7e0b3034 100644
--- a/pyjelly/integrations/rdflib/encode.py
+++ b/pyjelly/integrations/rdflib/encode.py
@@ -16,6 +16,17 @@
from pyjelly.serialize.encode import EncodedTerm, TermEncoder
+def _same_str(a: object, b: object) -> bool:
+ """Compare two optional strings (e.g., URIRefs) as plain strings."""
+ if a is b:
+ return True
+ if a is None or b is None:
+ return False
+ a_str: str = cast("str", a)
+ b_str: str = cast("str", b)
+ return a_str == b_str
+
+
@mypyc_attr(allow_interpreted_subclasses=True)
class RDFLibTermEncoder(TermEncoder):
def same_term(self, a: object, b: object) -> bool:
@@ -26,8 +37,9 @@ def same_term(self, a: object, b: object) -> bool:
by comparing the terms as strings:
- Two URIRefs or two BNodes are equal if their strings are.
- - Two Literals are never equal if their lexical forms differ. If they are
- the same, the language tag and datatype matter too: rdflib decides.
+ - Two Literals with different lexical forms are never equal. With the same
+ lexical form, language tag and datatype, they are equal. Otherwise rdflib
+ decides (e.g., for language tags that differ only in case).
- Terms of different classes among these three are never equal.
"""
if a is b:
@@ -44,6 +56,13 @@ def same_term(self, a: object, b: object) -> bool:
b_lex: str = cast("str", b)
if a_lex != b_lex:
return False
+ # Slots behind the language and datatype properties
+ a_node: Any = a
+ b_node: Any = b
+ if _same_str(a_node._language, b_node._language) and _same_str(
+ a_node._datatype, b_node._datatype
+ ):
+ return True
elif (cls is URIRef or cls is BNode or cls is Literal) and (
other_cls is URIRef or other_cls is BNode or other_cls is Literal
):
diff --git a/pyjelly/integrations/rdflib/parse.py b/pyjelly/integrations/rdflib/parse.py
index b33e81f4..4ec1db57 100644
--- a/pyjelly/integrations/rdflib/parse.py
+++ b/pyjelly/integrations/rdflib/parse.py
@@ -5,7 +5,7 @@
from typing import IO, Final, cast
from rdflib import Node
-from rdflib.graph import ConjunctiveGraph, Dataset, Graph
+from rdflib.graph import DATASET_DEFAULT_GRAPH_ID, ConjunctiveGraph, Dataset, Graph
from rdflib.parser import InputSource
from rdflib.parser import Parser as RDFLibParser
@@ -224,6 +224,7 @@ def parse_jelly_to_graph(
dataset_factory: Callable[[], Dataset] = Dataset,
*,
validate_iris: bool = False,
+ default_graph: Graph | None = None,
) -> Graph | Dataset:
"""
Add statements from Generator to provided Graph/Dataset.
@@ -238,6 +239,9 @@ def parse_jelly_to_graph(
but you can pass something else here.
validate_iris (bool): whether rdflib checks the IRIs. An invalid IRI is
still read, rdflib only logs a warning. Checking slows down parsing.
+ default_graph (Graph | None): graph for the quads in the default graph,
+ in the same store as the Dataset. By default, the Dataset's default
+ graph.
Returns:
Dataset | Graph: Dataset or Graph with statements.
@@ -260,7 +264,7 @@ def parse_jelly_to_graph(
inp=inp, frames=frames, options=options, validate_iris=validate_iris
)
if to_store:
- _add_to_store(sink, items)
+ _add_to_store(sink, items, default_graph)
return sink
# The Graph or Dataset class changes add(), so it gets every statement
@@ -271,7 +275,10 @@ def parse_jelly_to_graph(
sink.add(item)
if isinstance(item, Quad):
s, p, o, graph_name = item
- context = cast(Dataset, sink).get_context(graph_name)
+ if default_graph is not None and graph_name == DATASET_DEFAULT_GRAPH_ID:
+ context = default_graph
+ else:
+ context = cast(Dataset, sink).get_context(graph_name)
sink.add((s, p, o, context)) # type: ignore[arg-type]
return sink
@@ -289,8 +296,16 @@ def _context(
return context
-def _add_to_store(sink: Graph, items: Iterable[Statement | Prefix]) -> None:
- """Add statements to the Graph's or Dataset's store directly, in batches."""
+def _add_to_store(
+ sink: Graph,
+ items: Iterable[Statement | Prefix],
+ default_graph: Graph | None = None,
+) -> None:
+ """
+ Add statements to the Graph's or Dataset's store directly, in batches.
+
+ Quads in the default graph go to `default_graph`, if given.
+ """
store = sink.store
batch: list[tuple[Node, Node, Node, Graph]] = []
contexts: dict[GraphName, Graph] = {}
@@ -302,7 +317,10 @@ def _add_to_store(sink: Graph, items: Iterable[Statement | Prefix]) -> None:
if isinstance(item, Quad):
s, p, o, graph_name = item
if graph_name is not last_graph_name:
- context = _context(cast(Dataset, sink), contexts, graph_name)
+ if default_graph is not None and graph_name == DATASET_DEFAULT_GRAPH_ID:
+ context = default_graph
+ else:
+ context = _context(cast(Dataset, sink), contexts, graph_name)
last_graph_name = graph_name
batch.append((s, p, o, context))
elif isinstance(item, Triple):
@@ -421,9 +439,14 @@ def parse(
if inp is None:
msg = "expected source to be a stream of bytes"
raise TypeError(msg)
+ # The default graph of a stream of quads goes into the Graph that rdflib
+ # parses into, as with TriG and N-Quads: for Dataset.parse(), that is the
+ # Dataset's default graph
+ target = Graph(store=sink.store, identifier=sink.identifier)
parse_jelly_to_graph(
inp,
- graph_factory=lambda: Graph(store=sink.store, identifier=sink.identifier),
+ graph_factory=lambda: target,
dataset_factory=lambda: Dataset(store=sink.store),
validate_iris=validate_iris,
+ default_graph=target,
)
diff --git a/pyjelly/integrations/rdflib/serialize.py b/pyjelly/integrations/rdflib/serialize.py
index 63d31e81..d2c5e8e2 100644
--- a/pyjelly/integrations/rdflib/serialize.py
+++ b/pyjelly/integrations/rdflib/serialize.py
@@ -8,6 +8,7 @@
from pyjelly.integrations.rdflib.statements import Quad, Triple
from pyjelly.integrations.rdflib.walk import (
+ dataset_quads,
gspo_trees,
spo_tree,
write_tree,
@@ -16,7 +17,7 @@
from pyjelly.options import StreamParameters
from rdflib import Graph
-from rdflib.graph import Dataset, QuotedGraph
+from rdflib.graph import ConjunctiveGraph, Dataset, QuotedGraph
from rdflib.serializer import Serializer as RDFLibSerializer
@@ -64,8 +65,8 @@ def triples_stream_frames(
Graph/Dataset/Statements to serialize.
Notes:
- if Dataset is given, its graphs are unpacked and iterated over
- if flow is GraphsFrameFlow, emits a frame per graph.
+ if Dataset (or ConjunctiveGraph) is given, its graphs are unpacked and
+ iterated over. If flow is GraphsFrameFlow, emits a frame per graph.
Yields:
Generator[jelly.RdfStreamFrame]: jelly frames.
@@ -75,7 +76,13 @@ def triples_stream_frames(
if isinstance(data, Graph) and stream.options.params.namespace_declarations:
yield from namespace_declarations(data, stream)
- graphs = (data,) if not isinstance(data, Dataset) else data.graphs()
+ graphs: Iterable[Graph | Iterable[Triple]]
+ if isinstance(data, Dataset):
+ graphs = data.graphs()
+ elif isinstance(data, ConjunctiveGraph):
+ graphs = data.contexts()
+ else:
+ graphs = (data,)
for graph in graphs:
tree = spo_tree(graph) if isinstance(graph, Graph) else None
if tree is not None:
@@ -100,6 +107,8 @@ def quads_stream_frames(
Notes:
Emits one frame per dataset if flow is of DatasetsFrameFlow.
+ A ConjunctiveGraph is written as a Dataset, with its default context
+ as the default graph.
Args:
stream (QuadStream): stream that specifies quads processing
@@ -112,15 +121,13 @@ def quads_stream_frames(
stream.enroll()
if stream.options.params.namespace_declarations:
yield from namespace_declarations(data, stream) # type: ignore[arg-type]
- trees = gspo_trees(data) if isinstance(data, Dataset) else None
+ trees = gspo_trees(data) if isinstance(data, ConjunctiveGraph) else None
if trees is not None:
# An in-memory dataset, grouped by graph, subject and predicate
yield from write_trees(stream, trees)
else:
# Any other store, or a generator of quads, in its own order
- quads = (
- data.quads((None, None, None, None)) if isinstance(data, Dataset) else data
- )
+ quads = dataset_quads(data) if isinstance(data, ConjunctiveGraph) else data
yield from stream.quads(quads)
if frame := stream.flow.frame_from_dataset():
yield frame
@@ -141,8 +148,13 @@ def guess_options(sink: Graph | Dataset, *, grouped: bool = False) -> Serializer
2
>>> guess_options(Dataset(), grouped=True).logical_type
4
+
+ A ConjunctiveGraph (e.g., the one rdfpipe reads into) holds quads too:
+
+ >>> guess_options(ConjunctiveGraph()).logical_type
+ 2
"""
- if isinstance(sink, Dataset):
+ if isinstance(sink, ConjunctiveGraph):
logical_type = (
jelly.LOGICAL_STREAM_TYPE_DATASETS
if grouped
@@ -164,8 +176,8 @@ def guess_stream(options: SerializerOptions, sink: Graph | Dataset) -> Stream:
"""
Return an appropriate stream implementation for the given options.
- Notes: if base(!) logical type is GRAPHS and Dataset is given,
- initializes TripleStream
+ Notes: if base(!) logical type is GRAPHS and Dataset (or ConjunctiveGraph)
+ is given, initializes TripleStream
>>> graph_ser = RDFLibJellySerializer(Graph())
>>> ds_ser = RDFLibJellySerializer(Dataset())
@@ -177,7 +189,7 @@ def guess_stream(options: SerializerOptions, sink: Graph | Dataset) -> Stream:
"""
stream_cls: type[Stream]
if (options.logical_type % 10) != jelly.LOGICAL_STREAM_TYPE_GRAPHS and isinstance(
- sink, Dataset
+ sink, ConjunctiveGraph
):
stream_cls = QuadStream
else:
diff --git a/pyjelly/integrations/rdflib/sparql.py b/pyjelly/integrations/rdflib/sparql.py
new file mode 100644
index 00000000..ccdfa9e8
--- /dev/null
+++ b/pyjelly/integrations/rdflib/sparql.py
@@ -0,0 +1,375 @@
+"""
+Jelly-SPARQL support for rdflib's SPARQL query results (rdflib.query.Result).
+
+With pyjelly installed, rdflib reads and writes results in the "jelly" format:
+
+ result = graph.query("SELECT ?s ?o WHERE { ?s ?p ?o }")
+ result.serialize("results.jellys", format="jelly")
+ result = Result.parse(open("results.jellys", "rb"), format="jelly")
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Iterator, Mapping
+from pathlib import Path
+from typing import IO, cast
+from typing_extensions import override
+
+from rdflib.query import Result, ResultParser, ResultSerializer
+from rdflib.term import Identifier, Variable
+
+from pyjelly.errors import JellyConformanceError, JellyIncompleteResultsError
+from pyjelly.integrations.rdflib.decode import BindingRows, RDFLibAdapter
+from pyjelly.integrations.rdflib.encode import RDFLibTermEncoder
+from pyjelly.options import SparqlOptions
+from pyjelly.parse.decode import Adapter, ParserOptions
+from pyjelly.parse.sparql import (
+ AdapterFactory,
+ SparqlResultSet,
+ read_result_set,
+ read_result_sets,
+)
+from pyjelly.serialize.sparql import FrameWriter, SparqlStream, write_bindings
+
+#: Trailer error written for results that were read from a truncated stream
+TRUNCATED_ERROR = (
+ "the results were read from a stream that ended without indicating that they are "
+ "complete, so some rows may be missing"
+)
+
+
+class JellySparqlResult(Result):
+ """
+ An rdflib Result read from Jelly-SPARQL, with a few extra attributes.
+
+ Attributes:
+ links: link IRIs for the results, as in head.link of SPARQL JSON results.
+ truncated: whether the stream ended without a trailer, so some results may
+ be missing.
+ error: error message from the writer, if the results are incomplete. Only
+ set by parse_jelly_sparql_sequence(). parse_jelly_sparql() raises
+ JellyIncompleteResultsError instead.
+
+ """
+
+ def __init__(self, type_: str) -> None:
+ super().__init__(type_)
+ self.links: list[str] = []
+ self.truncated = False
+ self.error = ""
+
+
+def _adapter_factory(*, validate_iris: bool) -> AdapterFactory:
+ def make(options: ParserOptions) -> Adapter:
+ return RDFLibAdapter(options, validate_iris=validate_iris)
+
+ return make
+
+
+def _to_result(
+ result_set: SparqlResultSet,
+ binding_rows: BindingRows,
+ *,
+ lazy: bool,
+ raise_error: bool,
+) -> JellySparqlResult:
+ """Convert a result set with bindings as rows to an rdflib Result."""
+ if result_set.is_boolean:
+ result = JellySparqlResult("ASK")
+ # Read to the end of the stream: nothing may follow a boolean result
+ result_set.keep()
+ result.links = result_set.links
+ try:
+ result.askAnswer = result_set.boolean
+ except JellyIncompleteResultsError as error:
+ if raise_error:
+ error.result = result
+ raise
+ # Keep the value, even though the result is incomplete
+ result.askAnswer = result_set._boolean
+ result.error = error.error
+ result.truncated = result_set.truncated
+ return result
+ result = JellySparqlResult("SELECT")
+ # The same Variable objects as the keys of the bindings
+ result.vars = binding_rows.variables(result_set.variables)
+ result.links = result_set.links
+ if lazy:
+ # rdflib builds the bindings during iteration, as for its own query
+ # results
+ result.bindings = _read_bindings(result_set, result, raise_error=raise_error)
+ return result
+ bindings: list[Mapping[Variable, Identifier]] = []
+ result.bindings = bindings
+ try:
+ result_set.collect(bindings)
+ except JellyIncompleteResultsError as error:
+ # The result has the rows read before the error
+ _incomplete(result, error, raise_error=raise_error)
+ finally:
+ result.links = result_set.links
+ result.truncated = result_set.truncated
+ return result
+
+
+def _incomplete(
+ result: JellySparqlResult, error: JellyIncompleteResultsError, *, raise_error: bool
+) -> None:
+ result.error = error.error
+ if raise_error:
+ error.result = result
+ raise error
+
+
+def _read_bindings(
+ result_set: SparqlResultSet, result: JellySparqlResult, *, raise_error: bool
+) -> Iterator[Mapping[Variable, Identifier]]:
+ """Yield the bindings, then record the links, truncation and error."""
+ try:
+ yield from result_set
+ except JellyIncompleteResultsError as error:
+ _incomplete(result, error, raise_error=raise_error)
+ finally:
+ result.links = result_set.links
+ result.truncated = result_set.truncated
+
+
+def parse_jelly_sparql(
+ inp: IO[bytes] | bytes, *, validate_iris: bool = False, lazy: bool = False
+) -> JellySparqlResult:
+ """
+ Read SPARQL query results from Jelly-SPARQL.
+
+ Args:
+ inp (IO[bytes] | bytes): the stream, delimited or not. It must have one
+ result set (the FLAT stream type, the default).
+ validate_iris (bool): whether rdflib checks the IRIs. An invalid IRI is
+ still read, rdflib only logs a warning. Checking slows down parsing.
+ lazy (bool): read the rows during iteration instead of all at once. The
+ stream must stay open until the result has been iterated, and the
+ error of incomplete results is raised at the end of the iteration.
+
+ Returns:
+ JellySparqlResult: the results, an rdflib Result.
+
+ Raises:
+ JellyIncompleteResultsError: if the writer said that the results are
+ incomplete, e.g., because the query failed midway. Its `result` holds
+ the results read before the error.
+
+ """
+ adapter_factory = _adapter_factory(validate_iris=validate_iris)
+ binding_rows = BindingRows()
+ try:
+ result_set = read_result_set(inp, adapter_factory, row_factory=binding_rows)
+ except JellyIncompleteResultsError as error:
+ # An incomplete boolean result
+ result_set = cast("SparqlResultSet", error.result)
+ error.result = _to_result(
+ result_set, binding_rows, lazy=False, raise_error=False
+ )
+ raise
+ return _to_result(result_set, binding_rows, lazy=lazy, raise_error=True)
+
+
+def parse_jelly_sparql_sequence(
+ inp: IO[bytes] | bytes, *, validate_iris: bool = False
+) -> Iterator[JellySparqlResult]:
+ """
+ Read a sequence of SPARQL query results from one Jelly-SPARQL stream.
+
+ This reads PUNCTUATED streams, which hold any number of results. A FLAT
+ stream (the default) holds one.
+
+ An incomplete result does not stop the sequence. Instead, its `error`
+ attribute is set, so check it. The results after it are still read.
+
+ Args:
+ inp (IO[bytes] | bytes): the stream, delimited.
+ validate_iris (bool): whether rdflib checks the IRIs, see
+ parse_jelly_sparql().
+
+ Yields:
+ JellySparqlResult: the results, an rdflib Result for each result set.
+
+ """
+ adapter_factory = _adapter_factory(validate_iris=validate_iris)
+ binding_rows = BindingRows()
+ for result_set in read_result_sets(inp, adapter_factory, row_factory=binding_rows):
+ yield _to_result(result_set, binding_rows, lazy=False, raise_error=False)
+
+
+def sparql_stream(options: SparqlOptions | None = None) -> SparqlStream:
+ """Create a Jelly-SPARQL writer for rdflib terms."""
+ if options is None:
+ options = SparqlOptions()
+ encoder = RDFLibTermEncoder(lookup_preset=options.lookup_preset)
+ return SparqlStream(encoder, options)
+
+
+def _passed_on_error(result: Result) -> str:
+ """Get the error to pass on for a result read from Jelly-SPARQL, if any."""
+ error = getattr(result, "error", "")
+ if not error and getattr(result, "truncated", False):
+ return TRUNCATED_ERROR
+ return str(error)
+
+
+def _write_result(stream: SparqlStream, result: Result, writer: FrameWriter) -> None:
+ """Write the result of a SELECT or ASK query as one result set."""
+ links = getattr(result, "links", ())
+ if result.type == "ASK":
+ if result.askAnswer is None:
+ msg = "the result of the ASK query has no answer (askAnswer is None)"
+ raise JellyConformanceError(msg)
+ frame = stream.ask(
+ bool(result.askAnswer), links=links, error=_passed_on_error(result)
+ )
+ writer.write(frame)
+ return
+ if result.type != "SELECT":
+ msg = (
+ f"Jelly-SPARQL is for the results of SELECT and ASK queries, not "
+ f"{result.type}. Write graphs in Jelly-RDF instead."
+ )
+ raise JellyConformanceError(msg)
+ if result.vars is None:
+ msg = "the result of the SELECT query has no variables (vars is None)"
+ raise JellyConformanceError(msg)
+ # Check for the generator first, as reading result.bindings would build all
+ # bindings at once
+ made = getattr(result, "_genbindings", None)
+ bindings: Iterable[Mapping[Variable, Identifier]] = (
+ result.bindings if made is None else _made_bindings(result, made)
+ )
+ write_bindings(
+ stream,
+ result.vars,
+ bindings,
+ writer,
+ links=links,
+ error=_passed_on_error(result),
+ )
+
+
+def _made_bindings(
+ result: Result, made: Iterator[Mapping[Variable, Identifier]]
+) -> Iterator[Mapping[Variable, Identifier]]:
+ """
+ Yield the bindings of a result as its generator produces them.
+
+ Result.bindings would build all of them before anything is written, and if one
+ failed, nothing would be written at all. As in Result.__iter__(), the bindings
+ are also kept in the result.
+ """
+ result._genbindings = None
+ kept = result._bindings
+ # Bindings already built, e.g. by an earlier loop over the result that stopped
+ yield from list(kept)
+ for binding in made:
+ kept.append(binding)
+ yield binding
+
+
+def serialize_jelly_sparql(
+ result: Result, out: IO[bytes], options: SparqlOptions | None = None
+) -> None:
+ """
+ Write SPARQL query results (of a SELECT or ASK query) in Jelly-SPARQL.
+
+ If the results fail midway (e.g., the query fails), the stream is ended with
+ the error so readers know the results are incomplete, and the exception is
+ re-raised. Results read from Jelly-SPARQL that were incomplete or truncated
+ are written as incomplete.
+
+ Args:
+ result (Result): the results.
+ out (IO[bytes]): output stream.
+ options (SparqlOptions | None): options of the stream.
+
+ """
+ stream = sparql_stream(options)
+ _write_result(stream, result, FrameWriter(out, delimited=stream.options.delimited))
+
+
+def serialize_jelly_sparql_sequence(
+ results: Iterable[Result], out: IO[bytes], options: SparqlOptions | None = None
+) -> None:
+ """
+ Write a sequence of SPARQL query results in one Jelly-SPARQL stream.
+
+ The stream is PUNCTUATED, with one result set per result.
+
+ Args:
+ results (Iterable[Result]): the results.
+ out (IO[bytes]): output stream.
+ options (SparqlOptions | None): options of the stream, which must be
+ punctuated. Defaults to SparqlOptions(punctuated=True).
+
+ """
+ if options is None:
+ options = SparqlOptions(punctuated=True)
+ if not options.punctuated:
+ msg = "a sequence of results needs SparqlOptions(punctuated=True)"
+ raise ValueError(msg)
+ stream = sparql_stream(options)
+ writer = FrameWriter(out, delimited=True)
+ for result in results:
+ _write_result(stream, result, writer)
+
+
+class RDFLibJellySparqlParser(ResultParser):
+ """rdflib parser plugin for Jelly-SPARQL results, used by Result.parse()."""
+
+ @override
+ def parse( # type: ignore[override]
+ self,
+ source: IO[bytes] | bytes | str | Path,
+ content_type: str | None = None,
+ *,
+ validate_iris: bool = False,
+ lazy: bool = False,
+ ) -> Result:
+ """
+ Read the results.
+
+ Args:
+ source (IO[bytes] | bytes | str | Path): a stream, bytes, or a file
+ path.
+ content_type (str | None): unused.
+ validate_iris (bool): whether rdflib checks the IRIs, see
+ parse_jelly_sparql().
+ lazy (bool): read the rows during iteration, see parse_jelly_sparql().
+ Only for streams, not paths.
+
+ """
+ if not isinstance(source, (str, Path)):
+ return parse_jelly_sparql(source, validate_iris=validate_iris, lazy=lazy)
+ if lazy:
+ msg = "lazy reading needs a stream that stays open, not a path"
+ raise ValueError(msg)
+ with Path(source).open("rb") as inp:
+ return parse_jelly_sparql(inp, validate_iris=validate_iris)
+
+
+class RDFLibJellySparqlSerializer(ResultSerializer):
+ """rdflib serializer plugin for Jelly-SPARQL, used by Result.serialize()."""
+
+ @override
+ def serialize( # type: ignore[override]
+ self,
+ stream: IO[bytes],
+ encoding: str = "utf-8",
+ *,
+ options: SparqlOptions | None = None,
+ ) -> None:
+ """
+ Write the results.
+
+ Args:
+ stream (IO[bytes]): output stream.
+ encoding (str): unused, as Jelly is binary.
+ options (SparqlOptions | None): options of the stream.
+
+ """
+ serialize_jelly_sparql(self.result, stream, options)
diff --git a/pyjelly/integrations/rdflib/walk.py b/pyjelly/integrations/rdflib/walk.py
index 8732d456..d8b11c98 100644
--- a/pyjelly/integrations/rdflib/walk.py
+++ b/pyjelly/integrations/rdflib/walk.py
@@ -11,7 +11,8 @@
from collections.abc import Callable, Iterable, Iterator, Mapping
from typing import Any, NamedTuple
-from rdflib import Dataset, Graph
+from rdflib import Graph
+from rdflib.graph import DATASET_DEFAULT_GRAPH_ID, ConjunctiveGraph, Dataset
from rdflib.plugins.stores.memory import Memory
from rdflib.store import Store
@@ -108,12 +109,13 @@ def spo_tree(graph: Graph) -> Tree | None:
return _group(triples)
-def gspo_trees(dataset: Dataset) -> Iterator[tuple[Any, Tree]] | None:
+def gspo_trees(dataset: ConjunctiveGraph) -> Iterator[tuple[Any, Tree]] | None:
"""
Return the dataset's graphs, each as its name and its triples as a tree.
Only for rdflib's in-memory store, see spo_tree(). Each graph is grouped only
- when it's its turn, so at most one graph's tree is in memory at a time.
+ when it's its turn, so at most one graph's tree is in memory at a time. The
+ default graph is named DATASET_DEFAULT_GRAPH_ID, see dataset_quads().
"""
store = dataset.store
if type(store) is not Memory:
@@ -121,17 +123,47 @@ def gspo_trees(dataset: Dataset) -> Iterator[tuple[Any, Tree]] | None:
indexes = _memory_indexes(store)
if indexes is None:
# The private fields are not there: group with the public API
- return _group_quads(dataset.quads((None, None, None, None)))
- return _memory_gspo_trees(indexes)
+ return _group_quads(dataset_quads(dataset))
+ return _memory_gspo_trees(indexes, _default_graph_name(dataset))
-def _memory_gspo_trees(indexes: _MemoryIndexes) -> Iterator[tuple[Any, Tree]]:
+def dataset_quads(dataset: ConjunctiveGraph) -> Iterator[tuple[Any, Any, Any, Any]]:
+ """
+ Return the dataset's quads, with graph names rather than Graph objects.
+
+ The default graph is named DATASET_DEFAULT_GRAPH_ID. A Dataset already names
+ it so, but a ConjunctiveGraph (e.g., the one rdfpipe reads into) names it
+ with a random blank node. The quads of that graph go into the default graph,
+ as with rdflib's TriG serializer.
+ """
+ default = _default_graph_name(dataset)
+ for s, p, o, g in dataset.quads((None, None, None, None)):
+ name = getattr(g, "identifier", g)
+ if name is None or name == default:
+ name = DATASET_DEFAULT_GRAPH_ID
+ yield s, p, o, name
+
+
+def _default_graph_name(dataset: ConjunctiveGraph) -> Any:
+ """Return the name that rdflib gives the default graph of the dataset."""
+ if isinstance(dataset, Dataset):
+ # Dataset.default_context gives a deprecation warning
+ return DATASET_DEFAULT_GRAPH_ID
+ return dataset.default_context.identifier
+
+
+def _memory_gspo_trees(
+ indexes: _MemoryIndexes, default: Any
+) -> Iterator[tuple[Any, Tree]]:
for context, triples in indexes.context_triples.items():
# None is not a graph, but an index of all triples in the store
if context is None or not triples:
continue
graph = indexes.context_objects[context]
- yield getattr(graph, "identifier", graph), _group(triples)
+ name = getattr(graph, "identifier", graph)
+ if name == default:
+ name = DATASET_DEFAULT_GRAPH_ID
+ yield name, _group(triples)
def _group_quads(
diff --git a/pyjelly/jelly/__init__.py b/pyjelly/jelly/__init__.py
index 1d092574..0688f8d2 100644
--- a/pyjelly/jelly/__init__.py
+++ b/pyjelly/jelly/__init__.py
@@ -1,5 +1,7 @@
-from pyjelly.jelly import rdf_pb2
+from pyjelly.jelly import rdf_pb2, sparql_pb2
from pyjelly.jelly.rdf_pb2 import *
+from pyjelly.jelly.sparql_pb2 import *
# workaround for mypyc
globals().update(vars(rdf_pb2))
+globals().update(vars(sparql_pb2))
diff --git a/pyjelly/jelly/sparql_pb2.py b/pyjelly/jelly/sparql_pb2.py
new file mode 100644
index 00000000..5e87e2d3
--- /dev/null
+++ b/pyjelly/jelly/sparql_pb2.py
@@ -0,0 +1,49 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler. DO NOT EDIT!
+# NO CHECKED-IN PROTOBUF GENCODE
+# source: sparql.proto
+# Protobuf Python Version: 6.31.0
+"""Generated protocol buffer code."""
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import descriptor_pool as _descriptor_pool
+from google.protobuf import runtime_version as _runtime_version
+from google.protobuf import symbol_database as _symbol_database
+from google.protobuf.internal import builder as _builder
+_runtime_version.ValidateProtobufRuntimeVersion(
+ _runtime_version.Domain.PUBLIC,
+ 6,
+ 31,
+ 0,
+ '',
+ 'sparql.proto'
+)
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+from pyjelly.jelly import rdf_pb2 as rdf__pb2
+
+
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0csparql.proto\x12(eu.ostrzyciel.jelly.core.proto.v1.sparql\x1a\trdf.proto\"\xae\x02\n\x14SparqlResultsOptions\x12\x13\n\x0bstream_name\x18\x01 \x01(\t\x12O\n\x0bstream_type\x18\x02 \x01(\x0e\x32:.eu.ostrzyciel.jelly.core.proto.v1.sparql.SparqlStreamType\x12\x42\n\x0brdf_version\x18\x05 \x01(\x0e\x32-.eu.ostrzyciel.jelly.core.proto.v1.RdfVersion\x12\x1b\n\x13max_name_table_size\x18\t \x01(\r\x12\x1d\n\x15max_prefix_table_size\x18\n \x01(\r\x12\x1f\n\x17max_datatype_table_size\x18\x0b \x01(\r\x12\x0f\n\x07version\x18\x0f \x01(\r\" \n\x0fSparqlAskResult\x12\r\n\x05value\x18\x01 \x01(\x08\"%\n\x14SparqlResultsTrailer\x12\r\n\x05\x65rror\x18\x01 \x01(\t\"\xd8\x05\n\x12SparqlResultsFrame\x12O\n\x07options\x18\x01 \x01(\x0b\x32>.eu.ostrzyciel.jelly.core.proto.v1.sparql.SparqlResultsOptions\x12\x11\n\tvariables\x18\x02 \x03(\t\x12\x11\n\trow_count\x18\x03 \x01(\r\x12\x46\n\x05names\x18\x04 \x03(\x0b\x32\x37.eu.ostrzyciel.jelly.core.proto.v1.RdfLookupEntryPacked\x12I\n\x08prefixes\x18\x05 \x03(\x0b\x32\x37.eu.ostrzyciel.jelly.core.proto.v1.RdfLookupEntryPacked\x12J\n\tdatatypes\x18\x06 \x03(\x0b\x32\x37.eu.ostrzyciel.jelly.core.proto.v1.RdfLookupEntryPacked\x12=\n\x07\x63olumns\x18\x07 \x03(\x0b\x32,.eu.ostrzyciel.jelly.core.proto.v1.RdfColumn\x12M\n\nask_result\x18\x08 \x01(\x0b\x32\x39.eu.ostrzyciel.jelly.core.proto.v1.sparql.SparqlAskResult\x12O\n\x07trailer\x18\t \x01(\x0b\x32>.eu.ostrzyciel.jelly.core.proto.v1.sparql.SparqlResultsTrailer\x12\\\n\x08metadata\x18\x0f \x03(\x0b\x32J.eu.ostrzyciel.jelly.core.proto.v1.sparql.SparqlResultsFrame.MetadataEntry\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x0c:\x02\x38\x01*R\n\x10SparqlStreamType\x12\x1b\n\x17SPARQL_STREAM_TYPE_FLAT\x10\x00\x12!\n\x1dSPARQL_STREAM_TYPE_PUNCTUATED\x10\x01\x62\x06proto3')
+
+_globals = globals()
+_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
+_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'sparql_pb2', _globals)
+if not _descriptor._USE_C_DESCRIPTORS:
+ DESCRIPTOR._loaded_options = None
+ _globals['_SPARQLRESULTSFRAME_METADATAENTRY']._loaded_options = None
+ _globals['_SPARQLRESULTSFRAME_METADATAENTRY']._serialized_options = b'8\001'
+ _globals['_SPARQLSTREAMTYPE']._serialized_start=1178
+ _globals['_SPARQLSTREAMTYPE']._serialized_end=1260
+ _globals['_SPARQLRESULTSOPTIONS']._serialized_start=70
+ _globals['_SPARQLRESULTSOPTIONS']._serialized_end=372
+ _globals['_SPARQLASKRESULT']._serialized_start=374
+ _globals['_SPARQLASKRESULT']._serialized_end=406
+ _globals['_SPARQLRESULTSTRAILER']._serialized_start=408
+ _globals['_SPARQLRESULTSTRAILER']._serialized_end=445
+ _globals['_SPARQLRESULTSFRAME']._serialized_start=448
+ _globals['_SPARQLRESULTSFRAME']._serialized_end=1176
+ _globals['_SPARQLRESULTSFRAME_METADATAENTRY']._serialized_start=1129
+ _globals['_SPARQLRESULTSFRAME_METADATAENTRY']._serialized_end=1176
+# @@protoc_insertion_point(module_scope)
diff --git a/pyjelly/jelly/sparql_pb2.pyi b/pyjelly/jelly/sparql_pb2.pyi
new file mode 100644
index 00000000..205a10d9
--- /dev/null
+++ b/pyjelly/jelly/sparql_pb2.pyi
@@ -0,0 +1,77 @@
+from pyjelly.jelly import rdf_pb2 as _rdf_pb2
+from google.protobuf.internal import containers as _containers
+from google.protobuf.internal import enum_type_wrapper as _enum_type_wrapper
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from collections.abc import Iterable as _Iterable, Mapping as _Mapping
+from typing import ClassVar as _ClassVar, Optional as _Optional, Union as _Union
+
+DESCRIPTOR: _descriptor.FileDescriptor
+
+class SparqlStreamType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
+ __slots__ = ()
+ SPARQL_STREAM_TYPE_FLAT: _ClassVar[SparqlStreamType]
+ SPARQL_STREAM_TYPE_PUNCTUATED: _ClassVar[SparqlStreamType]
+SPARQL_STREAM_TYPE_FLAT: SparqlStreamType
+SPARQL_STREAM_TYPE_PUNCTUATED: SparqlStreamType
+
+class SparqlResultsOptions(_message.Message):
+ __slots__ = ("stream_name", "stream_type", "rdf_version", "max_name_table_size", "max_prefix_table_size", "max_datatype_table_size", "version")
+ STREAM_NAME_FIELD_NUMBER: _ClassVar[int]
+ STREAM_TYPE_FIELD_NUMBER: _ClassVar[int]
+ RDF_VERSION_FIELD_NUMBER: _ClassVar[int]
+ MAX_NAME_TABLE_SIZE_FIELD_NUMBER: _ClassVar[int]
+ MAX_PREFIX_TABLE_SIZE_FIELD_NUMBER: _ClassVar[int]
+ MAX_DATATYPE_TABLE_SIZE_FIELD_NUMBER: _ClassVar[int]
+ VERSION_FIELD_NUMBER: _ClassVar[int]
+ stream_name: str
+ stream_type: SparqlStreamType
+ rdf_version: _rdf_pb2.RdfVersion
+ max_name_table_size: int
+ max_prefix_table_size: int
+ max_datatype_table_size: int
+ version: int
+ def __init__(self, stream_name: _Optional[str] = ..., stream_type: _Optional[_Union[SparqlStreamType, str]] = ..., rdf_version: _Optional[_Union[_rdf_pb2.RdfVersion, str]] = ..., max_name_table_size: _Optional[int] = ..., max_prefix_table_size: _Optional[int] = ..., max_datatype_table_size: _Optional[int] = ..., version: _Optional[int] = ...) -> None: ...
+
+class SparqlAskResult(_message.Message):
+ __slots__ = ("value",)
+ VALUE_FIELD_NUMBER: _ClassVar[int]
+ value: bool
+ def __init__(self, value: bool = ...) -> None: ...
+
+class SparqlResultsTrailer(_message.Message):
+ __slots__ = ("error",)
+ ERROR_FIELD_NUMBER: _ClassVar[int]
+ error: str
+ def __init__(self, error: _Optional[str] = ...) -> None: ...
+
+class SparqlResultsFrame(_message.Message):
+ __slots__ = ("options", "variables", "row_count", "names", "prefixes", "datatypes", "columns", "ask_result", "trailer", "metadata")
+ class MetadataEntry(_message.Message):
+ __slots__ = ("key", "value")
+ KEY_FIELD_NUMBER: _ClassVar[int]
+ VALUE_FIELD_NUMBER: _ClassVar[int]
+ key: str
+ value: bytes
+ def __init__(self, key: _Optional[str] = ..., value: _Optional[bytes] = ...) -> None: ...
+ OPTIONS_FIELD_NUMBER: _ClassVar[int]
+ VARIABLES_FIELD_NUMBER: _ClassVar[int]
+ ROW_COUNT_FIELD_NUMBER: _ClassVar[int]
+ NAMES_FIELD_NUMBER: _ClassVar[int]
+ PREFIXES_FIELD_NUMBER: _ClassVar[int]
+ DATATYPES_FIELD_NUMBER: _ClassVar[int]
+ COLUMNS_FIELD_NUMBER: _ClassVar[int]
+ ASK_RESULT_FIELD_NUMBER: _ClassVar[int]
+ TRAILER_FIELD_NUMBER: _ClassVar[int]
+ METADATA_FIELD_NUMBER: _ClassVar[int]
+ options: SparqlResultsOptions
+ variables: _containers.RepeatedScalarFieldContainer[str]
+ row_count: int
+ names: _containers.RepeatedCompositeFieldContainer[_rdf_pb2.RdfLookupEntryPacked]
+ prefixes: _containers.RepeatedCompositeFieldContainer[_rdf_pb2.RdfLookupEntryPacked]
+ datatypes: _containers.RepeatedCompositeFieldContainer[_rdf_pb2.RdfLookupEntryPacked]
+ columns: _containers.RepeatedCompositeFieldContainer[_rdf_pb2.RdfColumn]
+ ask_result: SparqlAskResult
+ trailer: SparqlResultsTrailer
+ metadata: _containers.ScalarMap[str, bytes]
+ def __init__(self, options: _Optional[_Union[SparqlResultsOptions, _Mapping]] = ..., variables: _Optional[_Iterable[str]] = ..., row_count: _Optional[int] = ..., names: _Optional[_Iterable[_Union[_rdf_pb2.RdfLookupEntryPacked, _Mapping]]] = ..., prefixes: _Optional[_Iterable[_Union[_rdf_pb2.RdfLookupEntryPacked, _Mapping]]] = ..., datatypes: _Optional[_Iterable[_Union[_rdf_pb2.RdfLookupEntryPacked, _Mapping]]] = ..., columns: _Optional[_Iterable[_Union[_rdf_pb2.RdfColumn, _Mapping]]] = ..., ask_result: _Optional[_Union[SparqlAskResult, _Mapping]] = ..., trailer: _Optional[_Union[SparqlResultsTrailer, _Mapping]] = ..., metadata: _Optional[_Mapping[str, bytes]] = ...) -> None: ...
diff --git a/pyjelly/options.py b/pyjelly/options.py
index c6bd34aa..5bd7cac0 100644
--- a/pyjelly/options.py
+++ b/pyjelly/options.py
@@ -51,6 +51,23 @@
MIMETYPES = ("application/x-jelly-rdf",)
+#: Media type of Jelly-SPARQL
+SPARQL_MIMETYPE: Final = "application/x-jelly-sparql"
+#: File extension of Jelly-SPARQL
+SPARQL_EXTENSION: Final = ".jellys"
+
+# Jelly-SPARQL has its own version tags, separate from Jelly-RDF. 1 is 1.0.
+SPARQL_VERSION_1_0: Final[int] = 1
+SPARQL_MAX_VERSION: Final[int] = SPARQL_VERSION_1_0
+
+#: Default maximum values (rows times variables) per frame when writing
+DEFAULT_SPARQL_FRAME_VALUES: Final[int] = 4096
+#: Maximum rows per frame accepted when reading
+MAX_SPARQL_FRAME_ROWS: Final[int] = 1 << 20
+#: Maximum values per frame accepted when reading. A frame is decoded all at once,
+#: so this caps its memory use.
+MAX_SPARQL_FRAME_VALUES: Final[int] = 1 << 24
+
def is_row_layout(version: int) -> bool:
"""
@@ -93,6 +110,14 @@ def small(cls) -> Self:
return cls(max_names=128, max_prefixes=32, max_datatypes=32)
+#: Default lookup sizes of Jelly-SPARQL streams. As in Jelly-JVM, they are larger
+#: than for Jelly-RDF, because a row of results usually has more distinct terms
+#: than a statement.
+SPARQL_LOOKUP_PRESET: Final = LookupPreset(
+ max_names=8192, max_prefixes=1024, max_datatypes=64
+)
+
+
@dataclass(frozen=True)
class StreamTypes:
physical_type: jelly.PhysicalStreamType = jelly.PHYSICAL_STREAM_TYPE_UNSPECIFIED
@@ -218,3 +243,57 @@ def validate_type_compatibility(
logical_type_name = jelly.LogicalStreamType.Name(logical_type)
msg = f"{physical_type_name} is not compatible with {logical_type_name}"
raise JellyAssertionError(msg)
+
+
+@dataclass(frozen=True)
+class SparqlOptions:
+ """
+ Options for writing a Jelly-SPARQL stream.
+
+ Attributes:
+ lookup_preset: lookup sizes. The name lookup needs at least 128 entries.
+ rdf_version: RDF version of the terms in the stream. RDF 1.1 allows no
+ base directions or triple terms, RDF 1.2 Basic no triple terms.
+ punctuated: write a sequence of result sets (the PUNCTUATED stream type)
+ instead of one result set.
+ frame_values: maximum values (rows times variables) per frame. A frame
+ can end earlier if the lookups fill up.
+ delimited: write delimited frames, which files and HTTP need. A
+ non-delimited stream is a single frame, e.g. one message for a message
+ broker. All terms of the results must then fit in the lookups.
+ stream_name: name of the stream.
+
+ """
+
+ lookup_preset: LookupPreset = SPARQL_LOOKUP_PRESET
+ rdf_version: jelly.RdfVersion = jelly.RDF_VERSION_UNSPECIFIED
+ punctuated: bool = False
+ frame_values: int = DEFAULT_SPARQL_FRAME_VALUES
+ delimited: bool = True
+ stream_name: str = ""
+
+ def __post_init__(self) -> None:
+ max_names = self.lookup_preset.max_names
+ if max_names < MIN_COLUMN_NAME_LOOKUP_SIZE:
+ msg = (
+ f"the name lookup size is {max_names}, but in Jelly-SPARQL it must "
+ f"be at least {MIN_COLUMN_NAME_LOOKUP_SIZE}"
+ )
+ raise JellyConformanceError(msg)
+ if self.frame_values < 1:
+ msg = f"frame_values must be at least 1, not {self.frame_values}"
+ raise ValueError(msg)
+ if self.punctuated and not self.delimited:
+ msg = "a PUNCTUATED stream must be delimited"
+ raise ValueError(msg)
+
+ def rows_per_frame(self, variables: int) -> int:
+ """
+ Compute the maximum rows per frame for this number of variables.
+
+ >>> SparqlOptions().rows_per_frame(4)
+ 1024
+ >>> SparqlOptions(frame_values=10).rows_per_frame(0)
+ 10
+ """
+ return max(1, self.frame_values // max(1, variables))
diff --git a/pyjelly/parse/columns.py b/pyjelly/parse/columns.py
index 0135d41b..03c5df58 100644
--- a/pyjelly/parse/columns.py
+++ b/pyjelly/parse/columns.py
@@ -96,6 +96,34 @@ def _check_kinds(kinds: bytes, value_count: int, position: str) -> None:
raise JellyConformanceError(msg)
+# Above this count, cells are added in bulk. Below it, appending them one at a
+# time is cheaper than building a temporary list.
+_FEW_CELLS: Final = 16
+
+
+def _add_values(cells: list[Any], values: list[Any], start: int, count: int) -> None:
+ """Append values[start:start + count] to the cells."""
+ if count > _FEW_CELLS:
+ cells += values[start : start + count]
+ else:
+ for i in range(start, start + count):
+ cells.append(values[i])
+
+
+def _add_run(cells: list[Any], term: Any, count: int) -> None:
+ """Append `count` cells holding the same term."""
+ if count > _FEW_CELLS:
+ cells += [term] * count
+ else:
+ for _ in range(count):
+ cells.append(term)
+
+
+def _too_many_cells(position: str) -> JellyConformanceError:
+ msg = f"the {position} column has more cells than the frame has rows"
+ return JellyConformanceError(msg)
+
+
def _kinds_mismatch(position: str, kind: int) -> JellyConformanceError:
msg = (
f"the kinds of the {position} column refer to more {_KIND_NAMES[kind]} "
@@ -104,6 +132,54 @@ def _kinds_mismatch(position: str, kind: int) -> JellyConformanceError:
return JellyConformanceError(msg)
+def _layout_cells(
+ values: list[Any], layouts: list[int], rows: int, position: str
+) -> list[Any]:
+ """Expand a column's layout tokens into its cells, in order."""
+ cells: list[Any] = []
+ index = 0
+ k = 0
+ while k < len(layouts):
+ token = layouts[k]
+ k += 1
+ length = token & _LENGTH_MASK
+ if length == _LENGTH_MASK:
+ if k == len(layouts):
+ msg = f"corrupt layout of the {position} column: a length is cut off"
+ raise JellyConformanceError(msg)
+ length += layouts[k]
+ k += 1
+ skip = token >> _SKIP_SHIFT
+ if skip:
+ if index + skip > len(values):
+ msg = (
+ f"corrupt layout of the {position} column: it skips past its values"
+ )
+ raise JellyConformanceError(msg)
+ _add_values(cells, values, index, skip)
+ index += skip
+ if token & _UNBOUND:
+ term = None
+ count = length + 1
+ elif index < len(values):
+ term = values[index]
+ index += 1
+ count = length + 2
+ else:
+ msg = (
+ f"corrupt layout of the {position} column: a repeated value is past "
+ "its values"
+ )
+ raise JellyConformanceError(msg)
+ # Check before building the cells, since a run can claim any length
+ if len(cells) + count > rows:
+ raise _too_many_cells(position)
+ _add_run(cells, term, count)
+ if index < len(values):
+ cells += values[index:]
+ return cells
+
+
class _Runs:
"""
The cells of a column, in runs.
@@ -238,6 +314,9 @@ def __init__(self, decoder: Decoder) -> None:
self.options_read = False
# Statements of the current RDF Message so far, in this and earlier frames
self.rows_in_message = 0
+ #: Prefix for blank node labels, so that equal labels in different result
+ #: sets give different blank nodes (see pyjelly.parse.sparql)
+ self.bnode_scope = ""
def iter_frame(self, frame: jelly.RdfStreamFrame) -> Iterator[Any]:
"""
@@ -353,15 +432,19 @@ def message_lengths(self, batch: jelly.RdfColumnBatch, rows: int) -> list[int]:
return lengths
def ingest_entries(self, batch: jelly.RdfColumnBatch) -> None:
+ self.apply_entries(batch.names, batch.prefixes, batch.datatypes)
+
+ def apply_entries(self, names: Any, prefixes: Any, datatypes: Any) -> None:
+ """Apply a frame's packed lookup entries."""
decoder = self.decoder
- _apply_entries(batch.names, decoder.names)
- _apply_entries(batch.prefixes, decoder.prefixes)
- for entry in batch.datatypes:
+ _apply_entries(names, decoder.names)
+ _apply_entries(prefixes, decoder.prefixes)
+ for entry in datatypes:
for value in entry.values:
if value in (LANG_STRING_DATATYPE_IRI, DIR_LANG_STRING_DATATYPE_IRI):
msg = f"the datatype lookup cannot have {value}"
raise JellyConformanceError(msg)
- _apply_entries(batch.datatypes, decoder.datatypes)
+ _apply_entries(datatypes, decoder.datatypes)
def namespaces(self, batch: jelly.RdfColumnBatch) -> list[Any]:
adapter = self.decoder.adapter
@@ -459,6 +542,23 @@ def runs(
runs.add(None, rows - runs.cells)
return runs
+ def cells(self, column: jelly.RdfColumn, rows: int, position: str) -> list[Any]:
+ """
+ Decode a Jelly-SPARQL column into its cells, in order.
+
+ The column may hold terms of any type. Unbound cells are None.
+ """
+ values = self.run_values(column, _ANY, position)
+ layouts: list[int] = column.layouts[:]
+ # Without layout tokens, each value fills one cell
+ cells = _layout_cells(values, layouts, rows, position) if layouts else values
+ if len(cells) > rows:
+ raise _too_many_cells(position)
+ if len(cells) < rows:
+ # Pad with unbound cells
+ cells += [None] * (rows - len(cells))
+ return cells
+
def run_values(
self, column: jelly.RdfColumn, allowed: int, position: str
) -> list[Any]:
@@ -541,27 +641,42 @@ def iris(self, column: jelly.RdfColumn) -> list[Any]:
prefix_index = prefix_ids[0] if prefix_count == 1 else 0
name_index = 0
terms: list[Any] = []
- for i in range(count):
- if prefix_count > 1 and prefix_ids[i]:
- prefix_index = prefix_ids[i]
- name_index = name_ids[i] or name_index + 1
- name = names[name_index - 1]
- prefix = prefixes[prefix_index - 1] if prefix_index else ""
- if name is None or prefix is None:
- msg = (
- f"an IRI refers to name {name_index} or prefix {prefix_index}, "
- "which is not set"
- )
- raise JellyConformanceError(msg)
- cached = cache[name_index]
- if cached is not None and cached.name is name and cached.prefix is prefix:
- terms.append(cached.term)
- else:
- terms.append(decoder.new_iri(name_index, name, prefix))
+ try:
+ # Iterating a list is cheaper than indexing it
+ for i, name_id in enumerate(name_ids):
+ if prefix_count > 1 and prefix_ids[i]:
+ prefix_index = prefix_ids[i]
+ name_index = name_id or name_index + 1
+ name = names[name_index - 1]
+ prefix = prefixes[prefix_index - 1] if prefix_index else ""
+ if name is None or prefix is None:
+ msg = (
+ f"an IRI refers to name {name_index} or prefix "
+ f"{prefix_index}, which is not set"
+ )
+ raise JellyConformanceError(msg)
+ cached = cache[name_index]
+ if (
+ cached is not None
+ and cached.name is name
+ and cached.prefix is prefix
+ ):
+ terms.append(cached.term)
+ else:
+ terms.append(decoder.new_iri(name_index, name, prefix))
+ except IndexError:
+ msg = (
+ f"an IRI refers to name {name_index} or prefix {prefix_index}, "
+ "which is outside the lookups"
+ )
+ raise JellyConformanceError(msg) from None
return terms
def bnodes(self, column: jelly.RdfColumn) -> list[Any]:
adapter = self.decoder.adapter
+ scope = self.bnode_scope
+ if scope:
+ return [adapter.bnode(scope + label) for label in column.bnodes]
return [adapter.bnode(label) for label in column.bnodes]
def literals(self, column: jelly.RdfColumn) -> list[Any]:
@@ -605,7 +720,14 @@ def literal_kind(self, kind: int, langtags: Any, directions: Any) -> _LiteralKin
if not kind:
return _LiteralKind(decoder, None, None, None)
if kind & 1:
- datatype = decoder.datatypes.at((kind + 1) >> 1)
+ index = (kind + 1) >> 1
+ if index > decoder.datatypes.lookup_size:
+ msg = (
+ f"a literal kind refers to datatype {index}, which is outside the "
+ "datatype lookup"
+ )
+ raise JellyConformanceError(msg)
+ datatype = decoder.datatypes.at(index)
return _LiteralKind(decoder, None, datatype, None)
index = (kind >> 1) - 1
if index >= len(langtags):
@@ -646,7 +768,7 @@ def triple_term(self, triple: jelly.RdfTripleTerm, iris: IriSequence) -> Any:
if kind == "s_iri":
s = iris.term(triple.s_iri)
elif kind == "s_bnode":
- s = adapter.bnode(triple.s_bnode)
+ s = adapter.bnode(self.bnode_scope + triple.s_bnode)
else:
msg = "a triple term has no subject"
raise JellyConformanceError(msg)
@@ -658,7 +780,7 @@ def triple_term(self, triple: jelly.RdfTripleTerm, iris: IriSequence) -> Any:
if kind == "o_iri":
o = iris.term(triple.o_iri)
elif kind == "o_bnode":
- o = adapter.bnode(triple.o_bnode)
+ o = adapter.bnode(self.bnode_scope + triple.o_bnode)
elif kind == "o_literal":
o = self.triple_term_literal(triple.o_literal)
elif kind == "o_triple_term":
diff --git a/pyjelly/parse/ioutils.py b/pyjelly/parse/ioutils.py
index 7186abed..c03e153d 100644
--- a/pyjelly/parse/ioutils.py
+++ b/pyjelly/parse/ioutils.py
@@ -69,13 +69,19 @@ def single_byte_tags(descriptor: Descriptor) -> frozenset[int]:
)
"""Tag bytes a non-delimited frame can start with (currently 0x0A, 0x12, 0x7A)."""
+SPARQL_FRAME_TAGS: Final = single_byte_tags(
+ cast("Descriptor", jelly.SparqlResultsFrame.DESCRIPTOR)
+)
+"""Tag bytes that a non-delimited Jelly-SPARQL frame can start with."""
+
-def hint_bytes_needed(first_byte: int) -> int:
+def hint_bytes_needed(first_byte: int, tags: frozenset[int] = FRAME_TAGS) -> int:
"""
Return how many bytes `delimited_jelly_hint` needs to decide.
Args:
first_byte (int): first byte of the stream
+ tags (frozenset[int]): tag bytes a frame can start with
Returns:
int: number of bytes to read from the start of the stream, at most 128
@@ -87,17 +93,18 @@ def hint_bytes_needed(first_byte: int) -> int:
11
"""
- if first_byte in FRAME_TAGS:
+ if first_byte in tags:
return first_byte + 1
return 1
-def _is_frame_fields(data: bytes) -> bool:
+def _is_frame_fields(data: bytes, tags: frozenset[int]) -> bool:
"""
Check whether `data` is exactly a sequence of fields of a stream frame.
Args:
data (bytes): candidate frame body
+ tags (frozenset[int]): tag bytes of the frame's fields
Returns:
bool: True if the bytes make up whole fields and nothing is left over
@@ -108,7 +115,7 @@ def _is_frame_fields(data: bytes) -> bool:
while pos < end:
tag = data[pos]
pos += 1
- if tag not in FRAME_TAGS:
+ if tag not in tags:
return False
wire_type = tag & _WIRE_TYPE_MASK
if wire_type == _WIRE_VARINT:
@@ -138,7 +145,7 @@ def _is_frame_fields(data: bytes) -> bool:
return pos == end
-def delimited_jelly_hint(header: bytes) -> bool:
+def delimited_jelly_hint(header: bytes, tags: frozenset[int] = FRAME_TAGS) -> bool:
"""
Detect whether a Jelly stream is delimited, from its first bytes.
@@ -157,6 +164,8 @@ def delimited_jelly_hint(header: bytes) -> bool:
Args:
header (bytes): first bytes of the stream
+ tags (frozenset[int]): tag bytes a frame can start with, FRAME_TAGS for
+ Jelly-RDF and SPARQL_FRAME_TAGS for Jelly-SPARQL
Returns:
bool: True if the stream is delimited
@@ -199,13 +208,13 @@ def delimited_jelly_hint(header: bytes) -> bool:
if not header:
return False
first = header[0]
- if first not in FRAME_TAGS:
+ if first not in tags:
# Not a frame tag, so it is the size of the first frame. This includes
# 0, an empty frame: a stream of only empty frames is delimited too.
return True
if len(header) < first + 1:
return False
- return _is_frame_fields(header[1 : first + 1])
+ return _is_frame_fields(header[1 : first + 1], tags)
def _read_exactly(inp: IO[bytes], size: int) -> bytes:
@@ -231,7 +240,7 @@ def _read_exactly(inp: IO[bytes], size: int) -> bytes:
return b"".join(chunks)
-def read_hint_header(inp: IO[bytes]) -> bytes:
+def read_hint_header(inp: IO[bytes], tags: frozenset[int] = FRAME_TAGS) -> bytes:
"""
Read just enough of the start of the stream to detect delimiting.
@@ -239,6 +248,7 @@ def read_hint_header(inp: IO[bytes]) -> bytes:
Args:
inp (IO[bytes]): stream to read from
+ tags (frozenset[int]): tag bytes a frame can start with
Returns:
bytes: bytes read from the start of the stream
@@ -247,7 +257,7 @@ def read_hint_header(inp: IO[bytes]) -> bytes:
header = inp.read(1)
if not header:
return header
- remaining = hint_bytes_needed(header[0]) - 1
+ remaining = hint_bytes_needed(header[0], tags) - 1
if remaining:
header += _read_exactly(inp, remaining)
return header
@@ -274,6 +284,38 @@ def read(self, size: int = -1) -> bytes:
return taken
+def detect_delimited(
+ inp: IO[bytes], tags: frozenset[int] = FRAME_TAGS
+) -> tuple[IO[bytes], bool]:
+ """
+ Detect from the first bytes whether a stream is delimited.
+
+ Args:
+ inp (IO[bytes]): the stream
+ tags (frozenset[int]): tag bytes a frame can start with
+
+ Returns:
+ tuple[IO[bytes], bool]: a stream positioned at the start again, and
+ whether it is delimited
+
+ """
+ if inp.seekable():
+ header = read_hint_header(inp, tags)
+ inp.seek(-len(header), os.SEEK_CUR)
+ else:
+ # Input may not be seekable (e.g. a network stream). Then the bytes we read
+ # are put back in front of it. Only an unbuffered stream gets a buffer: a
+ # second buffer around a buffered stream (e.g. from socket.makefile())
+ # waits for the inner one to fill up, which blocks streaming.
+ # See also: https://github.com/Jelly-RDF/pyjelly/issues/298
+ buffered = inp
+ if not isinstance(inp, io.BufferedIOBase):
+ buffered = io.BufferedReader(inp) # type: ignore[arg-type, type-var, unused-ignore]
+ header = read_hint_header(buffered, tags)
+ inp = _StreamWithHeader(header, buffered) # type: ignore[assignment]
+ return inp, delimited_jelly_hint(header, tags)
+
+
def frame_iterator(inp: IO[bytes]) -> Generator[jelly.RdfStreamFrame]:
# protobuf annotates `input_bytes` as io.BytesIO, but any IO[bytes] works.
while frame := parse_length_prefixed(jelly.RdfStreamFrame, inp): # type: ignore[arg-type]
@@ -311,17 +353,7 @@ def get_options_and_frames(
stream types, lookup presets and other stream options
"""
- if inp.seekable():
- header = read_hint_header(inp)
- inp.seek(-len(header), os.SEEK_CUR)
- else:
- # Input may not be seekable (e.g. a network stream) -- then we need to buffer
- # it and put the bytes we looked at back in front of it.
- # See also: https://github.com/Jelly-RDF/pyjelly/issues/298
- buffered = io.BufferedReader(inp) # type: ignore[arg-type, type-var, unused-ignore]
- header = read_hint_header(buffered)
- inp = _StreamWithHeader(header, buffered) # type: ignore[assignment]
- is_delimited = delimited_jelly_hint(header)
+ inp, is_delimited = detect_delimited(inp)
if is_delimited:
first_frame = None
diff --git a/pyjelly/parse/lookup.py b/pyjelly/parse/lookup.py
index 17dd573c..94cc56fe 100644
--- a/pyjelly/parse/lookup.py
+++ b/pyjelly/parse/lookup.py
@@ -33,7 +33,12 @@ def assign_entry(self, index: int, value: str) -> None:
previous_index = self.last_assigned_index
if index == 0:
index = previous_index + 1
- assert index > 0
+ if index > self.lookup_size:
+ msg = (
+ f"a lookup entry has the index {index}, "
+ f"but the lookup has {self.lookup_size} entries"
+ )
+ raise JellyConformanceError(msg)
self.data[index - 1] = value
self.last_assigned_index = index
diff --git a/pyjelly/parse/sparql.py b/pyjelly/parse/sparql.py
new file mode 100644
index 00000000..f0180826
--- /dev/null
+++ b/pyjelly/parse/sparql.py
@@ -0,0 +1,713 @@
+"""
+Jelly-SPARQL reader: SPARQL query results in frames, with one column per variable.
+
+SparqlDecoder decodes frames one at a time. read_result_sets() groups the rows of
+the frames into result sets.
+"""
+
+from __future__ import annotations
+
+import io
+import warnings
+from collections.abc import Callable, Iterator
+from typing import IO, Any, Final
+
+from google.protobuf.message import DecodeError
+from google.protobuf.proto import parse, parse_length_prefixed
+
+from pyjelly import jelly
+from pyjelly.errors import (
+ JellyConformanceError,
+ JellyIncompleteResultsError,
+ JellyTruncatedResultsWarning,
+)
+from pyjelly.options import (
+ MAX_DATATYPE_LOOKUP_SIZE,
+ MAX_NAME_LOOKUP_SIZE,
+ MAX_PREFIX_LOOKUP_SIZE,
+ MAX_SPARQL_FRAME_ROWS,
+ MAX_SPARQL_FRAME_VALUES,
+ MIN_COLUMN_NAME_LOOKUP_SIZE,
+ SPARQL_MAX_VERSION,
+ VERSION_1_2,
+ LookupPreset,
+ StreamParameters,
+ StreamTypes,
+)
+from pyjelly.parse.columns import MAX_ROWS, ColumnDecoder
+from pyjelly.parse.decode import Adapter, Decoder, ParserOptions
+from pyjelly.parse.ioutils import SPARQL_FRAME_TAGS, detect_delimited
+
+#: Metadata key for the links of a result set (head.link in SPARQL JSON results)
+LINK_KEY: Final = "link"
+
+AdapterFactory = Callable[[ParserOptions], Adapter]
+#: Builds the rows of a frame from the variables, the cells (one list per variable,
+#: None for unbound) and the row count
+RowFactory = Callable[[list[str], list[list[Any]], int], list[Any]]
+
+
+def tuple_rows(variables: list[str], cells: list[list[Any]], rows: int) -> list[Any]:
+ """Build the rows of a frame as tuples, with one value per variable."""
+ if not variables:
+ return [()] * rows
+ return list(zip(*cells, strict=True))
+
+
+# Methods are called through the message classes, not the messages: upb tries
+# every attribute name of a message as a field name first
+_FRAME_HAS_FIELD: Final = jelly.SparqlResultsFrame.HasField
+
+# What protobuf raises for bytes it can't decode. The pure-Python implementation
+# raises UnicodeDecodeError for a string that isn't UTF-8, the others DecodeError.
+_DECODE_ERRORS: Final = (DecodeError, UnicodeDecodeError)
+
+
+_STREAM_TYPES: Final = (
+ jelly.SPARQL_STREAM_TYPE_FLAT,
+ jelly.SPARQL_STREAM_TYPE_PUNCTUATED,
+)
+_RDF_VERSIONS: Final = frozenset(jelly.RdfVersion.values())
+
+
+def check_sparql_options(options: jelly.SparqlResultsOptions) -> None:
+ """
+ Check that this parser supports a Jelly-SPARQL stream with these options.
+
+ Raises:
+ JellyConformanceError: if it can't.
+
+ """
+ version = options.version
+ if not 1 <= version <= SPARQL_MAX_VERSION:
+ msg = (
+ f"unsupported Jelly-SPARQL version {version}, "
+ f"expected 1 to {SPARQL_MAX_VERSION}"
+ )
+ raise JellyConformanceError(msg)
+ if options.stream_type not in _STREAM_TYPES:
+ msg = f"unknown stream type: {options.stream_type}"
+ raise JellyConformanceError(msg)
+ if options.rdf_version not in _RDF_VERSIONS:
+ msg = f"unknown RDF version: {options.rdf_version}"
+ raise JellyConformanceError(msg)
+ _check_size(
+ "name",
+ options.max_name_table_size,
+ MIN_COLUMN_NAME_LOOKUP_SIZE,
+ MAX_NAME_LOOKUP_SIZE,
+ )
+ _check_size("prefix", options.max_prefix_table_size, 0, MAX_PREFIX_LOOKUP_SIZE)
+ _check_size(
+ "datatype", options.max_datatype_table_size, 0, MAX_DATATYPE_LOOKUP_SIZE
+ )
+
+
+def _check_size(lookup: str, size: int, smallest: int, largest: int) -> None:
+ if not smallest <= size <= largest:
+ msg = (
+ f"the {lookup} lookup size is {size}, "
+ f"but it must be from {smallest} to {largest}"
+ )
+ raise JellyConformanceError(msg)
+
+
+def sparql_parser_options(options: jelly.SparqlResultsOptions) -> ParserOptions:
+ """
+ Build the adapter options for a Jelly-SPARQL stream.
+
+ Terms are decoded like in a Jelly-RDF 1.2 stream with the same lookup sizes and
+ RDF version. The stream types are left unspecified.
+ """
+ return ParserOptions(
+ stream_types=StreamTypes(),
+ lookup_preset=LookupPreset(
+ max_names=options.max_name_table_size,
+ max_prefixes=options.max_prefix_table_size,
+ max_datatypes=options.max_datatype_table_size,
+ ),
+ params=StreamParameters(
+ version=VERSION_1_2,
+ stream_name=options.stream_name,
+ rdf_version=options.rdf_version,
+ ),
+ )
+
+
+def decode_links(value: bytes) -> list[str] | None:
+ r"""
+ Split the "link" metadata value into IRIs. Return None if it isn't UTF-8.
+
+ >>> decode_links(b"http://example.org/a\nhttp://example.org/b")
+ ['http://example.org/a', 'http://example.org/b']
+ >>> decode_links(b"")
+ []
+ >>> decode_links(b"\xff") is None
+ True
+ """
+ try:
+ text = value.decode()
+ except UnicodeDecodeError:
+ return None
+ return text.split("\n") if text else []
+
+
+class DecodedFrame:
+ """Contents of one decoded Jelly-SPARQL frame."""
+
+ def __init__(self) -> None:
+ #: Whether this is the first frame of a result set
+ self.starts = False
+ #: Variables of the result set, if this frame starts a solution sequence
+ self.variables: list[str] = []
+ #: Boolean result, if the frame has one
+ self.boolean: bool | None = None
+ #: Rows of the frame, built by the decoder's row factory
+ self.rows: list[Any] = []
+ #: Link IRIs for the result set, if the frame has them
+ self.links: list[str] | None = None
+ #: Trailer: None if the frame has none, "" if the result set is complete,
+ #: otherwise the error message
+ self.trailer: str | None = None
+
+
+class SparqlDecoder:
+ """
+ Decoder of the frames of a Jelly-SPARQL stream, in stream order.
+
+ Args:
+ adapter_factory (AdapterFactory): builds the term adapter from the stream
+ options.
+ row_factory (RowFactory): builds the rows of a frame. Tuples by default.
+ max_rows (int): maximum rows per frame.
+ max_values (int): maximum values (rows times variables) per frame. A frame
+ is decoded all at once, so this caps its memory use.
+
+ """
+
+ def __init__(
+ self,
+ adapter_factory: AdapterFactory,
+ *,
+ row_factory: RowFactory = tuple_rows,
+ max_rows: int = MAX_SPARQL_FRAME_ROWS,
+ max_values: int = MAX_SPARQL_FRAME_VALUES,
+ ) -> None:
+ self.adapter_factory = adapter_factory
+ self.row_factory = row_factory
+ self.max_rows = min(max_rows, MAX_ROWS)
+ self.max_values = max_values
+ #: Stream options, once read
+ self.options: jelly.SparqlResultsOptions | None = None
+ self.decoder: Decoder | None = None
+ self.columns: ColumnDecoder | None = None
+ #: Whether the stream is a sequence of result sets
+ self.punctuated = False
+ # Variables declared by the first header of the result set, and whether a
+ # header is in effect. Repeated stream options cancel the header.
+ self.variables: list[str] | None = None
+ self.header_in_effect = False
+ # Whether the next frame starts a result set, whether the current result
+ # set is a boolean result, and whether a trailer came after the options
+ self.result_set_start = True
+ self.boolean_read = False
+ self.trailer_read = False
+ # Number of result sets started so far
+ self.result_sets = 0
+
+ def decode(self, frame: jelly.SparqlResultsFrame) -> DecodedFrame:
+ """Decode a frame."""
+ out = DecodedFrame()
+ has_options = _FRAME_HAS_FIELD(frame, "options")
+ if has_options:
+ self.read_options(frame.options)
+ elif self.options is None:
+ msg = "the stream does not start with the stream options"
+ raise JellyConformanceError(msg)
+ elif self.trailer_read and not self.punctuated:
+ msg = (
+ "a frame after the trailer must have the stream options: the trailer "
+ "ends the result set"
+ )
+ raise JellyConformanceError(msg)
+ if self.result_set_start:
+ self.start_result_set()
+ out.starts = True
+ if self.boolean_read:
+ msg = "no frame of the result set can follow its boolean result"
+ raise JellyConformanceError(msg)
+ if _FRAME_HAS_FIELD(frame, "ask_result"):
+ self.read_boolean(frame, out)
+ else:
+ self.read_solutions(frame, has_options=has_options, out=out)
+ links = frame.metadata.get(LINK_KEY)
+ if links is not None:
+ out.links = decode_links(links)
+ if _FRAME_HAS_FIELD(frame, "trailer"):
+ out.trailer = frame.trailer.error
+ self.trailer_read = True
+ self.result_set_start = self.punctuated
+ else:
+ self.result_set_start = False
+ return out
+
+ def read_options(self, options: jelly.SparqlResultsOptions) -> None:
+ """Read the stream options. Repeated options also empty the lookups."""
+ check_sparql_options(options)
+ punctuated = options.stream_type == jelly.SPARQL_STREAM_TYPE_PUNCTUATED
+ parser_options = sparql_parser_options(options)
+ current = self.options
+ if current is None:
+ decoder = Decoder(self.adapter_factory(parser_options))
+ self.decoder = decoder
+ self.columns = decoder.column_decoder
+ else:
+ if options.stream_type != current.stream_type:
+ msg = "the stream type must be the same in all stream options"
+ raise JellyConformanceError(msg)
+ if punctuated and not self.result_set_start:
+ msg = (
+ "in a PUNCTUATED stream, the stream options can only be repeated "
+ "in the first frame of a result set"
+ )
+ raise JellyConformanceError(msg)
+ current_decoder = self.decoder
+ assert current_decoder is not None
+ current_decoder.adapter.options = parser_options
+ current_decoder.new_lookups()
+ self.header_in_effect = False
+ self.options = options
+ self.punctuated = punctuated
+ if not punctuated:
+ self.trailer_read = False
+
+ def start_result_set(self) -> None:
+ """Reset the per-result-set state, for the next result set."""
+ self.variables = None
+ self.header_in_effect = False
+ self.boolean_read = False
+ self.trailer_read = False
+ if self.result_sets:
+ # Each result set has its own blank node labels
+ columns = self.columns
+ assert columns is not None
+ columns.bnode_scope = f"r{self.result_sets}-"
+ self.result_sets += 1
+
+ def read_boolean(self, frame: jelly.SparqlResultsFrame, out: DecodedFrame) -> None:
+ if not out.starts:
+ msg = "a boolean result can only be in the first frame of a result set"
+ raise JellyConformanceError(msg)
+ if frame.row_count or frame.variables or frame.columns:
+ msg = "a frame with a boolean result cannot have variables, rows or columns"
+ raise JellyConformanceError(msg)
+ self.boolean_read = True
+ out.boolean = frame.ask_result.value
+ # The next result set of a PUNCTUATED stream may refer to these entries
+ columns = self.columns
+ assert columns is not None
+ columns.apply_entries(frame.names, frame.prefixes, frame.datatypes)
+
+ def read_solutions(
+ self, frame: jelly.SparqlResultsFrame, *, has_options: bool, out: DecodedFrame
+ ) -> None:
+ variables = self.read_variables(frame, has_options=has_options)
+ if out.starts:
+ out.variables = variables
+ rows = frame.row_count
+ if rows > MAX_ROWS:
+ msg = f"a frame can have at most {MAX_ROWS} rows, not {rows}"
+ raise JellyConformanceError(msg)
+ if rows > self.max_rows:
+ msg = f"the frame has {rows} rows, more than the {self.max_rows} allowed"
+ raise JellyConformanceError(msg)
+ if rows * len(variables) > self.max_values:
+ msg = (
+ f"the frame has {rows} rows of {len(variables)} variables, more than "
+ f"the {self.max_values} values allowed"
+ )
+ raise JellyConformanceError(msg)
+
+ columns = self.columns
+ assert columns is not None
+ # Apply all lookup entries before decoding the columns that use them
+ columns.apply_entries(frame.names, frame.prefixes, frame.datatypes)
+ column_count = len(frame.columns)
+ if not variables:
+ if column_count:
+ msg = "a result set with no variables cannot have columns"
+ raise JellyConformanceError(msg)
+ out.rows = self.row_factory(variables, [], rows)
+ elif column_count == len(variables):
+ cells = [
+ columns.cells(column, rows, "?" + name)
+ for column, name in zip(frame.columns, variables, strict=True)
+ ]
+ out.rows = self.row_factory(variables, cells, rows)
+ elif column_count or rows:
+ msg = (
+ f"the frame has {column_count} columns, but the result set has "
+ f"{len(variables)} variables"
+ )
+ raise JellyConformanceError(msg)
+
+ def read_variables(
+ self, frame: jelly.SparqlResultsFrame, *, has_options: bool
+ ) -> list[str]:
+ """Read the frame's header, if any, and return the current variables."""
+ if frame.variables:
+ if not has_options and not self.result_set_start:
+ msg = (
+ "the variables can only be declared in the first frame of a "
+ "result set, or in a frame with the stream options"
+ )
+ raise JellyConformanceError(msg)
+ self.read_header(list(frame.variables))
+ elif not self.header_in_effect:
+ # A missing header next to the options, or at the start of a result
+ # set, means zero variables
+ if self.variables:
+ msg = (
+ "the frame repeats the stream options, so it must repeat the "
+ "variables too"
+ )
+ raise JellyConformanceError(msg)
+ self.read_header([])
+ variables = self.variables
+ assert variables is not None
+ return variables
+
+ def read_header(self, names: list[str]) -> None:
+ for name in names:
+ if not name:
+ msg = "variable names cannot be empty"
+ raise JellyConformanceError(msg)
+ if self.variables is None:
+ self.variables = names
+ elif names != self.variables:
+ msg = (
+ "the repeated variables must be the same as the first ones: "
+ f"{names} instead of {self.variables}"
+ )
+ raise JellyConformanceError(msg)
+ self.header_in_effect = True
+
+
+def sparql_frames(inp: IO[bytes] | bytes) -> Iterator[jelly.SparqlResultsFrame]:
+ """
+ Iterate the frames of a Jelly-SPARQL stream, delimited or not.
+
+ Raises:
+ JellyConformanceError: if the stream is empty, or isn't Jelly-SPARQL.
+
+ """
+ if isinstance(inp, (bytes, bytearray, memoryview)):
+ inp = io.BytesIO(inp)
+ inp, delimited = detect_delimited(inp, SPARQL_FRAME_TAGS)
+ if not delimited:
+ data = inp.read()
+ if not data:
+ msg = "the stream is empty, but it must have at least one frame"
+ raise JellyConformanceError(msg)
+ try:
+ return iter((parse(jelly.SparqlResultsFrame, data),))
+ except _DECODE_ERRORS as error:
+ raise _not_sparql(error) from None
+ return _delimited_frames(inp)
+
+
+def _not_sparql(error: Exception) -> JellyConformanceError:
+ msg = (
+ f"the stream is not valid Jelly-SPARQL ({error}). Is it cut short, or is it "
+ "Jelly-RDF?"
+ )
+ return JellyConformanceError(msg)
+
+
+def _delimited_frames(inp: IO[bytes]) -> Iterator[jelly.SparqlResultsFrame]:
+ while True:
+ try:
+ # protobuf annotates `input_bytes` as io.BytesIO, but any IO[bytes] works.
+ frame = parse_length_prefixed(jelly.SparqlResultsFrame, inp) # type: ignore[arg-type]
+ except _DECODE_ERRORS as error:
+ raise _not_sparql(error) from None
+ except ValueError as error:
+ if getattr(inp, "closed", False):
+ msg = (
+ "the input was closed before all results were read. Read the "
+ "rows before closing it."
+ )
+ raise ValueError(msg) from error
+ raise
+ if frame is None:
+ return
+ yield frame
+
+
+class ResultSetReader:
+ """Reads and decodes frames one at a time, for the result sets."""
+
+ def __init__(
+ self, frames: Iterator[jelly.SparqlResultsFrame], decoder: SparqlDecoder
+ ) -> None:
+ self.frames = frames
+ self.decoder = decoder
+ # A frame read ahead that belongs to the next result set
+ self.pending: DecodedFrame | None = None
+
+ def next(self) -> DecodedFrame | None:
+ """Return the next frame, or None at the end of the stream."""
+ frame = self.pending
+ if frame is not None:
+ self.pending = None
+ return frame
+ message = next(self.frames, None)
+ if message is None:
+ return None
+ return self.decoder.decode(message)
+
+
+def _incomplete(error: str, result: object) -> JellyIncompleteResultsError:
+ msg = f"the results are incomplete: {error}"
+ return JellyIncompleteResultsError(msg, error, result)
+
+
+class SparqlResultSet:
+ """
+ One result set of a Jelly-SPARQL stream: a solution sequence or a boolean result.
+
+ Rows are read from the stream lazily, during iteration. Each row is a tuple
+ with one value per variable, None if the variable is unbound. A loop that
+ stops early can be resumed later. Once all rows have been read, iterating
+ again raises an error. If the next result set is read first, the unread rows
+ are kept in memory.
+
+ If the producer ended the result set with an error, iterating raises
+ JellyIncompleteResultsError after the last row, and so does `boolean`. If the
+ stream ends without a trailer, `truncated` is set and a
+ JellyTruncatedResultsWarning is issued.
+ """
+
+ def __init__(self, reader: ResultSetReader, first: DecodedFrame) -> None:
+ self.reader = reader
+ self.first: DecodedFrame | None = first
+ #: Variables in projection order. Empty for a boolean result.
+ self.variables: list[str] = first.variables
+ self._boolean = first.boolean
+ #: Link IRIs for the result set (head.link in SPARQL JSON)
+ self.links: list[str] = first.links or []
+ #: Error message from the producer, if the result set is incomplete
+ self.error = ""
+ #: Whether the stream ended without a trailer, so rows may be missing.
+ #: Only known after all rows have been read.
+ self.truncated = False
+ # Row lists of the coming frames, and the unread rows of the current one
+ self._frames = self._read()
+ self._rows: Iterator[Any] = iter(())
+ # Whether an iteration has read all rows
+ self._done = False
+ if first.boolean is not None:
+ # A boolean result is a single frame, so its trailer is already known
+ if first.trailer:
+ self.error = first.trailer
+ elif first.trailer is None:
+ self._truncated()
+
+ @property
+ def boolean(self) -> bool | None:
+ """
+ Boolean result of an ASK query, or None for a solution sequence.
+
+ Raises:
+ JellyIncompleteResultsError: if the producer reported an error.
+
+ """
+ if self.error and self._boolean is not None:
+ raise _incomplete(self.error, self)
+ return self._boolean
+
+ @property
+ def is_boolean(self) -> bool:
+ """Whether this is a boolean result (of an ASK query)."""
+ return self._boolean is not None
+
+ def __iter__(self) -> Iterator[Any]:
+ self._check_not_done()
+ while True:
+ # A list iterator keeps its position when a loop stops early, so the
+ # next loop resumes from there
+ yield from self._rows
+ rows = next(self._frames, None)
+ if rows is None:
+ break
+ self._rows = iter(rows)
+ self._end()
+
+ def collect(self, rows: list[Any]) -> None:
+ """
+ Append all remaining rows to a list. Faster than iterating.
+
+ Raises:
+ JellyIncompleteResultsError: if the result set is incomplete. The rows
+ are appended first.
+
+ """
+ self._check_not_done()
+ rows.extend(self._rows)
+ for frame_rows in self._frames:
+ rows.extend(frame_rows)
+ self._end()
+
+ def _check_not_done(self) -> None:
+ if self._done:
+ msg = (
+ "the rows of the result set were already read: they can be "
+ "iterated only once"
+ )
+ raise RuntimeError(msg)
+
+ def _end(self) -> None:
+ self._done = True
+ if self.error:
+ raise _incomplete(self.error, self)
+
+ def _truncated(self) -> None:
+ if not self.truncated:
+ self.truncated = True
+ warnings.warn(
+ "the stream ended without saying that the results are complete, so "
+ "some may be missing",
+ JellyTruncatedResultsWarning,
+ stacklevel=3,
+ )
+
+ def _read(self) -> Iterator[list[Any]]:
+ """Yield each frame's rows until the result set ends, recording trailers."""
+ reader = self.reader
+ frame = self.first
+ self.first = None
+ ended = False
+ while frame is not None:
+ if frame.trailer is not None:
+ ended = True
+ if frame.trailer:
+ # Keep the first error, even if more rows follow in a
+ # concatenated stream
+ self.error = self.error or frame.trailer
+ else:
+ # In a FLAT stream, repeated options after a trailer continue the
+ # result set (concatenated streams), so the trailer no longer counts
+ ended = False
+ if frame.links is not None and not self.links:
+ self.links = frame.links
+ if frame.rows:
+ yield frame.rows
+ if ended and reader.decoder.punctuated:
+ return
+ frame = self._next_frame(ended=ended)
+ if frame is not None and frame.starts:
+ # This frame starts the next result set of a PUNCTUATED stream
+ reader.pending = frame
+ return
+ if not ended:
+ self._truncated()
+
+ def _next_frame(self, *, ended: bool) -> DecodedFrame | None:
+ """Read the next frame, or None at the end of the stream."""
+ try:
+ return self.reader.next()
+ except JellyConformanceError:
+ if ended and self.error:
+ # The trailer already said that the results are incomplete, so what
+ # follows doesn't matter. E.g., Fuseki appends a text message to the
+ # response of a query it cancels.
+ return None
+ raise
+
+ def keep(self) -> None:
+ """Read the remaining rows into memory, for iteration later."""
+ if not self._done:
+ kept = list(self._rows)
+ for rows in self._frames:
+ kept += rows
+ self._rows = iter(kept)
+
+
+def read_result_sets(
+ inp: IO[bytes] | bytes,
+ adapter_factory: AdapterFactory,
+ *,
+ row_factory: RowFactory = tuple_rows,
+ max_rows: int = MAX_SPARQL_FRAME_ROWS,
+ max_values: int = MAX_SPARQL_FRAME_VALUES,
+) -> Iterator[SparqlResultSet]:
+ """
+ Read the result sets of a Jelly-SPARQL stream.
+
+ A FLAT stream (the default) has one result set, a PUNCTUATED stream any
+ number. Moving on to the next result set reads the rest of the current one
+ and keeps its unread rows in memory.
+
+ Args:
+ inp (IO[bytes] | bytes): the stream, delimited or not.
+ adapter_factory (AdapterFactory): builds the term adapter from the stream
+ options.
+ row_factory (RowFactory): builds the rows. Tuples by default.
+ max_rows (int): maximum rows per frame.
+ max_values (int): maximum values (rows times variables) per frame.
+
+ Yields:
+ SparqlResultSet: the result sets, in order.
+
+ """
+ decoder = SparqlDecoder(
+ adapter_factory,
+ row_factory=row_factory,
+ max_rows=max_rows,
+ max_values=max_values,
+ )
+ reader = ResultSetReader(sparql_frames(inp), decoder)
+ while (first := reader.next()) is not None:
+ result_set = SparqlResultSet(reader, first)
+ yield result_set
+ result_set.keep()
+
+
+def read_result_set(
+ inp: IO[bytes] | bytes,
+ adapter_factory: AdapterFactory,
+ *,
+ row_factory: RowFactory = tuple_rows,
+ max_rows: int = MAX_SPARQL_FRAME_ROWS,
+ max_values: int = MAX_SPARQL_FRAME_VALUES,
+) -> SparqlResultSet:
+ """
+ Read the single result set of a FLAT Jelly-SPARQL stream.
+
+ The first frame is read now, the rows during iteration. For PUNCTUATED
+ streams, use read_result_sets().
+
+ Raises:
+ JellyConformanceError: if the stream is PUNCTUATED.
+ JellyIncompleteResultsError: for a boolean result that the producer
+ could not complete.
+
+ """
+ result_sets = read_result_sets(
+ inp,
+ adapter_factory,
+ row_factory=row_factory,
+ max_rows=max_rows,
+ max_values=max_values,
+ )
+ result_set = next(result_sets)
+ if result_set.reader.decoder.punctuated:
+ msg = (
+ "the stream is a sequence of result sets (PUNCTUATED). Read it with "
+ "parse_jelly_sparql_sequence() of pyjelly.integrations.rdflib.sparql "
+ "or pyjelly.integrations.generic.sparql"
+ )
+ raise JellyConformanceError(msg)
+ if result_set.is_boolean and result_set.error:
+ raise _incomplete(result_set.error, result_set)
+ return result_set
diff --git a/pyjelly/serialize/encode.py b/pyjelly/serialize/encode.py
index e958c4c2..262968b7 100644
--- a/pyjelly/serialize/encode.py
+++ b/pyjelly/serialize/encode.py
@@ -157,7 +157,7 @@ def new_epoch(self) -> None:
self.prefixes.lookup.new_epoch()
self.datatypes.lookup.new_epoch()
- def new_frame(self, iris: int) -> None:
+ def new_frame(self, iris: int, literals: int = 1) -> None:
"""
Start encoding a new frame in the column layout.
@@ -167,12 +167,13 @@ def new_frame(self, iris: int) -> None:
Args:
iris (int): the most IRIs that one statement can have.
+ literals (int): maximum number of literals in one statement or row. In a
+ statement, only the object can be a literal.
"""
self.names.lookup.new_epoch(_budget(self.names, iris))
self.prefixes.lookup.new_epoch(_budget(self.prefixes, iris))
- # Only the object can be a literal
- self.datatypes.lookup.new_epoch(_budget(self.datatypes, 1))
+ self.datatypes.lookup.new_epoch(_budget(self.datatypes, literals))
self.triple_prefix_index = -1
self.triple_name_index = 0
diff --git a/pyjelly/serialize/ioutils.py b/pyjelly/serialize/ioutils.py
index 1180df6e..829bb98c 100644
--- a/pyjelly/serialize/ioutils.py
+++ b/pyjelly/serialize/ioutils.py
@@ -1,14 +1,34 @@
from typing import IO
-from google.protobuf.proto import serialize_length_prefixed
+from google.protobuf.message import Message
-from pyjelly import jelly
+_VARINT_PAYLOAD = 0x7F
+_VARINT_CONTINUATION = 0x80
-def write_delimited(frame: jelly.RdfStreamFrame, output_stream: IO[bytes]) -> None:
- # protobuf annotates `output` as io.BytesIO, but any IO[bytes] works at runtime.
- serialize_length_prefixed(frame, output_stream) # type: ignore[arg-type]
+def varint(value: int) -> bytes:
+ r"""
+ Encode a non-negative integer as a protobuf varint.
+ >>> varint(300)
+ b'\xac\x02'
+ """
+ out = bytearray()
+ while value > _VARINT_PAYLOAD:
+ out.append((value & _VARINT_PAYLOAD) | _VARINT_CONTINUATION)
+ value >>= 7
+ out.append(value)
+ return bytes(out)
-def write_single(frame: jelly.RdfStreamFrame, output_stream: IO[bytes]) -> None:
+
+def write_delimited(frame: Message, output_stream: IO[bytes]) -> None:
+ """Write a frame prefixed with its size (the delimited variant of Jelly)."""
+ # protobuf's serialize_length_prefixed() needs write() to return the number of
+ # bytes written, which not every stream does (e.g., rdflib's result streams)
+ data = frame.SerializeToString()
+ output_stream.write(varint(len(data)))
+ output_stream.write(data)
+
+
+def write_single(frame: Message, output_stream: IO[bytes]) -> None:
output_stream.write(frame.SerializeToString(deterministic=True))
diff --git a/pyjelly/serialize/lookup.py b/pyjelly/serialize/lookup.py
index 2853025a..9b1e27c1 100644
--- a/pyjelly/serialize/lookup.py
+++ b/pyjelly/serialize/lookup.py
@@ -139,8 +139,8 @@ def victim(self) -> int:
return index
index = 1 if index == size else index + 1
msg = (
- f"a lookup of size {size} is too small to encode a single statement: "
- "the statement uses every one of its entries"
+ f"a lookup of size {size} is too small to encode a single statement or "
+ "row of results: it uses every one of its entries"
)
raise JellyConformanceError(msg)
diff --git a/pyjelly/serialize/sparql.py b/pyjelly/serialize/sparql.py
new file mode 100644
index 00000000..b9f61358
--- /dev/null
+++ b/pyjelly/serialize/sparql.py
@@ -0,0 +1,576 @@
+"""Jelly-SPARQL writer: SPARQL query results in frames."""
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Iterator, Mapping, Sequence
+from contextlib import suppress
+from typing import IO, Any, Final
+
+from pyjelly import jelly
+from pyjelly.errors import JellyConformanceError, JellyIncompleteResultsError
+from pyjelly.options import SPARQL_VERSION_1_0, SparqlOptions
+from pyjelly.serialize.columns import MAX_ROWS, Column
+from pyjelly.serialize.encode import EncodedTerm, TermEncoder
+from pyjelly.serialize.ioutils import write_delimited, write_single
+
+#: Metadata key for the links of a result set (head.link in SPARQL JSON results)
+LINK_KEY: Final = "link"
+
+# A result column accepts every term type
+_ANY: Final = 0b1111
+
+
+def links_value(links: Sequence[str]) -> bytes:
+ r"""
+ Join link IRIs with LF, for the "link" metadata key.
+
+ >>> links_value(["http://example.org/a", "http://example.org/b"])
+ b'http://example.org/a\nhttp://example.org/b'
+ """
+ for link in links:
+ if "\n" in link:
+ msg = f"a link cannot contain the LF character, as no IRI can: {link!r}"
+ raise JellyConformanceError(msg)
+ return "\n".join(links).encode()
+
+
+def variable_names(variables: Iterable[str]) -> list[str]:
+ """
+ Strip a leading "?" or "$" from each variable name.
+
+ >>> variable_names(["?s", "$p", "o"])
+ ['s', 'p', 'o']
+ """
+ names = []
+ for variable in variables:
+ name = str(variable)
+ if name[:1] in ("?", "$"):
+ name = name[1:]
+ if not name:
+ msg = "variable names cannot be empty"
+ raise JellyConformanceError(msg)
+ names.append(name)
+ return names
+
+
+def _failed() -> JellyConformanceError:
+ msg = (
+ "a row failed to encode, so the stream can only be ended, with an error: "
+ "end(error)"
+ )
+ return JellyConformanceError(msg)
+
+
+class VariableColumn(Column):
+ """Cells of one variable in the frame being built."""
+
+ def __init__(self) -> None:
+ super().__init__(_ANY, "variable")
+ # Prefix and name indices of the last IRI in this column's triple terms.
+ # The encoder holds them while it encodes this column (see TermEncoder).
+ self.triple_prefix_index = -1
+ self.triple_name_index = 0
+ # Largest number of IRIs in one triple term of this column so far
+ self.max_triple_iris = 1
+
+
+class SparqlStream:
+ """
+ Writer of SPARQL query results as Jelly-SPARQL frames.
+
+ A result set is either a solution sequence (SELECT) or a boolean result (ASK).
+ For a solution sequence, call start(), then row() for each row, then end().
+ row() returns a frame when one is full, and end() returns the last frame. ask()
+ returns the single frame of a boolean result.
+
+ A stream holds one result set, or any number of them with
+ SparqlOptions(punctuated=True).
+
+ Args:
+ encoder (TermEncoder): term encoder, with the same lookup sizes as the
+ options.
+ options (SparqlOptions | None): options of the stream.
+
+ """
+
+ def __init__(
+ self, encoder: TermEncoder, options: SparqlOptions | None = None
+ ) -> None:
+ if options is None:
+ options = SparqlOptions()
+ if encoder.lookup_preset != options.lookup_preset:
+ msg = "the encoder must have the lookup sizes of the options"
+ raise ValueError(msg)
+ self.encoder = encoder
+ self.options = options
+ encoder.column_layout = True
+ encoder.rdf_version = options.rdf_version
+ self.encoded = EncodedTerm()
+ #: Variables of the current result set, None between result sets
+ self.variables: list[str] | None = None
+ self.columns: list[VariableColumn] = []
+ # Reused buffer for the values of a binding, see binding()
+ self.values: list[object | None] = []
+ self.row_count = 0
+ self.rows_per_frame = 1
+ # Set when the single frame of a non-delimited stream is full
+ self.full = False
+ self.links: list[str] = []
+ # Whether the stream options and the header still have to be written
+ self.options_pending = True
+ self.header_pending = False
+ # `ended`: the single result set of a FLAT stream is done. `failed`: a row
+ # failed to encode. Either way, nothing more can be written.
+ self.ended = False
+ self.failed = False
+ # Column whose triple-term IRI state the encoder currently holds
+ self.triple_owner: VariableColumn | None = None
+
+ def start(self, variables: Iterable[str], links: Sequence[str] = ()) -> None:
+ """
+ Start a solution sequence.
+
+ Args:
+ variables (Iterable[str]): variable names, in projection order. A
+ leading "?" or "$" is removed.
+ links (Sequence[str]): link IRIs for the result set, as in head.link of
+ SPARQL JSON results.
+
+ """
+ self._check_can_start()
+ names = variable_names(variables)
+ self.variables = names
+ self.columns = [VariableColumn() for _ in names]
+ self.values = [None] * len(names)
+ self.rows_per_frame = (
+ min(self.options.rows_per_frame(len(names)), MAX_ROWS)
+ if self.options.delimited
+ else MAX_ROWS
+ )
+ self.header_pending = True
+ self.links = list(links)
+ self.new_frame()
+
+ def _check_can_start(self) -> None:
+ if self.failed:
+ raise _failed()
+ if self.ended:
+ msg = (
+ "the stream has ended. A FLAT stream has one result set, use "
+ "SparqlOptions(punctuated=True) to write more"
+ )
+ raise JellyConformanceError(msg)
+ if self.variables is not None:
+ msg = "the result set has not ended yet, end it with end()"
+ raise JellyConformanceError(msg)
+
+ def new_frame(self) -> None:
+ columns = self.columns
+ iris = len(columns)
+ for column in columns:
+ # A triple term can take more than one IRI per cell
+ iris += column.max_triple_iris - 1
+ column.triple_prefix_index = -1
+ column.triple_name_index = 0
+ self.encoder.new_frame(iris, len(columns))
+ self.triple_owner = None
+ self.row_count = 0
+ self.full = False
+
+ def row(self, values: Sequence[object | None]) -> jelly.SparqlResultsFrame | None:
+ """
+ Add a row (solution) to the result set.
+
+ Args:
+ values (Sequence[object | None]): one value per variable, None if the
+ variable is unbound.
+
+ Returns:
+ jelly.SparqlResultsFrame | None: the frame, if this row filled it.
+
+ """
+ columns = self.columns
+ count = len(values)
+ self._check_row(count)
+ try:
+ # mypyc indexes a typed tuple or list directly. Any other sequence goes
+ # through the generic item lookup with a boxed int, which is slow.
+ if type(values) is tuple:
+ row_tuple: tuple[object | None, ...] = values
+ for i in range(count):
+ self.add(columns[i], row_tuple[i])
+ elif type(values) is list:
+ row_list: list[object | None] = values
+ for i in range(count):
+ self.add(columns[i], row_list[i])
+ else:
+ for i in range(count):
+ self.add(columns[i], values[i])
+ except BaseException:
+ # The frame is half-written and can't be finished
+ self.failed = True
+ raise
+ self.row_count += 1
+ # End the frame as soon as it's full, rather than at the start of the next
+ # row, so that a failing next row can't lose it
+ if self.row_count >= self.rows_per_frame or not self.encoder.has_room():
+ if self.options.delimited:
+ return self.end_frame()
+ self.full = True
+ return None
+
+ def _check_row(self, count: int) -> None:
+ """Check that a row of `count` values can be added."""
+ if self.variables is None or self.failed:
+ if self.failed:
+ raise _failed()
+ msg = "start a result set with start() before adding rows"
+ raise JellyConformanceError(msg)
+ if count != len(self.columns):
+ msg = f"expected {len(self.columns)} values in the row, got {count}"
+ raise JellyConformanceError(msg)
+ if self.full:
+ msg = (
+ "the results do not fit in one frame: the lookups can't hold all of "
+ "their terms. A non-delimited stream can only have one frame. Write "
+ "a delimited stream, or use larger lookups."
+ )
+ raise JellyConformanceError(msg)
+
+ def binding(
+ self, binding: Mapping[Any, object | None], variables: Sequence[object]
+ ) -> jelly.SparqlResultsFrame | None:
+ """
+ Add a row (solution) given as a mapping from variables to values.
+
+ Args:
+ binding (Mapping[Any, object | None]): values of the bound variables,
+ keyed by variable or by variable name.
+ variables (Sequence[object]): variables of the result set, in order.
+
+ Returns:
+ jelly.SparqlResultsFrame | None: the frame, if this row filled it.
+
+ """
+ values = self.values
+ if len(variables) != len(values):
+ msg = f"expected {len(values)} variables, got {len(variables)}"
+ raise JellyConformanceError(msg)
+ # A typed list is cheaper to index, see row()
+ keys: list[object] = variables if type(variables) is list else list(variables)
+ bound = 0
+ if type(binding) is dict:
+ # Same loop, but with direct dict calls instead of method lookups
+ mapping: dict[Any, object | None] = binding
+ for i in range(len(values)):
+ value = mapping.get(keys[i])
+ values[i] = value
+ if value is not None:
+ bound += 1
+ else:
+ for i in range(len(values)):
+ value = binding.get(keys[i])
+ values[i] = value
+ if value is not None:
+ bound += 1
+ if bound != len(binding):
+ # Some keys are not the variable objects themselves (rdflib's
+ # Variable("x") does not equal "x"), so look the values up by name
+ by_name = {str(key): value for key, value in binding.items()}
+ for i in range(len(values)):
+ values[i] = by_name.get(str(variables[i]))
+ return self.row(values)
+
+ def rows(
+ self, rows: Iterable[Sequence[object | None]]
+ ) -> Iterator[jelly.SparqlResultsFrame]:
+ """
+ Add many rows, see row().
+
+ Yields:
+ jelly.SparqlResultsFrame: each frame as it fills up.
+
+ """
+ for values in rows:
+ frame = self.row(values)
+ if frame is not None:
+ yield frame
+
+ def add(self, column: VariableColumn, term: object | None) -> None:
+ """Add a value to a column. None is an unbound cell."""
+ if term is None:
+ if not column.run_unbound:
+ column.end_run()
+ column.run_unbound = True
+ column.run_length += 1
+ return
+ if (
+ column.run_length
+ and not column.run_unbound
+ and self.encoder.same_term(column.run_term, term)
+ ):
+ column.run_length += 1
+ return
+ column.end_run()
+ if self.triple_owner is not None and self.triple_owner is not column:
+ self._switch_triple_iris(column)
+ encoded = self.encoded
+ self.encoder.encode_spo(term, encoded)
+ column.add_value(encoded)
+ column.run_term = term
+ column.run_length = 1
+ if encoded.triple_term is not None:
+ self._triple_term_added(column)
+
+ def _switch_triple_iris(self, column: VariableColumn) -> None:
+ """Swap the encoder's triple-term IRI state over to another column."""
+ owner = self.triple_owner
+ assert owner is not None
+ encoder = self.encoder
+ owner.triple_prefix_index = encoder.triple_prefix_index
+ owner.triple_name_index = encoder.triple_name_index
+ encoder.triple_prefix_index = column.triple_prefix_index
+ encoder.triple_name_index = column.triple_name_index
+ self.triple_owner = column
+
+ def _triple_term_added(self, column: VariableColumn) -> None:
+ self.encoded.triple_term = None
+ if self.triple_owner is None:
+ self.triple_owner = column
+ iris = self.encoder.triple_iris
+ if iris > column.max_triple_iris:
+ # Every following row may need this many IRIs for the column
+ self.encoder.reserve_iris(iris - column.max_triple_iris)
+ column.max_triple_iris = iris
+
+ def end_frame(self) -> jelly.SparqlResultsFrame:
+ """End the current frame and return it."""
+ if self.variables is None or self.failed:
+ if self.failed:
+ raise _failed()
+ msg = "there is no result set in progress"
+ raise JellyConformanceError(msg)
+ frame = jelly.SparqlResultsFrame()
+ self.write_frame(frame)
+ return frame
+
+ def write_frame(self, frame: jelly.SparqlResultsFrame) -> None:
+ """Fill in the frame: options, header, lookup entries and rows."""
+ self.write_start(frame)
+ encoder = self.encoder
+ row_count = self.row_count
+ frame.row_count = row_count
+ encoder.names.packed.write(frame.names)
+ encoder.prefixes.packed.write(frame.prefixes)
+ encoder.datatypes.packed.write(frame.datatypes)
+ # Columns are left out of a frame with no rows
+ if row_count:
+ for column in self.columns:
+ column.write(frame.columns.add())
+ self.new_frame()
+
+ def write_start(self, frame: jelly.SparqlResultsFrame) -> None:
+ """Add the stream options and the header, if they are still due."""
+ if self.options_pending:
+ frame.options.CopyFrom(self.stream_options())
+ self.options_pending = False
+ if self.header_pending:
+ assert self.variables is not None
+ frame.variables.extend(self.variables)
+ self.header_pending = False
+ if self.links:
+ frame.metadata[LINK_KEY] = links_value(self.links)
+
+ def end(self, error: str = "") -> jelly.SparqlResultsFrame:
+ """
+ End the result set and return its last frame, which has the trailer.
+
+ Args:
+ error (str): reason the result set is incomplete, if it is. It tells
+ readers that some results are missing.
+
+ Returns:
+ jelly.SparqlResultsFrame: the last frame.
+
+ """
+ frame = jelly.SparqlResultsFrame()
+ if self.failed:
+ if not error or self.variables is None:
+ raise _failed()
+ # Drop the frame's rows and their lookup entries. The lookups are then
+ # out of sync with the reader, so the stream must end here.
+ self.write_start(frame)
+ self.ended = True
+ elif self.variables is None:
+ msg = "there is no result set to end, start one with start()"
+ raise JellyConformanceError(msg)
+ else:
+ self.write_frame(frame)
+ if not self.options.punctuated:
+ self.ended = True
+ frame.trailer.SetInParent()
+ frame.trailer.error = error
+ self.variables = None
+ return frame
+
+ def ask(
+ self,
+ value: bool, # noqa: FBT001
+ links: Sequence[str] = (),
+ error: str = "",
+ ) -> jelly.SparqlResultsFrame:
+ """
+ Return the frame of a boolean result (of an ASK query).
+
+ Args:
+ value (bool): the result.
+ links (Sequence[str]): link IRIs for the result.
+ error (str): reason the result is incomplete, if it is.
+
+ Returns:
+ jelly.SparqlResultsFrame: the result's only frame.
+
+ """
+ self._check_can_start()
+ frame = jelly.SparqlResultsFrame()
+ self.write_start(frame)
+ frame.ask_result.value = value
+ frame.trailer.SetInParent()
+ frame.trailer.error = error
+ if links:
+ frame.metadata[LINK_KEY] = links_value(links)
+ if not self.options.punctuated:
+ self.ended = True
+ return frame
+
+ def stream_options(self) -> jelly.SparqlResultsOptions:
+ options = self.options
+ lookup_preset = options.lookup_preset
+ return jelly.SparqlResultsOptions(
+ stream_name=options.stream_name,
+ stream_type=(
+ jelly.SPARQL_STREAM_TYPE_PUNCTUATED
+ if options.punctuated
+ else jelly.SPARQL_STREAM_TYPE_FLAT
+ ),
+ rdf_version=options.rdf_version,
+ max_name_table_size=lookup_preset.max_names,
+ max_prefix_table_size=lookup_preset.max_prefixes,
+ max_datatype_table_size=lookup_preset.max_datatypes,
+ version=SPARQL_VERSION_1_0,
+ )
+
+
+def error_message(error: BaseException) -> str:
+ """
+ Format an exception as the error message of a trailer.
+
+ >>> error_message(KeyError("x"))
+ "KeyError: 'x'"
+ >>> error_message(JellyIncompleteResultsError("...", "timeout"))
+ 'timeout'
+ """
+ if isinstance(error, JellyIncompleteResultsError):
+ # Passing on the error of a stream that was itself incomplete
+ return error.error
+ message = str(error)
+ name = type(error).__name__
+ return f"{name}: {message}" if message else name
+
+
+class FrameWriter:
+ """
+ Writer of frames to a file, either delimited or as a single frame.
+
+ Each frame is flushed, so a reader on the other end of a connection gets the
+ rows right away.
+ """
+
+ def __init__(self, out: IO[bytes], *, delimited: bool) -> None:
+ self.out = out
+ self.delimited = delimited
+ self.written = False
+
+ def write(self, frame: jelly.SparqlResultsFrame) -> None:
+ if self.delimited:
+ write_delimited(frame, self.out)
+ self.out.flush()
+ return
+ if self.written:
+ msg = (
+ "the results do not fit in one frame. A non-delimited stream can "
+ "only have one frame. Write a delimited stream, or use larger lookups."
+ )
+ raise JellyConformanceError(msg)
+ write_single(frame, self.out)
+ self.out.flush()
+ self.written = True
+
+
+def write_solutions( # noqa: PLR0913 keyword-only options
+ stream: SparqlStream,
+ variables: Iterable[str],
+ rows: Iterable[Sequence[object | None]],
+ writer: FrameWriter,
+ *,
+ links: Sequence[str] = (),
+ error: str = "",
+) -> None:
+ """
+ Write a solution sequence (the result of a SELECT query) as one result set.
+
+ If iterating the rows raises (e.g., the query fails midway), the result set is
+ ended with that error so readers know it is incomplete, and the exception is
+ re-raised. Pass `error` if the rows are already known to be incomplete.
+ """
+ stream.start(variables, links)
+ try:
+ for values in rows:
+ frame = stream.row(values)
+ if frame is not None:
+ writer.write(frame)
+ except Exception as failure:
+ _end_with(stream, writer, failure)
+ raise
+ writer.write(stream.end(error))
+
+
+def write_bindings( # noqa: PLR0913 keyword-only options
+ stream: SparqlStream,
+ variables: Sequence[object],
+ bindings: Iterable[Mapping[Any, object | None]],
+ writer: FrameWriter,
+ *,
+ links: Sequence[str] = (),
+ error: str = "",
+) -> None:
+ """
+ Like write_solutions(), with rows given as bindings.
+
+ Args:
+ stream (SparqlStream): the stream.
+ variables (Sequence[object]): the variables, as str objects (e.g., rdflib's
+ Variable). Their names are written.
+ bindings (Iterable[Mapping[Any, object | None]]): the rows, see
+ SparqlStream.binding().
+ writer (FrameWriter): writer of the frames.
+ links (Sequence[str]): link IRIs for the results.
+ error (str): reason the rows are incomplete, if known.
+
+ """
+ stream.start([str(variable) for variable in variables], links)
+ try:
+ for binding in bindings:
+ frame = stream.binding(binding, variables)
+ if frame is not None:
+ writer.write(frame)
+ except Exception as failure:
+ _end_with(stream, writer, failure)
+ raise
+ writer.write(stream.end(error))
+
+
+def _end_with(stream: SparqlStream, writer: FrameWriter, failure: Exception) -> None:
+ """End the result set with an error, if writing the frame still works."""
+ with suppress(Exception):
+ writer.write(stream.end(error_message(failure)))
diff --git a/pyproject.toml b/pyproject.toml
index 6a5d08e4..ee344982 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -37,7 +37,9 @@ include = [
"pyjelly/parse/lookup.py",
"pyjelly/parse/ioutils.py",
"pyjelly/parse/columns.py",
+ "pyjelly/parse/sparql.py",
"pyjelly/serialize/columns.py",
+ "pyjelly/serialize/sparql.py",
"pyjelly/serialize/streams.py",
"pyjelly/serialize/flows.py",
"pyjelly/parse/decode.py",
@@ -74,7 +76,7 @@ archs = ["x86_64", "aarch64"]
name = "pyjelly"
version = "0.0.0"
readme = "README.md"
-description = "Jelly-RDF implementation for Python"
+description = "Jelly-RDF and Jelly-SPARQL implementation for Python"
authors = [
{ name = "NeverBlink et al.", email = "contact@neverblink.eu" }
]
@@ -124,6 +126,16 @@ jelly = "pyjelly.integrations.rdflib.parse:RDFLibJellyParser"
jelly = "pyjelly.integrations.rdflib.serialize:RDFLibJellySerializer"
"application/x-jelly-rdf" = "pyjelly.integrations.rdflib.serialize:RDFLibJellySerializer"
+[project.entry-points."rdf.plugins.resultparser"]
+jelly = "pyjelly.integrations.rdflib.sparql:RDFLibJellySparqlParser"
+jellys = "pyjelly.integrations.rdflib.sparql:RDFLibJellySparqlParser"
+"application/x-jelly-sparql" = "pyjelly.integrations.rdflib.sparql:RDFLibJellySparqlParser"
+
+[project.entry-points."rdf.plugins.resultserializer"]
+jelly = "pyjelly.integrations.rdflib.sparql:RDFLibJellySparqlSerializer"
+jellys = "pyjelly.integrations.rdflib.sparql:RDFLibJellySparqlSerializer"
+"application/x-jelly-sparql" = "pyjelly.integrations.rdflib.sparql:RDFLibJellySparqlSerializer"
+
[dependency-groups]
dev = ["mypy>=2.4", "pre-commit>=4.2.0", "ruff>=0.11.8"]
# grpc doesn't support PyPy at the time of writing, see grpc/grpc#4221
@@ -177,7 +189,8 @@ exclude = [
".*_pb2(_grpc)?\\.pyi?",
"examples/",
"submodules/",
- "tests/conformance_tests/rdf/"
+ "tests/conformance_tests/rdf/",
+ "tests/conformance_tests/sparql/"
]
[[tool.mypy.overrides]]
@@ -190,10 +203,10 @@ extra-standard-library = ["typing_extensions"]
[tool.pytest.ini_options]
pythonpath = "."
-addopts = ["--import-mode=importlib", "--doctest-modules", "--ignore-glob=docs/examples/**", "--ignore-glob=examples/**", "--ignore-glob=submodules/**", "--ignore-glob=tests/conformance_tests/rdf/**"]
+addopts = ["--import-mode=importlib", "--doctest-modules", "--ignore-glob=docs/examples/**", "--ignore-glob=examples/**", "--ignore-glob=submodules/**", "--ignore-glob=tests/conformance_tests/rdf/**", "--ignore-glob=tests/conformance_tests/sparql/**"]
[tool.ruff]
-extend-exclude = ["*{_pb2,_pb2_grpc}.{py,pyi}", "submodules", "tests/conformance_tests/rdf"]
+extend-exclude = ["*{_pb2,_pb2_grpc}.{py,pyi}", "submodules", "tests/conformance_tests/rdf", "tests/conformance_tests/sparql"]
[tool.setuptools]
packages = ["pyjelly"]
diff --git a/scripts/compile-proto b/scripts/compile-proto
index 9cd41ec5..45032d5a 100755
--- a/scripts/compile-proto
+++ b/scripts/compile-proto
@@ -9,8 +9,11 @@ OUT_DIR=${2:-"pyjelly/jelly"}
command mkdir -p "$OUT_DIR"
-: Compiling rdf.proto...
-python -m grpc_tools.protoc "$PROTO_PATH/rdf.proto" --proto_path="$PROTO_PATH" --{python,pyi}_out="$OUT_DIR"
+: Compiling rdf.proto and sparql.proto...
+python -m grpc_tools.protoc "$PROTO_PATH/rdf.proto" "$PROTO_PATH/sparql.proto" --proto_path="$PROTO_PATH" --{python,pyi}_out="$OUT_DIR"
+
+: sparql.proto imports rdf.proto, which is in this package...
+sed -i 's/^import rdf_pb2 as/from pyjelly.jelly import rdf_pb2 as/' "$OUT_DIR/sparql_pb2.py" "$OUT_DIR/sparql_pb2.pyi"
# : Compiling grpc.proto...
# python -m grpc_tools.protoc "$PROTO_PATH/grpc.proto" --proto_path="$PROTO_PATH" --{python,pyi,grpc_python}_out="$OUT_DIR"
diff --git a/tests/conformance_tests/generate_conformance_report.py b/tests/conformance_tests/generate_conformance_report.py
index 548bbd62..4fc9c1b9 100644
--- a/tests/conformance_tests/generate_conformance_report.py
+++ b/tests/conformance_tests/generate_conformance_report.py
@@ -78,6 +78,8 @@ def __init__(
self,
path_to_rdflib_report: str,
path_to_generic_report: str,
+ path_to_rdflib_sparql_report: str,
+ path_to_generic_sparql_report: str,
) -> None:
"""
Initialize ConformanceReportPlugin.
@@ -85,10 +87,16 @@ def __init__(
Args:
path_to_rdflib_report (str): report path for RDFLib integration
path_to_generic_report (str): report path for pyjelly with Generic API
+ path_to_rdflib_sparql_report (str): report path of the Jelly-SPARQL tests
+ of the RDFLib integration
+ path_to_generic_sparql_report (str): report path of the Jelly-SPARQL
+ tests of pyjelly with Generic API
"""
self.path_to_rdflib_report = path_to_rdflib_report
self.path_to_generic_report = path_to_generic_report
+ self.path_to_rdflib_sparql_report = path_to_rdflib_sparql_report
+ self.path_to_generic_sparql_report = path_to_generic_sparql_report
self._results: defaultdict[str, list[Result]] = defaultdict(list)
self._case_by_id: dict[str, object | None] = defaultdict(list)
@@ -263,10 +271,19 @@ def pytest_sessionfinish(self) -> None:
Builds RDF Graph using RDFLib and serializes to Turtle.
"""
- g_generic = self._initialize_graph(implementation="generic")
- g_rdflib = self._initialize_graph(implementation="RDFLib")
+ rdf_graphs = (
+ self._initialize_graph(implementation="generic"),
+ self._initialize_graph(implementation="RDFLib"),
+ )
+ sparql_graphs = (
+ self._initialize_graph(implementation="generic"),
+ self._initialize_graph(implementation="RDFLib"),
+ )
for k, v in self._results.items():
+ # The tests of Jelly-SPARQL have reports of their own
+ is_sparql = "/tests/sparql/" in str(k)
+ g_generic, g_rdflib = sparql_graphs if is_sparql else rdf_graphs
if len(v) == 1:
r = v[0]
if (
@@ -311,8 +328,10 @@ def pytest_sessionfinish(self) -> None:
g_rdflib, r.outcome, k, ASSERTOR, IMPL
)
- g_rdflib.serialize(self.path_to_rdflib_report, format="turtle")
- g_generic.serialize(self.path_to_generic_report, format="turtle")
+ rdf_graphs[1].serialize(self.path_to_rdflib_report, format="turtle")
+ rdf_graphs[0].serialize(self.path_to_generic_report, format="turtle")
+ sparql_graphs[1].serialize(self.path_to_rdflib_sparql_report, format="turtle")
+ sparql_graphs[0].serialize(self.path_to_generic_sparql_report, format="turtle")
def main() -> int:
@@ -321,6 +340,8 @@ def main() -> int:
conformance_report_plugin = ConformanceReportPlugin(
path_to_rdflib_report="pyjelly RDFLib.ttl",
path_to_generic_report="pyjelly Generic API.ttl",
+ path_to_rdflib_sparql_report="pyjelly RDFLib SPARQL.ttl",
+ path_to_generic_sparql_report="pyjelly Generic API SPARQL.ttl",
)
return pytest.main(args=[tests_dir], plugins=[conformance_report_plugin])
diff --git a/tests/conformance_tests/sparql b/tests/conformance_tests/sparql
new file mode 120000
index 00000000..98943cec
--- /dev/null
+++ b/tests/conformance_tests/sparql
@@ -0,0 +1 @@
+../../submodules/protobuf/test/sparql
\ No newline at end of file
diff --git a/tests/conformance_tests/test_rdf/_common.py b/tests/conformance_tests/test_rdf/_common.py
index e791aa78..5ee7a5eb 100644
--- a/tests/conformance_tests/test_rdf/_common.py
+++ b/tests/conformance_tests/test_rdf/_common.py
@@ -10,7 +10,7 @@
MF = Namespace("http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#")
# Requirements of the tests that the rdflib integration can't pass: rdflib has no
-# generalized statements, quoted triples, triple terms, or base directions
+# generalized statements, quoted triples, triple terms, or base directions.
NOT_RDFLIB = {
JELLYT.requirementRdfStar,
JELLYT.requirementGeneralizedRdf,
@@ -24,7 +24,7 @@ class ConformanceCase:
uri: str
name: str
positive: bool
- #: The suite of the test: "v1_1" or "v1_2"
+ #: The suite of the test: "v1_1" or "v1_2", or "sparql"
suite: str
actions: list[Path]
results: list[Path]
@@ -52,18 +52,22 @@ def graphs(self) -> bool:
def messages(self) -> bool:
return JELLYT.requirementMessages in self.requirements
+ @property
+ def punctuated(self) -> bool:
+ return JELLYT.requirementPunctuated in self.requirements
+
@property
def options(self) -> Path | None:
"""The stream options of a to_jelly test."""
for path in self.actions:
- if path.name == "stream_options.jelly":
+ if path.stem == "stream_options":
return path
return None
@property
def inputs(self) -> list[Path]:
"""The input files, without the stream options."""
- return [path for path in self.actions if path.name != "stream_options.jelly"]
+ return [path for path in self.actions if path.stem != "stream_options"]
def load_cases(manifest_path: Path) -> list[ConformanceCase]:
diff --git a/tests/conformance_tests/test_sparql/test_parse.py b/tests/conformance_tests/test_sparql/test_parse.py
new file mode 100644
index 00000000..3b957b3e
--- /dev/null
+++ b/tests/conformance_tests/test_sparql/test_parse.py
@@ -0,0 +1,95 @@
+"""The "from Jelly" conformance tests of Jelly-SPARQL."""
+
+from __future__ import annotations
+
+from typing import IO, Any
+
+import pytest
+from rdflib.query import Result
+
+from pyjelly.errors import JellyIncompleteResultsError
+from pyjelly.integrations.generic.sparql import (
+ parse_jelly_sparql as generic_parse_jelly_sparql,
+)
+from pyjelly.integrations.generic.sparql import (
+ parse_jelly_sparql_sequence as generic_parse_jelly_sparql_sequence,
+)
+from pyjelly.integrations.rdflib.sparql import parse_jelly_sparql_sequence
+from tests.conformance_tests.test_rdf._common import ConformanceCase, load_cases
+from tests.meta import SPARQL_FROM_JELLY_MANIFEST
+from tests.utils.sparql_compare import (
+ Results,
+ assert_same_result_sets,
+ generic_results,
+ rdflib_results,
+ read_srj,
+)
+
+CASES = load_cases(SPARQL_FROM_JELLY_MANIFEST)
+
+
+def _params(*, positive: bool, rdflib: bool = False) -> list[Any]:
+ return [
+ pytest.param(case, id=case.id)
+ for case in CASES
+ if case.positive == positive and (case.rdflib or not rdflib)
+ ]
+
+
+def _expected(case: ConformanceCase) -> list[Results]:
+ return [read_srj(path) for path in case.results]
+
+
+@pytest.mark.parametrize("case", _params(positive=True, rdflib=True))
+def test_rdflib_parses_positive(case: ConformanceCase) -> None:
+ with case.actions[0].open("rb") as file:
+ if case.punctuated:
+ results = [rdflib_results(r) for r in parse_jelly_sparql_sequence(file)]
+ else:
+ results = [rdflib_results(Result.parse(file, format="jelly"))]
+ assert_same_result_sets(_expected(case), results, hint=case.id)
+
+
+@pytest.mark.parametrize("case", _params(positive=True))
+def test_generic_parses_positive(case: ConformanceCase) -> None:
+ with case.actions[0].open("rb") as file:
+ if case.punctuated:
+ results = [
+ generic_results(r) for r in generic_parse_jelly_sparql_sequence(file)
+ ]
+ else:
+ results = [generic_results(generic_parse_jelly_sparql(file))]
+ assert_same_result_sets(_expected(case), results, hint=case.id)
+
+
+def _read_all_rdflib(file: IO[bytes], *, punctuated: bool) -> None:
+ if not punctuated:
+ Result.parse(file, format="jelly")
+ return
+ # An incomplete result doesn't stop a sequence, it sets `error` instead
+ for result in parse_jelly_sparql_sequence(file):
+ if result.error:
+ raise JellyIncompleteResultsError(result.error, result.error)
+
+
+@pytest.mark.parametrize("case", _params(positive=False, rdflib=True))
+def test_rdflib_parsing_fails_negative(case: ConformanceCase) -> None:
+ with (
+ pytest.raises(Exception), # noqa: B017,PT011 any error will do
+ case.actions[0].open("rb") as file,
+ ):
+ _read_all_rdflib(file, punctuated=case.punctuated)
+
+
+def _read_all(file: IO[bytes]) -> None:
+ for result_set in generic_parse_jelly_sparql_sequence(file):
+ list(result_set)
+
+
+@pytest.mark.parametrize("case", _params(positive=False))
+def test_generic_parsing_fails_negative(case: ConformanceCase) -> None:
+ with (
+ pytest.raises(Exception), # noqa: B017,PT011 any error will do
+ case.actions[0].open("rb") as file,
+ ):
+ _read_all(file)
diff --git a/tests/conformance_tests/test_sparql/test_serialize.py b/tests/conformance_tests/test_sparql/test_serialize.py
new file mode 100644
index 00000000..3d2f2622
--- /dev/null
+++ b/tests/conformance_tests/test_sparql/test_serialize.py
@@ -0,0 +1,130 @@
+"""The "to Jelly" conformance tests of Jelly-SPARQL."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+import pytest
+from rdflib.query import Result
+
+from pyjelly import jelly
+from pyjelly.integrations.generic.sparql import (
+ parse_jelly_sparql_sequence,
+ sparql_stream,
+)
+from pyjelly.integrations.rdflib.sparql import (
+ serialize_jelly_sparql,
+ serialize_jelly_sparql_sequence,
+)
+from pyjelly.options import LookupPreset, SparqlOptions
+from pyjelly.parse.sparql import sparql_frames
+from pyjelly.serialize.sparql import FrameWriter, write_solutions
+from tests.conformance_tests.test_rdf._common import ConformanceCase, load_cases
+from tests.meta import SPARQL_TO_JELLY_MANIFEST, TEST_OUTPUTS_DIR
+from tests.utils.sparql_compare import (
+ assert_same_result_sets,
+ generic_results,
+ generic_rows,
+ read_srj,
+)
+
+CASES = load_cases(SPARQL_TO_JELLY_MANIFEST)
+
+
+def _params(*, positive: bool, rdflib: bool = False) -> list[Any]:
+ return [
+ pytest.param(case, id=case.id)
+ for case in CASES
+ if case.positive == positive and (case.rdflib or not rdflib)
+ ]
+
+
+def _output(case: ConformanceCase) -> Path:
+ return TEST_OUTPUTS_DIR / f"{case.id}.jellys"
+
+
+def _stream_options(case: ConformanceCase) -> jelly.SparqlResultsOptions:
+ assert case.options
+ with case.options.open("rb") as file:
+ return next(sparql_frames(file)).options
+
+
+def _options(case: ConformanceCase) -> SparqlOptions:
+ """Read a test's options from its stream options file."""
+ options = _stream_options(case)
+ return SparqlOptions(
+ lookup_preset=LookupPreset(
+ max_names=options.max_name_table_size,
+ max_prefixes=options.max_prefix_table_size,
+ max_datatypes=options.max_datatype_table_size,
+ ),
+ rdf_version=options.rdf_version,
+ punctuated=options.stream_type == jelly.SPARQL_STREAM_TYPE_PUNCTUATED,
+ stream_name=options.stream_name,
+ )
+
+
+def _write_generic(case: ConformanceCase) -> None:
+ stream = sparql_stream(_options(case))
+ with _output(case).open("wb") as file:
+ writer = FrameWriter(file, delimited=True)
+ for path in case.inputs:
+ results = read_srj(path)
+ if results.boolean is not None:
+ writer.write(stream.ask(results.boolean))
+ else:
+ write_solutions(
+ stream, results.variables, generic_rows(results), writer
+ )
+
+
+def _write_rdflib(case: ConformanceCase) -> None:
+ results = []
+ for path in case.inputs:
+ with path.open("rb") as file:
+ results.append(Result.parse(file, format="json"))
+ options = _options(case)
+ with _output(case).open("wb") as file:
+ if options.punctuated:
+ serialize_jelly_sparql_sequence(results, file, options)
+ else:
+ (result,) = results
+ serialize_jelly_sparql(result, file, options)
+
+
+def _check(case: ConformanceCase) -> None:
+ output = _output(case)
+ with output.open("rb") as file:
+ frames = list(sparql_frames(file))
+ assert frames[0].options == _stream_options(case)
+ assert frames[-1].HasField("trailer")
+ assert not frames[-1].trailer.error
+ with output.open("rb") as file:
+ actual = [generic_results(r) for r in parse_jelly_sparql_sequence(file)]
+ expected = [read_srj(path) for path in case.inputs]
+ assert_same_result_sets(expected, actual, hint=case.id)
+
+
+@pytest.mark.parametrize("case", _params(positive=True, rdflib=True))
+def test_serializes_rdflib_positive(case: ConformanceCase) -> None:
+ _write_rdflib(case)
+ _check(case)
+
+
+@pytest.mark.parametrize("case", _params(positive=True))
+def test_serializes_generic_positive(case: ConformanceCase) -> None:
+ _write_generic(case)
+ _check(case)
+
+
+@pytest.mark.parametrize("case", _params(positive=False, rdflib=True))
+def test_serializing_fails_rdflib_negative(case: ConformanceCase) -> None:
+ with pytest.raises(Exception): # noqa: B017,PT011 any error will do
+ _write_rdflib(case)
+
+
+@pytest.mark.parametrize("case", _params(positive=False))
+def test_serializing_fails_generic_negative(case: ConformanceCase) -> None:
+ with pytest.raises(Exception): # noqa: B017,PT011 any error will do
+ _write_generic(case)
diff --git a/tests/integration_tests/test_generic/test_sparql.py b/tests/integration_tests/test_generic/test_sparql.py
new file mode 100644
index 00000000..6c759aed
--- /dev/null
+++ b/tests/integration_tests/test_generic/test_sparql.py
@@ -0,0 +1,314 @@
+"""Jelly-SPARQL with the generic API."""
+
+from __future__ import annotations
+
+import io
+from collections.abc import Iterator
+from pathlib import Path
+
+import pytest
+
+from pyjelly import jelly
+from pyjelly.errors import (
+ JellyConformanceError,
+ JellyIncompleteResultsError,
+ JellyTruncatedResultsWarning,
+)
+from pyjelly.integrations.generic.generic_sink import (
+ IRI,
+ BlankNode,
+ Literal,
+ Node,
+ Triple,
+)
+from pyjelly.integrations.generic.sparql import (
+ parse_jelly_sparql,
+ parse_jelly_sparql_sequence,
+ serialize_jelly_sparql,
+ serialize_jelly_sparql_ask,
+ serialize_jelly_sparql_sequence,
+ sparql_stream,
+)
+from pyjelly.options import LookupPreset, SparqlOptions
+from pyjelly.parse.sparql import sparql_frames
+from pyjelly.serialize.sparql import FrameWriter, write_solutions
+
+EX = "http://example.org/"
+XSD_INTEGER = "http://www.w3.org/2001/XMLSchema#integer"
+
+A = IRI(EX + "a")
+P = IRI(EX + "p")
+
+ROWS: list[tuple[Node | None, ...]] = [
+ (A, Literal("1", datatype=XSD_INTEGER), None),
+ (A, Literal("hello", "en"), Literal("مرحبا", "ar", direction="rtl")),
+ (BlankNode("b"), None, Triple(A, P, Literal("x"))),
+ (None, None, None),
+ (
+ BlankNode("b"),
+ Literal("hello", "en"),
+ Triple(BlankNode("c"), P, Triple(A, P, A)),
+ ),
+]
+
+
+def _write(
+ rows: list[tuple[Node | None, ...]], options: SparqlOptions | None = None
+) -> io.BytesIO:
+ out = io.BytesIO()
+ serialize_jelly_sparql(["s", "o", "t"], rows, out, options)
+ out.seek(0)
+ return out
+
+
+def test_round_trip() -> None:
+ results = parse_jelly_sparql(_write(ROWS))
+ assert results.variables == ["s", "o", "t"]
+ assert not results.is_boolean
+ assert list(results) == ROWS
+ assert not results.truncated
+ assert not results.error
+
+
+@pytest.mark.parametrize("frame_size", [1, 3, 7, 1000])
+def test_frames(frame_size: int) -> None:
+ out = _write(ROWS * 20, SparqlOptions(frame_values=frame_size))
+ frames = list(sparql_frames(out))
+ rows_per_frame = max(1, frame_size // 3)
+ assert all(frame.row_count <= rows_per_frame for frame in frames)
+ out.seek(0)
+ assert list(parse_jelly_sparql(out)) == ROWS * 20
+
+
+def test_rdf_version() -> None:
+ options = SparqlOptions(rdf_version=jelly.RDF_VERSION_1_2_BASIC)
+ with pytest.raises(JellyConformanceError, match=r"RDF 1\.2 Basic"):
+ _write(ROWS, options)
+ options = SparqlOptions(rdf_version=jelly.RDF_VERSION_1_1)
+ with pytest.raises(JellyConformanceError, match=r"RDF 1\.1"):
+ _write(ROWS[1:2], options)
+
+
+def test_triple_terms_in_several_columns() -> None:
+ # Each column keeps its own IRI state for its triple terms
+ t1 = Triple(IRI(EX + "s1"), P, IRI(EX + "o1"))
+ t2 = Triple(IRI(EX + "s2"), IRI(EX + "q"), Triple(A, P, IRI(EX + "o2")))
+ rows: list[tuple[Node | None, ...]] = [
+ (t1, t2, IRI(EX + "z")),
+ (t2, t1, t1),
+ (t1, None, t2),
+ ] * 50
+ options = SparqlOptions(
+ lookup_preset=LookupPreset(max_names=128, max_prefixes=4, max_datatypes=4),
+ frame_values=9,
+ )
+ assert list(parse_jelly_sparql(_write(rows, options))) == rows
+
+
+def test_lookups_full() -> None:
+ rows: list[tuple[Node | None, ...]] = [
+ (IRI(f"{EX}{i}"), Literal(str(i), datatype=f"{EX}type{i % 40}"), None)
+ for i in range(2000)
+ ]
+ options = SparqlOptions(
+ lookup_preset=LookupPreset(max_names=128, max_prefixes=2, max_datatypes=16)
+ )
+ out = _write(rows, options)
+ frames = list(sparql_frames(out))
+ assert len(frames) > 2000 // 128
+ out.seek(0)
+ assert list(parse_jelly_sparql(out)) == rows
+
+
+def test_ask() -> None:
+ out = io.BytesIO()
+ serialize_jelly_sparql_ask(True, out, links=[EX + "doc"]) # noqa: FBT003
+ out.seek(0)
+ results = parse_jelly_sparql(out)
+ assert results.is_boolean
+ assert results.boolean is True
+ assert results.variables == []
+ assert results.links == [EX + "doc"]
+ assert list(results) == []
+
+
+def test_non_delimited() -> None:
+ out = _write(ROWS * 10, SparqlOptions(delimited=False))
+ (frame,) = sparql_frames(out)
+ assert frame.row_count == len(ROWS) * 10
+ out.seek(0)
+ assert list(parse_jelly_sparql(out)) == ROWS * 10
+
+
+def test_non_delimited_too_large() -> None:
+ rows: list[tuple[Node | None, ...]] = [
+ (IRI(f"{EX}{i}"), None, None) for i in range(200)
+ ]
+ options = SparqlOptions(
+ lookup_preset=LookupPreset(max_names=128, max_prefixes=0, max_datatypes=0),
+ delimited=False,
+ )
+ with pytest.raises(JellyConformanceError, match="one frame"):
+ _write(rows, options)
+
+
+def _failing_rows() -> Iterator[tuple[Node | None, ...]]:
+ yield from ROWS[:2]
+ msg = "timeout"
+ raise TimeoutError(msg)
+
+
+def test_incomplete() -> None:
+ out = io.BytesIO()
+ with pytest.raises(TimeoutError):
+ serialize_jelly_sparql(["s", "o", "t"], _failing_rows(), out)
+ out.seek(0)
+ results = parse_jelly_sparql(out)
+ rows: list[tuple[Node | None, ...]] = []
+ with pytest.raises(JellyIncompleteResultsError, match="timeout"):
+ rows.extend(results)
+ # The rows before the error were read
+ assert rows == ROWS[:2]
+ assert results.error == "TimeoutError: timeout"
+
+
+def test_sequence_of_result_sets() -> None:
+ options = SparqlOptions(punctuated=True)
+ stream = sparql_stream(options)
+ out = io.BytesIO()
+ writer = FrameWriter(out, delimited=True)
+ write_solutions(stream, ["s", "o", "t"], ROWS, writer)
+ writer.write(stream.ask(False)) # noqa: FBT003
+ write_solutions(stream, ["x"], [(BlankNode("b"),)], writer, links=[EX + "l"])
+ out.seek(0)
+ first, second, third = parse_jelly_sparql_sequence(out)
+ assert list(first) == ROWS
+ assert second.boolean is False
+ assert third.variables == ["x"]
+ assert third.links == [EX + "l"]
+ (row,) = list(third)
+ # Each result set has its own blank node labels
+ assert row[0] != ROWS[2][0]
+
+
+def test_rows_are_kept_for_later() -> None:
+ stream = sparql_stream(SparqlOptions(punctuated=True, frame_values=3))
+ out = io.BytesIO()
+ writer = FrameWriter(out, delimited=True)
+ write_solutions(stream, ["s", "o", "t"], ROWS, writer)
+ write_solutions(stream, ["s", "o", "t"], ROWS[:1], writer)
+ out.seek(0)
+ first, second = parse_jelly_sparql_sequence(out)
+ assert list(second) == ROWS[:1]
+ assert list(first) == ROWS
+
+
+def test_flat_reader_rejects_punctuated() -> None:
+ out = io.BytesIO()
+ serialize_jelly_sparql_ask(
+ True, # noqa: FBT003
+ out,
+ SparqlOptions(punctuated=True),
+ )
+ out.seek(0)
+ with pytest.raises(JellyConformanceError, match="parse_jelly_sparql_sequence"):
+ parse_jelly_sparql(out)
+
+
+def test_incomplete_boolean_result() -> None:
+ data = io.BytesIO()
+ FrameWriter(data, delimited=True).write(
+ sparql_stream().ask(True, error="no answer") # noqa: FBT003
+ )
+ with pytest.raises(JellyIncompleteResultsError, match="no answer"):
+ parse_jelly_sparql(data.getvalue())
+
+
+def test_rows_are_iterated_once() -> None:
+ results = parse_jelly_sparql(_write(ROWS))
+ assert list(results) == ROWS
+ with pytest.raises(RuntimeError, match="only once"):
+ list(results)
+
+
+def test_iteration_can_be_picked_up_again() -> None:
+ results = parse_jelly_sparql(_write(ROWS, SparqlOptions(frame_values=3)))
+ for _ in results:
+ break
+ assert list(results) == ROWS[1:]
+
+
+def test_closed_input(tmp_path: Path) -> None:
+ path = tmp_path / "results.jellys"
+ path.write_bytes(_write(ROWS, SparqlOptions(frame_values=3)).getvalue())
+ with path.open("rb") as file:
+ results = parse_jelly_sparql(file)
+ with pytest.raises(ValueError, match="closed before all results were read"):
+ list(results)
+
+
+def test_truncated() -> None:
+ data = _write(ROWS, SparqlOptions(frame_values=3)).getvalue()
+ frames = list(sparql_frames(data))
+ cut = data[: -(frames[-1].ByteSize() + 1)]
+ results = parse_jelly_sparql(cut)
+ with pytest.warns(JellyTruncatedResultsWarning):
+ rows = list(results)
+ assert results.truncated
+ assert len(rows) == len(ROWS)
+
+
+def test_sequence_writer() -> None:
+ out = io.BytesIO()
+ serialize_jelly_sparql_sequence(
+ [(["s", "o", "t"], ROWS), False, (["x"], [(A,)])],
+ out,
+ )
+ first, second, third = parse_jelly_sparql_sequence(out.getvalue())
+ assert list(first) == ROWS
+ assert second.boolean is False
+ assert list(third) == [(A,)]
+
+
+def test_row_failing_after_a_full_frame() -> None:
+ # With 128 names, the 128th row with a new IRI fills the frame. The frame must
+ # still be written when the next row fails.
+ options = SparqlOptions(
+ lookup_preset=LookupPreset(max_names=128, max_prefixes=0, max_datatypes=0)
+ )
+ good: list[tuple[Node | None, ...]] = [(IRI(f"{EX}{i}"),) for i in range(128)]
+
+ def rows() -> Iterator[tuple[object, ...]]:
+ yield from good
+ yield (42,)
+
+ out = io.BytesIO()
+ with pytest.raises(NotImplementedError):
+ serialize_jelly_sparql(["x"], rows(), out, options) # type: ignore[arg-type]
+ out.seek(0)
+ results = parse_jelly_sparql(out)
+ read: list[tuple[Node | None, ...]] = []
+ with pytest.raises(JellyIncompleteResultsError, match="NotImplementedError"):
+ read.extend(results)
+ assert read == good
+
+
+def test_garbage_after_an_error_trailer() -> None:
+ # As Fuseki does when it cancels a query: after the error trailer, it appends
+ # a text message to the response
+ out = io.BytesIO()
+ with pytest.raises(TimeoutError):
+ serialize_jelly_sparql(["s", "o", "t"], _failing_rows(), out)
+ data = out.getvalue() + b"\n## Query cancelled due to timeout ##\n"
+ results = parse_jelly_sparql(data)
+ rows: list[tuple[Node | None, ...]] = []
+ with pytest.raises(JellyIncompleteResultsError, match="TimeoutError"):
+ rows.extend(results)
+ assert rows == ROWS[:2]
+
+
+def test_garbage_after_a_complete_trailer() -> None:
+ data = _write(ROWS).getvalue() + b"\n## not Jelly ##\n"
+ results = parse_jelly_sparql(data)
+ with pytest.raises(JellyConformanceError):
+ list(results)
diff --git a/tests/integration_tests/test_rdflib/test_parse_quads.py b/tests/integration_tests/test_rdflib/test_parse_quads.py
new file mode 100644
index 00000000..c5dbd0c7
--- /dev/null
+++ b/tests/integration_tests/test_rdflib/test_parse_quads.py
@@ -0,0 +1,63 @@
+"""Reading a Jelly stream of quads into rdflib's Graph, Dataset and ConjunctiveGraph."""
+
+from __future__ import annotations
+
+import io
+
+import pytest
+from rdflib import Dataset, Graph, URIRef
+from rdflib.graph import ConjunctiveGraph
+
+TRIG = """
+ .
+ {
+ .
+}
+"""
+DEFAULT_TRIPLE = (
+ URIRef("http://example.org/a"),
+ URIRef("http://example.org/p"),
+ URIRef("http://example.org/b"),
+)
+NAMED_TRIPLE = (
+ URIRef("http://example.org/c"),
+ URIRef("http://example.org/p"),
+ URIRef("http://example.org/d"),
+)
+NAMED_GRAPH = URIRef("http://example.org/g")
+
+
+def _jelly_quads() -> bytes:
+ dataset = Dataset()
+ dataset.parse(data=TRIG, format="trig")
+ data = dataset.serialize(format="jelly", encoding="utf-8")
+ assert isinstance(data, bytes)
+ return data
+
+
+@pytest.mark.parametrize("format_", ["jelly", "trig"])
+def test_quads_into_graph(format_: str) -> None:
+ data = _jelly_quads() if format_ == "jelly" else TRIG.encode()
+ graph = Graph()
+ graph.parse(io.BytesIO(data), format=format_)
+ assert set(graph) == {DEFAULT_TRIPLE}
+ named = Graph(store=graph.store, identifier=NAMED_GRAPH)
+ assert set(named) == {NAMED_TRIPLE}
+
+
+@pytest.mark.parametrize("format_", ["jelly", "trig"])
+def test_quads_into_dataset(format_: str) -> None:
+ data = _jelly_quads() if format_ == "jelly" else TRIG.encode()
+ dataset = Dataset()
+ dataset.parse(io.BytesIO(data), format=format_)
+ assert set(dataset.default_graph) == {DEFAULT_TRIPLE}
+ assert set(dataset.graph(NAMED_GRAPH)) == {NAMED_TRIPLE}
+
+
+@pytest.mark.parametrize("format_", ["jelly", "trig"])
+def test_quads_into_conjunctive_graph(format_: str) -> None:
+ data = _jelly_quads() if format_ == "jelly" else TRIG.encode()
+ graph = ConjunctiveGraph()
+ graph.parse(io.BytesIO(data), format=format_)
+ assert set(graph.default_context) == {DEFAULT_TRIPLE}
+ assert set(graph.get_context(NAMED_GRAPH)) == {NAMED_TRIPLE}
diff --git a/tests/integration_tests/test_rdflib/test_sparql.py b/tests/integration_tests/test_rdflib/test_sparql.py
new file mode 100644
index 00000000..9fc0090e
--- /dev/null
+++ b/tests/integration_tests/test_rdflib/test_sparql.py
@@ -0,0 +1,494 @@
+"""Jelly-SPARQL with rdflib: writing and reading query results via the plugins."""
+
+from __future__ import annotations
+
+import io
+import socket
+import threading
+from collections.abc import Iterator
+from http.server import BaseHTTPRequestHandler, HTTPServer
+from pathlib import Path
+from typing import Any
+
+import pytest
+from rdflib import BNode, Graph, Literal, URIRef, Variable
+from rdflib.plugins.stores.sparqlstore import SPARQLStore
+from rdflib.query import Result, ResultRow
+
+from pyjelly import jelly
+from pyjelly.errors import (
+ JellyConformanceError,
+ JellyIncompleteResultsError,
+ JellyTruncatedResultsWarning,
+)
+from pyjelly.integrations.rdflib.sparql import (
+ JellySparqlResult,
+ parse_jelly_sparql,
+ parse_jelly_sparql_sequence,
+ serialize_jelly_sparql,
+ serialize_jelly_sparql_sequence,
+ sparql_stream,
+)
+from pyjelly.options import LookupPreset, SparqlOptions
+from pyjelly.parse.sparql import sparql_frames
+from pyjelly.serialize.sparql import FrameWriter
+
+EX = "http://example.org/"
+
+DATA = f"""
+@prefix ex: <{EX}> .
+ex:a ex:name "Alice" ; ex:age 42 ; ex:knows ex:b, _:c .
+ex:b ex:name "Bob"@en .
+_:c ex:name "Carol" .
+"""
+
+SELECT = """
+SELECT ?person ?name ?age WHERE {
+ ?person ?name .
+ OPTIONAL { ?person ?age }
+}
+ORDER BY ?name
+"""
+
+
+@pytest.fixture
+def graph() -> Graph:
+ return Graph().parse(data=DATA, format="turtle")
+
+
+def _round_trip(result: Result, **kwargs: Any) -> JellySparqlResult:
+ data = result.serialize(format="jelly", **kwargs)
+ assert data is not None
+ parsed = Result.parse(io.BytesIO(data), format="jelly")
+ assert isinstance(parsed, JellySparqlResult)
+ return parsed
+
+
+def test_select_round_trip(graph: Graph) -> None:
+ result = graph.query(SELECT)
+ parsed = _round_trip(result)
+ assert parsed.type == "SELECT"
+ assert parsed.vars == [Variable("person"), Variable("name"), Variable("age")]
+ assert parsed == result
+ assert not parsed.truncated
+ rows = {row.name: row for row in parsed if isinstance(row, ResultRow)}
+ assert rows[Literal("Alice")].age == Literal(42)
+ assert rows[Literal("Bob", lang="en")].person == URIRef(EX + "b")
+ assert isinstance(rows[Literal("Carol")].person, BNode)
+ # Unbound variables are left out of the bindings, as in rdflib
+ assert sum(Variable("age") in binding for binding in parsed.bindings) == 1
+
+
+@pytest.mark.parametrize("query", ["ASK { ?s ?p 42 }", "ASK { ?s ?p 43 }"])
+def test_ask_round_trip(graph: Graph, query: str) -> None:
+ result = graph.query(query)
+ parsed = _round_trip(result)
+ assert parsed.type == "ASK"
+ assert parsed.askAnswer is result.askAnswer
+ assert bool(parsed) is bool(result)
+
+
+@pytest.mark.parametrize("name", ["application/x-jelly-sparql", "jellys"])
+def test_other_format_names(graph: Graph, name: str) -> None:
+ result = graph.query(SELECT)
+ data = result.serialize(format=name)
+ assert data is not None
+ assert Result.parse(io.BytesIO(data), format=name) == result
+
+
+def test_media_type(graph: Graph) -> None:
+ result = graph.query(SELECT)
+ data = result.serialize(format="application/x-jelly-sparql")
+ assert data is not None
+ parsed = Result.parse(
+ io.BytesIO(data), content_type="application/x-jelly-sparql; charset=utf-8"
+ )
+ assert parsed == result
+
+
+def test_files(graph: Graph, tmp_path: Path) -> None:
+ result = graph.query(SELECT)
+ path = tmp_path / "results.jellys"
+ result.serialize(str(path), format="jelly")
+ # Paths work too, although rdflib's type hints only allow streams
+ assert Result.parse(str(path), format="jelly") == result # type: ignore[arg-type]
+ assert Result.parse(path, format="jelly") == result # type: ignore[arg-type]
+ with path.open("rb") as file:
+ assert parse_jelly_sparql(file) == result
+
+
+def test_empty_and_zero_variable_results(graph: Graph) -> None:
+ empty = graph.query("SELECT ?s WHERE { ?s ?p 'nothing' }")
+ parsed = _round_trip(empty)
+ assert parsed.vars == [Variable("s")]
+ assert len(parsed) == 0
+ # One solution that binds no variables
+ one = graph.query("SELECT * WHERE {}")
+ parsed = _round_trip(one)
+ assert parsed.vars == []
+ assert parsed.bindings == [{}]
+
+
+def test_options(graph: Graph) -> None:
+ options = SparqlOptions(
+ lookup_preset=LookupPreset(max_names=128, max_prefixes=0, max_datatypes=8),
+ frame_values=2,
+ stream_name="people",
+ )
+ result = graph.query(SELECT)
+ data = result.serialize(format="jelly", options=options)
+ assert data is not None
+ frames = list(sparql_frames(io.BytesIO(data)))
+ # 3 rows of 3 variables in frames of at most 2 values: one row per frame
+ assert [frame.row_count for frame in frames] == [1, 1, 1, 0]
+ assert frames[0].options.stream_name == "people"
+ assert frames[0].options.max_prefix_table_size == 0
+ assert Result.parse(io.BytesIO(data), format="jelly") == result
+
+
+def test_non_delimited(graph: Graph) -> None:
+ result = graph.query(SELECT)
+ data = result.serialize(format="jelly", options=SparqlOptions(delimited=False))
+ assert data is not None
+ (frame,) = sparql_frames(io.BytesIO(data))
+ assert frame.row_count == len(result)
+ assert frame.HasField("trailer")
+ assert Result.parse(io.BytesIO(data), format="jelly") == result
+
+
+def test_links() -> None:
+ result = JellySparqlResult("SELECT")
+ result.vars = [Variable("x")]
+ result.bindings = [{Variable("x"): URIRef(EX + "a")}]
+ result.links = [EX + "about"]
+ parsed = _round_trip(result)
+ assert parsed.links == [EX + "about"]
+
+
+def _failing_bindings() -> Iterator[dict[Variable, Any]]:
+ yield {Variable("x"): URIRef(EX + "a")}
+ msg = "the endpoint went away"
+ raise RuntimeError(msg)
+
+
+def test_error_midway_ends_the_stream_with_it() -> None:
+ result = Result("SELECT")
+ result.vars = [Variable("x")]
+ result.bindings = _failing_bindings()
+ out = io.BytesIO()
+ with pytest.raises(RuntimeError, match="went away"):
+ serialize_jelly_sparql(result, out)
+ out.seek(0)
+ with pytest.raises(JellyIncompleteResultsError, match="went away") as info:
+ Result.parse(out, format="jelly")
+ assert info.value.error == "RuntimeError: the endpoint went away"
+ partial = info.value.result
+ assert isinstance(partial, JellySparqlResult)
+ assert partial.bindings == [{Variable("x"): URIRef(EX + "a")}]
+
+
+def test_truncated(graph: Graph) -> None:
+ result = graph.query(SELECT)
+ data = result.serialize(format="jelly", options=SparqlOptions(frame_values=3))
+ assert data is not None
+ # Cut off the last frame, which has the trailer
+ frames = list(sparql_frames(io.BytesIO(data)))
+ cut = data[: -(frames[-1].ByteSize() + 1)]
+ parsed = Result.parse(io.BytesIO(cut), format="jelly")
+ assert parsed.truncated
+ assert len(parsed) == len(result)
+
+
+def test_sequence_of_results(graph: Graph) -> None:
+ results = [graph.query(SELECT), graph.query("ASK { ?s ?p ?o }")]
+ out = io.BytesIO()
+ serialize_jelly_sparql_sequence(results, out)
+ out.seek(0)
+ parsed = list(parse_jelly_sparql_sequence(out))
+ assert parsed == results
+ out.seek(0)
+ with pytest.raises(JellyConformanceError, match="PUNCTUATED"):
+ parse_jelly_sparql(out)
+
+
+def test_sequence_needs_punctuated_options(graph: Graph) -> None:
+ with pytest.raises(ValueError, match="punctuated"):
+ serialize_jelly_sparql_sequence(
+ [graph.query(SELECT)], io.BytesIO(), SparqlOptions()
+ )
+
+
+def test_graph_results_are_not_jelly_sparql(graph: Graph) -> None:
+ result = graph.query("CONSTRUCT WHERE { ?s ?p ?o }")
+ with pytest.raises(JellyConformanceError, match="CONSTRUCT"):
+ serialize_jelly_sparql(result, io.BytesIO())
+
+
+def test_rdf_12_terms_are_not_supported() -> None:
+ from pyjelly.integrations.generic.generic_sink import IRI, Triple # noqa: PLC0415
+ from pyjelly.integrations.generic.sparql import ( # noqa: PLC0415
+ serialize_jelly_sparql as generic_serialize,
+ )
+
+ out = io.BytesIO()
+ iri = IRI(EX + "a")
+ generic_serialize(["t"], [(Triple(iri, iri, iri),)], out)
+ out.seek(0)
+ with pytest.raises(NotImplementedError, match="triple terms"):
+ Result.parse(out, format="jelly")
+
+
+class _Endpoint(BaseHTTPRequestHandler):
+ """A SPARQL endpoint that answers every query with the same Jelly-SPARQL."""
+
+ body = b""
+ accept = ""
+
+ def do_GET(self) -> None:
+ type(self).accept = self.headers["Accept"]
+ self.send_response(200)
+ self.send_header("Content-Type", "application/x-jelly-sparql")
+ self.end_headers()
+ self.wfile.write(self.body)
+
+ def log_message(self, *args: object) -> None:
+ pass
+
+
+def test_sparql_store(graph: Graph) -> None:
+ result = graph.query(SELECT)
+ body = result.serialize(format="jelly")
+ assert body is not None
+ _Endpoint.body = body
+ server = HTTPServer(("127.0.0.1", 0), _Endpoint)
+ thread = threading.Thread(target=server.serve_forever, daemon=True)
+ thread.start()
+ try:
+ store = SPARQLStore(
+ f"http://127.0.0.1:{server.server_port}/sparql",
+ returnFormat="application/x-jelly-sparql",
+ )
+ remote = Graph(store=store).query(SELECT)
+ finally:
+ server.shutdown()
+ assert _Endpoint.accept == "application/x-jelly-sparql"
+ assert remote == result
+
+
+def test_validate_iris_is_passed_on(caplog: pytest.LogCaptureFixture) -> None:
+ result = Result("SELECT")
+ result.vars = [Variable("x")]
+ result.bindings = [{Variable("x"): URIRef(EX + "a b")}]
+ data = result.serialize(format="jelly")
+ assert data is not None
+ caplog.clear()
+ Result.parse(io.BytesIO(data), format="jelly")
+ assert not caplog.records
+ Result.parse(io.BytesIO(data), format="jelly", validate_iris=True)
+ assert "does not look like a valid URI" in caplog.text
+
+
+def test_frames_of_many_terms() -> None:
+ # More distinct IRIs than the name lookup holds, so frames end early
+ options = SparqlOptions(
+ lookup_preset=LookupPreset(max_names=128, max_prefixes=8, max_datatypes=8)
+ )
+ result = Result("SELECT")
+ result.vars = [Variable("s"), Variable("o")]
+ result.bindings = [
+ {Variable("s"): URIRef(f"{EX}s{i}"), Variable("o"): Literal(i)}
+ for i in range(1000)
+ ]
+ out = io.BytesIO()
+ serialize_jelly_sparql(result, out, options)
+ out.seek(0)
+ frames = list(sparql_frames(out))
+ max_names = options.lookup_preset.max_names
+ assert len(frames) > len(result) // max_names
+ assert all(frame.row_count <= max_names for frame in frames)
+ out.seek(0)
+ assert Result.parse(out, format="jelly") == result
+
+
+def test_stream_options_are_written_once(graph: Graph) -> None:
+ data = graph.query(SELECT).serialize(
+ format="jelly", options=SparqlOptions(frame_values=3)
+ )
+ assert data is not None
+ frames = list(sparql_frames(io.BytesIO(data)))
+ has_options = [frame.HasField("options") for frame in frames]
+ assert has_options == [True] + [False] * (len(frames) - 1)
+ assert frames[0].options.version == 1
+ assert frames[0].options.stream_type == jelly.SPARQL_STREAM_TYPE_FLAT
+ assert list(frames[0].variables) == ["person", "name", "age"]
+ assert all(not frame.variables for frame in frames[1:])
+
+
+def test_lazy(graph: Graph) -> None:
+ result = graph.query(SELECT)
+ data = result.serialize(format="jelly", options=SparqlOptions(frame_values=3))
+ assert data is not None
+ inp = io.BytesIO(data)
+ parsed = Result.parse(inp, format="jelly", lazy=True)
+ # Only the first frame was read
+ assert inp.tell() < len(data)
+ assert parsed.vars == result.vars
+ assert len(list(parsed)) == len(result)
+ assert inp.tell() == len(data)
+ assert parsed == result
+ assert isinstance(parsed, JellySparqlResult)
+ assert not parsed.truncated
+
+
+def test_lazy_incomplete() -> None:
+ result = Result("SELECT")
+ result.vars = [Variable("x")]
+ result.bindings = _failing_bindings()
+ out = io.BytesIO()
+ with pytest.raises(RuntimeError):
+ serialize_jelly_sparql(result, out)
+ out.seek(0)
+ parsed = parse_jelly_sparql(out, lazy=True)
+ rows: list[object] = []
+ with pytest.raises(JellyIncompleteResultsError): # noqa: PT012
+ # Not rows.extend(parsed): that calls len(parsed), which reads all rows
+ for row in parsed:
+ rows.append(row) # noqa: PERF402
+ assert len(rows) == 1
+
+
+def test_lazy_needs_a_stream(tmp_path: Path) -> None:
+ path = tmp_path / "results.jellys"
+ with pytest.raises(ValueError, match="stream"):
+ Result.parse(path, format="jelly", lazy=True) # type: ignore[arg-type]
+
+
+def test_partly_iterated_result(graph: Graph) -> None:
+ result = graph.query(SELECT)
+ for _ in result:
+ break
+ parsed = _round_trip(result)
+ assert len(parsed) == len(graph.query(SELECT))
+
+
+def test_bindings_with_names_as_keys() -> None:
+ result = Result("SELECT")
+ result.vars = [Variable("x"), Variable("y")]
+ result.bindings = [{"x": URIRef(EX + "a")}, {Variable("y"): Literal(1)}] # type: ignore[dict-item]
+ parsed = _round_trip(result)
+ assert parsed.bindings == [
+ {Variable("x"): URIRef(EX + "a")},
+ {Variable("y"): Literal(1)},
+ ]
+
+
+def test_results_without_variables_or_answer() -> None:
+ with pytest.raises(JellyConformanceError, match="no variables"):
+ serialize_jelly_sparql(Result("SELECT"), io.BytesIO())
+ with pytest.raises(JellyConformanceError, match="no answer"):
+ serialize_jelly_sparql(Result("ASK"), io.BytesIO())
+
+
+def test_sequence_goes_on_after_incomplete_results(graph: Graph) -> None:
+ failing = Result("SELECT")
+ failing.vars = [Variable("x")]
+ failing.bindings = _failing_bindings()
+ out = io.BytesIO()
+ with pytest.raises(RuntimeError):
+ serialize_jelly_sparql_sequence([graph.query("ASK { ?s ?p ?o }"), failing], out)
+ serialize_jelly_sparql_sequence([graph.query(SELECT)], out)
+ out.seek(0)
+ first, second, third = parse_jelly_sparql_sequence(out)
+ assert first.askAnswer is True
+ assert not first.error
+ assert second.error == "RuntimeError: the endpoint went away"
+ assert len(second) == 1
+ # Not equal, because each result set in a sequence has its own blank nodes
+ assert len(third) == len(graph.query(SELECT))
+
+
+def test_lazy_links() -> None:
+ result = JellySparqlResult("SELECT")
+ result.vars = [Variable("x")]
+ result.bindings = [{Variable("x"): URIRef(EX + "a")}]
+ result.links = [EX + "about"]
+ data = result.serialize(format="jelly")
+ assert data is not None
+ parsed = Result.parse(io.BytesIO(data), format="jelly", lazy=True)
+ assert isinstance(parsed, JellySparqlResult)
+ assert parsed.links == [EX + "about"]
+
+
+def test_incomplete_and_truncated_results_are_passed_on(graph: Graph) -> None:
+ data = graph.query(SELECT).serialize(
+ format="jelly", options=SparqlOptions(frame_values=3)
+ )
+ assert data is not None
+ frames = list(sparql_frames(io.BytesIO(data)))
+ cut = data[: -(frames[-1].ByteSize() + 1)]
+ with pytest.warns(JellyTruncatedResultsWarning):
+ parsed = Result.parse(io.BytesIO(cut), format="jelly")
+ assert parsed.truncated
+ # When written again, the results are marked as possibly incomplete
+ again = parsed.serialize(format="jelly")
+ assert again is not None
+ with pytest.raises(JellyIncompleteResultsError, match="ended without"):
+ Result.parse(io.BytesIO(again), format="jelly")
+
+
+def test_cut_frame() -> None:
+ result = Result("SELECT")
+ result.vars = [Variable("x")]
+ result.bindings = [{Variable("x"): URIRef(EX + "a")}]
+ data = result.serialize(format="jelly")
+ assert data is not None
+ with pytest.raises(JellyConformanceError, match="cut short"):
+ Result.parse(io.BytesIO(data[:-3]), format="jelly")
+
+
+def test_jelly_rdf_is_not_jelly_sparql(graph: Graph) -> None:
+ data = graph.serialize(format="jelly", encoding="utf-8")
+ with pytest.raises(JellyConformanceError):
+ Result.parse(io.BytesIO(data), format="jelly")
+
+
+def test_bytes(graph: Graph) -> None:
+ data = graph.query(SELECT).serialize(format="jelly")
+ assert data is not None
+ assert parse_jelly_sparql(data) == graph.query(SELECT)
+
+
+def test_unknown_arguments(graph: Graph) -> None:
+ data = graph.query(SELECT).serialize(format="jelly")
+ assert data is not None
+ with pytest.raises(TypeError, match="lazzy"):
+ Result.parse(io.BytesIO(data), format="jelly", lazzy=True)
+ with pytest.raises(TypeError, match="option"):
+ graph.query(SELECT).serialize(format="jelly", option=SparqlOptions())
+
+
+def test_lazy_reading_from_a_socket() -> None:
+ reader_socket, writer_socket = socket.socketpair()
+ first_row_read = threading.Event()
+ waited: list[bool] = []
+
+ def produce() -> None:
+ with writer_socket, writer_socket.makefile("wb") as out:
+ stream = sparql_stream(SparqlOptions(frame_values=1))
+ writer = FrameWriter(out, delimited=True)
+ stream.start(["x"])
+ frame = stream.row([URIRef(EX + "a")])
+ assert frame is not None
+ writer.write(frame)
+ # The reader gets the first row before the rest is written
+ waited.append(first_row_read.wait(timeout=10))
+ writer.write(stream.end())
+
+ producer = threading.Thread(target=produce)
+ producer.start()
+ with reader_socket, reader_socket.makefile("rb") as inp:
+ parsed = parse_jelly_sparql(inp, lazy=True)
+ for _ in parsed:
+ first_row_read.set()
+ producer.join()
+ assert waited == [True]
diff --git a/tests/meta.py b/tests/meta.py
index 9b0b1004..bf1352b8 100644
--- a/tests/meta.py
+++ b/tests/meta.py
@@ -16,3 +16,7 @@
RDF_TO_JELLY_MANIFESTS = [
RDF_TESTS_DIR / suite / "to_jelly" / "manifest.ttl" for suite in RDF_TEST_SUITES
]
+
+SPARQL_TESTS_DIR = CONFORMANCE_TESTS_DIR / "sparql"
+SPARQL_FROM_JELLY_MANIFEST = SPARQL_TESTS_DIR / "from_jelly" / "manifest.ttl"
+SPARQL_TO_JELLY_MANIFEST = SPARQL_TESTS_DIR / "to_jelly" / "manifest.ttl"
diff --git a/tests/unit_tests/test_parse/test_lookup_decoder.py b/tests/unit_tests/test_parse/test_lookup_decoder.py
index 9a306c96..ab1b0568 100644
--- a/tests/unit_tests/test_parse/test_lookup_decoder.py
+++ b/tests/unit_tests/test_parse/test_lookup_decoder.py
@@ -24,3 +24,10 @@ def test_at_invalid_index() -> None:
decoder.at(2)
assert "invalid resolved index 2" in str(excinfo.value)
assert decoder.last_reused_index == 2
+
+
+def test_assign_entry_outside_lookup() -> None:
+ decoder = LookupDecoder(lookup_size=2)
+ decoder.assign_entry(2, "b")
+ with pytest.raises(JellyConformanceError, match="the lookup has 2 entries"):
+ decoder.assign_entry(0, "c")
diff --git a/tests/unit_tests/test_parse/test_sparql.py b/tests/unit_tests/test_parse/test_sparql.py
new file mode 100644
index 00000000..0cfa86de
--- /dev/null
+++ b/tests/unit_tests/test_parse/test_sparql.py
@@ -0,0 +1,79 @@
+from __future__ import annotations
+
+import io
+
+import pytest
+
+from pyjelly import jelly
+from pyjelly.errors import JellyConformanceError
+from pyjelly.integrations.generic.decode import GenericStatementSinkAdapter
+from pyjelly.integrations.generic.generic_sink import IRI
+from pyjelly.options import MAX_SPARQL_FRAME_ROWS, MAX_SPARQL_FRAME_VALUES
+from pyjelly.parse.sparql import SparqlDecoder, read_result_set
+from pyjelly.serialize.ioutils import write_delimited, write_single
+
+OPTIONS = jelly.SparqlResultsOptions(max_name_table_size=128, version=1)
+
+
+def _decoder(
+ *, max_rows: int = MAX_SPARQL_FRAME_ROWS, max_values: int = MAX_SPARQL_FRAME_VALUES
+) -> SparqlDecoder:
+ return SparqlDecoder(
+ GenericStatementSinkAdapter, max_rows=max_rows, max_values=max_values
+ )
+
+
+def _frame(**fields: object) -> jelly.SparqlResultsFrame:
+ return jelly.SparqlResultsFrame(options=OPTIONS, **fields) # type: ignore[arg-type]
+
+
+def test_row_limit() -> None:
+ decoder = _decoder(max_rows=10)
+ with pytest.raises(JellyConformanceError, match="more than the 10 allowed"):
+ decoder.decode(_frame(variables=["x"], row_count=11))
+
+
+def test_value_limit() -> None:
+ # Checked before any cells are built
+ decoder = _decoder(max_values=100)
+ frame = _frame(variables=["x", "y"], row_count=51)
+ with pytest.raises(JellyConformanceError, match="100 values allowed"):
+ decoder.decode(frame)
+
+
+def test_zero_variables() -> None:
+ decoded = _decoder().decode(_frame(row_count=3))
+ assert decoded.variables == []
+ assert decoded.rows == [(), (), ()]
+
+
+def test_column_padded_with_unbound_cells() -> None:
+ column = jelly.RdfColumn(name_ids=[1])
+ frame = _frame(
+ variables=["x"],
+ row_count=3,
+ names=[jelly.RdfLookupEntryPacked(values=["http://example.org/a"])],
+ columns=[column],
+ )
+ decoded = _decoder().decode(frame)
+ assert decoded.rows == [(IRI("http://example.org/a"),), (None,), (None,)]
+
+
+def test_links_not_utf8_are_ignored() -> None:
+ decoded = _decoder().decode(_frame(metadata={"link": b"\xff"}))
+ assert decoded.links is None
+
+
+@pytest.mark.parametrize("delimited", [True, False])
+def test_read_delimited_or_not(*, delimited: bool) -> None:
+ out = io.BytesIO()
+ frame = _frame(variables=["x"], trailer=jelly.SparqlResultsTrailer())
+ if delimited:
+ write_delimited(frame, out)
+ else:
+ write_single(frame, out)
+ out.seek(0)
+ results = read_result_set(out, GenericStatementSinkAdapter)
+ assert results.variables == ["x"]
+ assert list(results) == []
+ assert not results.truncated
diff --git a/tests/unit_tests/test_serialize/test_rdflib_walk.py b/tests/unit_tests/test_serialize/test_rdflib_walk.py
index 5ab45514..6d05ee98 100644
--- a/tests/unit_tests/test_serialize/test_rdflib_walk.py
+++ b/tests/unit_tests/test_serialize/test_rdflib_walk.py
@@ -6,9 +6,10 @@
import pytest
from rdflib import RDF, XSD, BNode, Dataset, Graph, Literal, URIRef
-from rdflib.graph import DATASET_DEFAULT_GRAPH_ID
+from rdflib.graph import DATASET_DEFAULT_GRAPH_ID, ConjunctiveGraph
from rdflib.plugins.stores.memory import Memory, SimpleMemory
+from pyjelly import jelly
from pyjelly.integrations.rdflib import walk
from pyjelly.options import VERSION_1_1, VERSION_1_2, StreamParameters
from pyjelly.serialize.streams import SerializerOptions
@@ -175,6 +176,46 @@ def quads(dataset: Graph) -> Iterable[Any]:
assert round_trip(shared, set, version) == set(shared)
+@pytest.mark.parametrize("version", [VERSION_1_1, VERSION_1_2])
+@pytest.mark.parametrize("path", ["fast", "public", "other store"])
+def test_serialize_conjunctive_graph(
+ path: str, version: int, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ # rdfpipe reads into a ConjunctiveGraph. Its default context is named with a
+ # random blank node, and is written as the default graph, as with TriG
+ if path == "public":
+ public_path_only(monkeypatch)
+ store = OtherStore if path == "other store" else Memory
+ graph = ConjunctiveGraph(store=store())
+ add_statements(graph.default_context)
+ named = graph.get_context(ex("g"))
+ add_statements(named)
+ named.add((ex("s"), ex("p"), ex("only in g")))
+
+ out = io.BytesIO()
+ options = SerializerOptions(params=StreamParameters(version=version))
+ graph.serialize(destination=out, format="jelly", options=options)
+ read = Dataset()
+ read.parse(data=out.getvalue(), format="jelly")
+
+ assert {g.identifier for g in read.graphs()} == {DATASET_DEFAULT_GRAPH_ID, ex("g")}
+ assert set(read.default_graph) == set(graph.default_context)
+ assert set(read.graph(ex("g"))) == set(named)
+
+
+def test_serialize_conjunctive_graph_as_graphs() -> None:
+ # A stream of graphs has one frame per context of the ConjunctiveGraph
+ graph = ConjunctiveGraph()
+ add_statements(graph.default_context)
+ graph.get_context(ex("g")).add((ex("s"), ex("p"), ex("o")))
+ out = io.BytesIO()
+ options = SerializerOptions(logical_type=jelly.LOGICAL_STREAM_TYPE_GRAPHS)
+ graph.serialize(destination=out, format="jelly", options=options)
+ read = Graph()
+ read.parse(data=out.getvalue(), format="jelly")
+ assert set(read) == set(graph.default_context) | {(ex("s"), ex("p"), ex("o"))}
+
+
def test_other_stores_are_not_grouped_in_memory() -> None:
# Such a store can be far too big to copy into memory: its statements are
# written in its own order
diff --git a/tests/unit_tests/test_serialize/test_sparql.py b/tests/unit_tests/test_serialize/test_sparql.py
new file mode 100644
index 00000000..47577112
--- /dev/null
+++ b/tests/unit_tests/test_serialize/test_sparql.py
@@ -0,0 +1,147 @@
+from __future__ import annotations
+
+import pytest
+
+from pyjelly import jelly
+from pyjelly.errors import JellyConformanceError
+from pyjelly.integrations.generic.generic_sink import IRI, Literal
+from pyjelly.integrations.generic.serialize import GenericSinkTermEncoder
+from pyjelly.integrations.generic.sparql import sparql_stream
+from pyjelly.options import LookupPreset, SparqlOptions
+from pyjelly.serialize.sparql import SparqlStream, links_value
+
+A = IRI("http://example.org/a")
+
+
+def test_variable_names() -> None:
+ stream = sparql_stream()
+ stream.start(["?s", "$p", "o"])
+ frame = stream.end()
+ assert list(frame.variables) == ["s", "p", "o"]
+
+
+@pytest.mark.parametrize("name", ["", "?", "$"])
+def test_empty_variable_name(name: str) -> None:
+ with pytest.raises(JellyConformanceError, match="cannot be empty"):
+ sparql_stream().start([name])
+
+
+def test_first_frame() -> None:
+ stream = sparql_stream(SparqlOptions(stream_name="s"))
+ stream.start(["x"], links=["http://example.org/l1", "http://example.org/l2"])
+ stream.row([A])
+ frame = stream.end()
+ assert frame.options == jelly.SparqlResultsOptions(
+ stream_name="s",
+ max_name_table_size=8192,
+ max_prefix_table_size=1024,
+ max_datatype_table_size=64,
+ version=1,
+ )
+ assert frame.metadata["link"] == b"http://example.org/l1\nhttp://example.org/l2"
+ assert frame.row_count == 1
+ assert frame.HasField("trailer")
+ assert not frame.trailer.error
+
+
+def test_frame_without_rows_has_no_columns() -> None:
+ stream = sparql_stream()
+ stream.start(["x", "y"])
+ frame = stream.end()
+ assert frame.row_count == 0
+ assert not frame.columns
+
+
+def test_unbound_column_is_an_empty_message() -> None:
+ stream = sparql_stream()
+ stream.start(["x", "y"])
+ stream.row([A, None])
+ frame = stream.end()
+ assert len(frame.columns) == 2
+ assert frame.columns[1] == jelly.RdfColumn()
+
+
+def test_row_length() -> None:
+ stream = sparql_stream()
+ stream.start(["x", "y"])
+ with pytest.raises(JellyConformanceError, match="expected 2 values"):
+ stream.row([A])
+
+
+def test_order_of_calls() -> None:
+ stream = sparql_stream()
+ with pytest.raises(JellyConformanceError, match="start a result set"):
+ stream.row([A])
+ with pytest.raises(JellyConformanceError, match="no result set to end"):
+ stream.end()
+ stream.start(["x"])
+ with pytest.raises(JellyConformanceError, match="not ended yet"):
+ stream.start(["x"])
+ with pytest.raises(JellyConformanceError, match="not ended yet"):
+ stream.ask(True) # noqa: FBT003
+ stream.end()
+ # A FLAT stream has one result set
+ with pytest.raises(JellyConformanceError, match="punctuated=True"):
+ stream.start(["x"])
+ with pytest.raises(JellyConformanceError, match="punctuated=True"):
+ stream.ask(True) # noqa: FBT003
+
+
+def test_failed_row() -> None:
+ stream = sparql_stream(
+ SparqlOptions(lookup_preset=LookupPreset(max_names=128, max_datatypes=0))
+ )
+ stream.start(["x"])
+ stream.row([A])
+ with pytest.raises(JellyConformanceError, match="datatype lookup"):
+ stream.row([Literal("1", datatype="http://example.org/type")])
+ with pytest.raises(JellyConformanceError, match="failed to encode"):
+ stream.row([A])
+ with pytest.raises(JellyConformanceError, match="failed to encode"):
+ stream.end()
+ frame = stream.end("the row failed")
+ # The frame's rows are dropped
+ assert frame.row_count == 0
+ assert frame.HasField("options")
+ assert list(frame.variables) == ["x"]
+ assert frame.trailer.error == "the row failed"
+ with pytest.raises(JellyConformanceError, match="failed to encode"):
+ stream.end("again")
+
+
+def test_punctuated() -> None:
+ stream = sparql_stream(SparqlOptions(punctuated=True))
+ first = stream.ask(True) # noqa: FBT003
+ assert first.HasField("options")
+ assert first.options.stream_type == jelly.SPARQL_STREAM_TYPE_PUNCTUATED
+ stream.start(["x"])
+ stream.row([A])
+ second = stream.end()
+ assert not second.HasField("options")
+ assert list(second.variables) == ["x"]
+ third = stream.ask(False) # noqa: FBT003
+ assert third.HasField("trailer")
+ assert not third.ask_result.value
+
+
+def test_punctuated_must_be_delimited() -> None:
+ with pytest.raises(ValueError, match="delimited"):
+ SparqlOptions(punctuated=True, delimited=False)
+
+
+def test_options_checks() -> None:
+ with pytest.raises(JellyConformanceError, match="at least 128"):
+ SparqlOptions(lookup_preset=LookupPreset(max_names=64))
+ with pytest.raises(ValueError, match="frame_values"):
+ SparqlOptions(frame_values=0)
+
+
+def test_encoder_lookups_must_match() -> None:
+ encoder = GenericSinkTermEncoder(lookup_preset=LookupPreset())
+ with pytest.raises(ValueError, match="lookup sizes"):
+ SparqlStream(encoder, SparqlOptions())
+
+
+def test_links_with_lf() -> None:
+ with pytest.raises(JellyConformanceError, match="LF"):
+ links_value(["http://example.org/a\nb"])
diff --git a/tests/utils/sparql_compare.py b/tests/utils/sparql_compare.py
new file mode 100644
index 00000000..ef72853f
--- /dev/null
+++ b/tests/utils/sparql_compare.py
@@ -0,0 +1,160 @@
+"""
+Helpers for Jelly-SPARQL tests: reading SPARQL JSON results and comparing results.
+
+Two results are equivalent if both are boolean results with the same value, or if
+they have the same variables and the same rows in the same order, with a
+one-to-one mapping between their blank nodes. Links are not compared.
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Iterable
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from rdflib.query import Result
+
+from pyjelly.integrations.generic.generic_sink import (
+ IRI,
+ BlankNode,
+ Literal,
+ Node,
+ Triple,
+)
+from pyjelly.parse.sparql import SparqlResultSet
+from tests.utils.rdf_compare import (
+ Term,
+ _BlankNodes,
+ _literal,
+ generic_term,
+ rdflib_term,
+)
+
+Row = tuple[Term | None, ...]
+
+
+@dataclass
+class Results:
+ """Query results, with terms in the form used by rdf_compare."""
+
+ variables: list[str] = field(default_factory=list)
+ rows: list[Row] = field(default_factory=list)
+ boolean: bool | None = None
+
+
+def _json_term(value: dict[str, Any]) -> Term:
+ kind = value["type"]
+ if kind == "uri":
+ return ("iri", value["value"])
+ if kind == "bnode":
+ return ("bnode", value["value"])
+ if kind in ("literal", "typed-literal"):
+ return _literal(
+ value["value"],
+ value.get("xml:lang"),
+ value.get("datatype"),
+ value.get("its:dir"),
+ )
+ if kind == "triple":
+ triple = value["value"]
+ return (
+ "triple",
+ _json_term(triple["subject"]),
+ _json_term(triple["predicate"]),
+ _json_term(triple["object"]),
+ )
+ msg = f"unknown term type: {kind}"
+ raise ValueError(msg)
+
+
+def read_srj(path: Path) -> Results:
+ """Read a file of SPARQL JSON results."""
+ data = json.loads(path.read_text("utf-8"))
+ if "boolean" in data:
+ return Results(boolean=bool(data["boolean"]))
+ variables: list[str] = data["head"]["vars"]
+ rows = [
+ tuple(_json_term(b[name]) if name in b else None for name in variables)
+ for b in data["results"]["bindings"]
+ ]
+ return Results(variables, rows)
+
+
+def generic_node(term: Term) -> Node:
+ """Convert a term to a generic API term."""
+ kind = term[0]
+ if kind == "iri":
+ return IRI(term[1])
+ if kind == "bnode":
+ return BlankNode(term[1])
+ if kind == "literal":
+ return Literal(term[1], term[2], term[3], term[4])
+ if kind == "triple":
+ return Triple(*map(generic_node, term[1:]))
+ msg = f"unknown term: {term!r}"
+ raise ValueError(msg)
+
+
+def generic_rows(results: Results) -> list[tuple[Node | None, ...]]:
+ """Convert the rows to generic API terms."""
+ return [
+ tuple(None if term is None else generic_node(term) for term in row)
+ for row in results.rows
+ ]
+
+
+def generic_results(result_set: SparqlResultSet) -> Results:
+ """Read all rows of a generic API result set."""
+ rows = [
+ tuple(None if value is None else generic_term(value) for value in row)
+ for row in result_set
+ ]
+ return Results(result_set.variables, rows, result_set.boolean)
+
+
+def rdflib_results(result: Result) -> Results:
+ if result.type == "ASK":
+ return Results(boolean=bool(result.askAnswer))
+ variables = result.vars or []
+ rows = [
+ tuple(
+ None if binding.get(v) is None else rdflib_term(binding[v])
+ for v in variables
+ )
+ for binding in result.bindings
+ ]
+ return Results([str(v) for v in variables], rows)
+
+
+def assert_same_results(expected: Results, actual: Results, hint: str = "") -> None:
+ """Check that two results are equivalent (see the module)."""
+ assert actual.boolean == expected.boolean, (
+ f"{hint}: expected the boolean result {expected.boolean}, got {actual.boolean}"
+ )
+ assert actual.variables == expected.variables, (
+ f"{hint}: expected the variables {expected.variables}, got {actual.variables}"
+ )
+ assert len(actual.rows) == len(expected.rows), (
+ f"{hint}: expected {len(expected.rows)} rows, got {len(actual.rows)}"
+ )
+ blank_nodes = _BlankNodes()
+ for index, (e, a) in enumerate(zip(expected.rows, actual.rows, strict=True)):
+ same = len(e) == len(a) and all(
+ x is y if x is None or y is None else blank_nodes.same(x, y)
+ for x, y in zip(e, a, strict=True)
+ )
+ assert same, f"{hint}: row {index} differs: expected {e}, got {a}"
+
+
+def assert_same_result_sets(
+ expected: Iterable[Results], actual: Iterable[Results], hint: str = ""
+) -> None:
+ expected = list(expected)
+ actual = list(actual)
+ assert len(actual) == len(expected), (
+ f"{hint}: expected {len(expected)} result sets, got {len(actual)}"
+ )
+ for index, (e, a) in enumerate(zip(expected, actual, strict=True)):
+ assert_same_results(e, a, hint=f"{hint}: result set {index}")