From 662b348e703601243dd3ec1fce11e25f6489cd87 Mon Sep 17 00:00:00 2001 From: Ostrzyciel Date: Mon, 26 Oct 2026 17:52:03 +0100 Subject: [PATCH 01/24] Generate the Jelly-SPARQL protobuf module sparql.proto imports rdf.proto, so the script points the generated import at pyjelly.jelly. The messages are exported from pyjelly.jelly, like the ones of Jelly-RDF. --- pyjelly/jelly/__init__.py | 4 +- pyjelly/jelly/sparql_pb2.py | 49 +++++++++++++++++++++++ pyjelly/jelly/sparql_pb2.pyi | 77 ++++++++++++++++++++++++++++++++++++ scripts/compile-proto | 7 +++- 4 files changed, 134 insertions(+), 3 deletions(-) create mode 100644 pyjelly/jelly/sparql_pb2.py create mode 100644 pyjelly/jelly/sparql_pb2.pyi diff --git a/pyjelly/jelly/__init__.py b/pyjelly/jelly/__init__.py index 1d092574..0688f8d2 100644 --- a/pyjelly/jelly/__init__.py +++ b/pyjelly/jelly/__init__.py @@ -1,5 +1,7 @@ -from pyjelly.jelly import rdf_pb2 +from pyjelly.jelly import rdf_pb2, sparql_pb2 from pyjelly.jelly.rdf_pb2 import * +from pyjelly.jelly.sparql_pb2 import * # workaround for mypyc globals().update(vars(rdf_pb2)) +globals().update(vars(sparql_pb2)) diff --git a/pyjelly/jelly/sparql_pb2.py b/pyjelly/jelly/sparql_pb2.py new file mode 100644 index 00000000..5e87e2d3 --- /dev/null +++ b/pyjelly/jelly/sparql_pb2.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: sparql.proto +# Protobuf Python Version: 6.31.0 +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, + 6, + 31, + 0, + '', + 'sparql.proto' +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from pyjelly.jelly import rdf_pb2 as rdf__pb2 + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0csparql.proto\x12(eu.ostrzyciel.jelly.core.proto.v1.sparql\x1a\trdf.proto\"\xae\x02\n\x14SparqlResultsOptions\x12\x13\n\x0bstream_name\x18\x01 \x01(\t\x12O\n\x0bstream_type\x18\x02 \x01(\x0e\x32:.eu.ostrzyciel.jelly.core.proto.v1.sparql.SparqlStreamType\x12\x42\n\x0brdf_version\x18\x05 \x01(\x0e\x32-.eu.ostrzyciel.jelly.core.proto.v1.RdfVersion\x12\x1b\n\x13max_name_table_size\x18\t \x01(\r\x12\x1d\n\x15max_prefix_table_size\x18\n \x01(\r\x12\x1f\n\x17max_datatype_table_size\x18\x0b \x01(\r\x12\x0f\n\x07version\x18\x0f \x01(\r\" \n\x0fSparqlAskResult\x12\r\n\x05value\x18\x01 \x01(\x08\"%\n\x14SparqlResultsTrailer\x12\r\n\x05\x65rror\x18\x01 \x01(\t\"\xd8\x05\n\x12SparqlResultsFrame\x12O\n\x07options\x18\x01 \x01(\x0b\x32>.eu.ostrzyciel.jelly.core.proto.v1.sparql.SparqlResultsOptions\x12\x11\n\tvariables\x18\x02 \x03(\t\x12\x11\n\trow_count\x18\x03 \x01(\r\x12\x46\n\x05names\x18\x04 \x03(\x0b\x32\x37.eu.ostrzyciel.jelly.core.proto.v1.RdfLookupEntryPacked\x12I\n\x08prefixes\x18\x05 \x03(\x0b\x32\x37.eu.ostrzyciel.jelly.core.proto.v1.RdfLookupEntryPacked\x12J\n\tdatatypes\x18\x06 \x03(\x0b\x32\x37.eu.ostrzyciel.jelly.core.proto.v1.RdfLookupEntryPacked\x12=\n\x07\x63olumns\x18\x07 \x03(\x0b\x32,.eu.ostrzyciel.jelly.core.proto.v1.RdfColumn\x12M\n\nask_result\x18\x08 \x01(\x0b\x32\x39.eu.ostrzyciel.jelly.core.proto.v1.sparql.SparqlAskResult\x12O\n\x07trailer\x18\t \x01(\x0b\x32>.eu.ostrzyciel.jelly.core.proto.v1.sparql.SparqlResultsTrailer\x12\\\n\x08metadata\x18\x0f \x03(\x0b\x32J.eu.ostrzyciel.jelly.core.proto.v1.sparql.SparqlResultsFrame.MetadataEntry\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x0c:\x02\x38\x01*R\n\x10SparqlStreamType\x12\x1b\n\x17SPARQL_STREAM_TYPE_FLAT\x10\x00\x12!\n\x1dSPARQL_STREAM_TYPE_PUNCTUATED\x10\x01\x62\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'sparql_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + DESCRIPTOR._loaded_options = None + _globals['_SPARQLRESULTSFRAME_METADATAENTRY']._loaded_options = None + _globals['_SPARQLRESULTSFRAME_METADATAENTRY']._serialized_options = b'8\001' + _globals['_SPARQLSTREAMTYPE']._serialized_start=1178 + _globals['_SPARQLSTREAMTYPE']._serialized_end=1260 + _globals['_SPARQLRESULTSOPTIONS']._serialized_start=70 + _globals['_SPARQLRESULTSOPTIONS']._serialized_end=372 + _globals['_SPARQLASKRESULT']._serialized_start=374 + _globals['_SPARQLASKRESULT']._serialized_end=406 + _globals['_SPARQLRESULTSTRAILER']._serialized_start=408 + _globals['_SPARQLRESULTSTRAILER']._serialized_end=445 + _globals['_SPARQLRESULTSFRAME']._serialized_start=448 + _globals['_SPARQLRESULTSFRAME']._serialized_end=1176 + _globals['_SPARQLRESULTSFRAME_METADATAENTRY']._serialized_start=1129 + _globals['_SPARQLRESULTSFRAME_METADATAENTRY']._serialized_end=1176 +# @@protoc_insertion_point(module_scope) diff --git a/pyjelly/jelly/sparql_pb2.pyi b/pyjelly/jelly/sparql_pb2.pyi new file mode 100644 index 00000000..205a10d9 --- /dev/null +++ b/pyjelly/jelly/sparql_pb2.pyi @@ -0,0 +1,77 @@ +from pyjelly.jelly import rdf_pb2 as _rdf_pb2 +from google.protobuf.internal import containers as _containers +from google.protobuf.internal import enum_type_wrapper as _enum_type_wrapper +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from collections.abc import Iterable as _Iterable, Mapping as _Mapping +from typing import ClassVar as _ClassVar, Optional as _Optional, Union as _Union + +DESCRIPTOR: _descriptor.FileDescriptor + +class SparqlStreamType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): + __slots__ = () + SPARQL_STREAM_TYPE_FLAT: _ClassVar[SparqlStreamType] + SPARQL_STREAM_TYPE_PUNCTUATED: _ClassVar[SparqlStreamType] +SPARQL_STREAM_TYPE_FLAT: SparqlStreamType +SPARQL_STREAM_TYPE_PUNCTUATED: SparqlStreamType + +class SparqlResultsOptions(_message.Message): + __slots__ = ("stream_name", "stream_type", "rdf_version", "max_name_table_size", "max_prefix_table_size", "max_datatype_table_size", "version") + STREAM_NAME_FIELD_NUMBER: _ClassVar[int] + STREAM_TYPE_FIELD_NUMBER: _ClassVar[int] + RDF_VERSION_FIELD_NUMBER: _ClassVar[int] + MAX_NAME_TABLE_SIZE_FIELD_NUMBER: _ClassVar[int] + MAX_PREFIX_TABLE_SIZE_FIELD_NUMBER: _ClassVar[int] + MAX_DATATYPE_TABLE_SIZE_FIELD_NUMBER: _ClassVar[int] + VERSION_FIELD_NUMBER: _ClassVar[int] + stream_name: str + stream_type: SparqlStreamType + rdf_version: _rdf_pb2.RdfVersion + max_name_table_size: int + max_prefix_table_size: int + max_datatype_table_size: int + version: int + def __init__(self, stream_name: _Optional[str] = ..., stream_type: _Optional[_Union[SparqlStreamType, str]] = ..., rdf_version: _Optional[_Union[_rdf_pb2.RdfVersion, str]] = ..., max_name_table_size: _Optional[int] = ..., max_prefix_table_size: _Optional[int] = ..., max_datatype_table_size: _Optional[int] = ..., version: _Optional[int] = ...) -> None: ... + +class SparqlAskResult(_message.Message): + __slots__ = ("value",) + VALUE_FIELD_NUMBER: _ClassVar[int] + value: bool + def __init__(self, value: bool = ...) -> None: ... + +class SparqlResultsTrailer(_message.Message): + __slots__ = ("error",) + ERROR_FIELD_NUMBER: _ClassVar[int] + error: str + def __init__(self, error: _Optional[str] = ...) -> None: ... + +class SparqlResultsFrame(_message.Message): + __slots__ = ("options", "variables", "row_count", "names", "prefixes", "datatypes", "columns", "ask_result", "trailer", "metadata") + class MetadataEntry(_message.Message): + __slots__ = ("key", "value") + KEY_FIELD_NUMBER: _ClassVar[int] + VALUE_FIELD_NUMBER: _ClassVar[int] + key: str + value: bytes + def __init__(self, key: _Optional[str] = ..., value: _Optional[bytes] = ...) -> None: ... + OPTIONS_FIELD_NUMBER: _ClassVar[int] + VARIABLES_FIELD_NUMBER: _ClassVar[int] + ROW_COUNT_FIELD_NUMBER: _ClassVar[int] + NAMES_FIELD_NUMBER: _ClassVar[int] + PREFIXES_FIELD_NUMBER: _ClassVar[int] + DATATYPES_FIELD_NUMBER: _ClassVar[int] + COLUMNS_FIELD_NUMBER: _ClassVar[int] + ASK_RESULT_FIELD_NUMBER: _ClassVar[int] + TRAILER_FIELD_NUMBER: _ClassVar[int] + METADATA_FIELD_NUMBER: _ClassVar[int] + options: SparqlResultsOptions + variables: _containers.RepeatedScalarFieldContainer[str] + row_count: int + names: _containers.RepeatedCompositeFieldContainer[_rdf_pb2.RdfLookupEntryPacked] + prefixes: _containers.RepeatedCompositeFieldContainer[_rdf_pb2.RdfLookupEntryPacked] + datatypes: _containers.RepeatedCompositeFieldContainer[_rdf_pb2.RdfLookupEntryPacked] + columns: _containers.RepeatedCompositeFieldContainer[_rdf_pb2.RdfColumn] + ask_result: SparqlAskResult + trailer: SparqlResultsTrailer + metadata: _containers.ScalarMap[str, bytes] + def __init__(self, options: _Optional[_Union[SparqlResultsOptions, _Mapping]] = ..., variables: _Optional[_Iterable[str]] = ..., row_count: _Optional[int] = ..., names: _Optional[_Iterable[_Union[_rdf_pb2.RdfLookupEntryPacked, _Mapping]]] = ..., prefixes: _Optional[_Iterable[_Union[_rdf_pb2.RdfLookupEntryPacked, _Mapping]]] = ..., datatypes: _Optional[_Iterable[_Union[_rdf_pb2.RdfLookupEntryPacked, _Mapping]]] = ..., columns: _Optional[_Iterable[_Union[_rdf_pb2.RdfColumn, _Mapping]]] = ..., ask_result: _Optional[_Union[SparqlAskResult, _Mapping]] = ..., trailer: _Optional[_Union[SparqlResultsTrailer, _Mapping]] = ..., metadata: _Optional[_Mapping[str, bytes]] = ...) -> None: ... diff --git a/scripts/compile-proto b/scripts/compile-proto index 9cd41ec5..45032d5a 100755 --- a/scripts/compile-proto +++ b/scripts/compile-proto @@ -9,8 +9,11 @@ OUT_DIR=${2:-"pyjelly/jelly"} command mkdir -p "$OUT_DIR" -: Compiling rdf.proto... -python -m grpc_tools.protoc "$PROTO_PATH/rdf.proto" --proto_path="$PROTO_PATH" --{python,pyi}_out="$OUT_DIR" +: Compiling rdf.proto and sparql.proto... +python -m grpc_tools.protoc "$PROTO_PATH/rdf.proto" "$PROTO_PATH/sparql.proto" --proto_path="$PROTO_PATH" --{python,pyi}_out="$OUT_DIR" + +: sparql.proto imports rdf.proto, which is in this package... +sed -i 's/^import rdf_pb2 as/from pyjelly.jelly import rdf_pb2 as/' "$OUT_DIR/sparql_pb2.py" "$OUT_DIR/sparql_pb2.pyi" # : Compiling grpc.proto... # python -m grpc_tools.protoc "$PROTO_PATH/grpc.proto" --proto_path="$PROTO_PATH" --{python,pyi,grpc_python}_out="$OUT_DIR" From 86889ae2d73a4c1512d6fc6211526ad0fed6bc73 Mon Sep 17 00:00:00 2001 From: Ostrzyciel Date: Mon, 26 Oct 2026 17:52:22 +0100 Subject: [PATCH 02/24] Implement Jelly-SPARQL Read and write SPARQL query results (SELECT and ASK) in Jelly-SPARQL 1.0, following Jelly-JVM. The columns are those of Jelly-RDF 1.2, so the encoder and decoder reuse them. - pyjelly.serialize.sparql: SparqlStream writes the frames of result sets. Frames hold 4096 values by default and end early when the lookups are full. If getting the rows fails midway, the result set ends with a trailer that has the error. - pyjelly.parse.sparql: SparqlDecoder checks and decodes the frames, and read_result_set(s) gives result sets whose rows are read as they are iterated. A trailer with an error raises JellyIncompleteResultsError after the rows; a stream that ends without a trailer is marked as truncated. - rdflib: Result.parse() and Result.serialize() read and write the "jelly" format (and application/x-jelly-sparql, so rdflib's SPARQLStore can ask endpoints for it). - The generic API reads and writes results with its own terms, including triple terms and literals with a base direction. - PUNCTUATED streams (a sequence of result sets) are supported, with blank nodes scoped to each result set. Shared code changes: TermEncoder.new_frame() budgets the datatypes of several literals, the frame writer no longer needs write() to return a count (rdflib's result streams don't), delimiting detection takes the tags of the frame type, and out-of-range lookup references are reported as JellyConformanceError instead of IndexError. --- pyjelly/errors.py | 15 + pyjelly/integrations/generic/sparql.py | 114 ++++ pyjelly/integrations/rdflib/decode.py | 18 +- pyjelly/integrations/rdflib/sparql.py | 293 +++++++++ pyjelly/options.py | 79 +++ pyjelly/parse/columns.py | 146 ++++- pyjelly/parse/ioutils.py | 68 ++- pyjelly/parse/lookup.py | 7 +- pyjelly/parse/sparql.py | 566 ++++++++++++++++++ pyjelly/serialize/encode.py | 7 +- pyjelly/serialize/ioutils.py | 32 +- pyjelly/serialize/lookup.py | 4 +- pyjelly/serialize/sparql.py | 445 ++++++++++++++ pyproject.toml | 12 +- .../test_generic/test_sparql.py | 209 +++++++ .../test_rdflib/test_sparql.py | 309 ++++++++++ .../test_parse/test_lookup_decoder.py | 7 + tests/unit_tests/test_parse/test_sparql.py | 74 +++ .../unit_tests/test_serialize/test_sparql.py | 147 +++++ 19 files changed, 2494 insertions(+), 58 deletions(-) create mode 100644 pyjelly/integrations/generic/sparql.py create mode 100644 pyjelly/integrations/rdflib/sparql.py create mode 100644 pyjelly/parse/sparql.py create mode 100644 pyjelly/serialize/sparql.py create mode 100644 tests/integration_tests/test_generic/test_sparql.py create mode 100644 tests/integration_tests/test_rdflib/test_sparql.py create mode 100644 tests/unit_tests/test_parse/test_sparql.py create mode 100644 tests/unit_tests/test_serialize/test_sparql.py diff --git a/pyjelly/errors.py b/pyjelly/errors.py index 6ffd686c..1c924326 100644 --- a/pyjelly/errors.py +++ b/pyjelly/errors.py @@ -8,3 +8,18 @@ class JellyAssertionError(AssertionError): class JellyNotImplementedError(NotImplementedError): """Raised when a future feature is not yet implemented.""" + + +class JellyIncompleteResultsError(Exception): + """ + Raised when the producer of SPARQL results says that they are incomplete. + + For example, when the query failed midway. The rows read before are valid. + """ + + def __init__(self, message: str, error: str, result: object = None) -> None: + super().__init__(message) + #: Why the producer could not complete the results, in its words + self.error = error + #: The results that were read, if the reader keeps them + self.result = result diff --git a/pyjelly/integrations/generic/sparql.py b/pyjelly/integrations/generic/sparql.py new file mode 100644 index 00000000..a72a6549 --- /dev/null +++ b/pyjelly/integrations/generic/sparql.py @@ -0,0 +1,114 @@ +""" +Jelly-SPARQL with the generic API: SPARQL query results, without rdflib. + +The values of the results are the terms of the generic API (IRI, Literal, +BlankNode, and Triple for triple terms), and None for unbound variables. +""" + +from __future__ import annotations + +from collections.abc import Iterable, Iterator, Sequence +from typing import IO + +from pyjelly.integrations.generic.decode import GenericStatementSinkAdapter +from pyjelly.integrations.generic.generic_sink import Node +from pyjelly.integrations.generic.serialize import GenericSinkTermEncoder +from pyjelly.options import SparqlOptions +from pyjelly.parse.sparql import SparqlResultSet, read_result_set, read_result_sets +from pyjelly.serialize.sparql import FrameWriter, SparqlStream, write_solutions + + +def parse_jelly_sparql(inp: IO[bytes]) -> SparqlResultSet: + """ + Read SPARQL query results from Jelly-SPARQL. + + The rows are read as they are iterated, so the input must stay open until then. + + Args: + inp (IO[bytes]): the stream, delimited or not. It must have one result set + (the FLAT stream type, the default). + + Returns: + SparqlResultSet: the results. Iterating it gives the rows, as tuples with + one value per variable, None if the variable is unbound. + + """ + return read_result_set(inp, GenericStatementSinkAdapter) + + +def parse_jelly_sparql_results(inp: IO[bytes]) -> Iterator[SparqlResultSet]: + """ + Read a sequence of SPARQL query results from Jelly-SPARQL. + + A stream of the PUNCTUATED type has any number of result sets, one of the FLAT + type has one. The rows are read as they are iterated. Rows not yet iterated when + the next result set is read are kept in memory. + + Args: + inp (IO[bytes]): the stream, delimited. + + Yields: + SparqlResultSet: the results, one per result set. + + """ + return read_result_sets(inp, GenericStatementSinkAdapter) + + +def sparql_stream(options: SparqlOptions | None = None) -> SparqlStream: + """Return a stream that writes Jelly-SPARQL with terms of the generic API.""" + if options is None: + options = SparqlOptions() + encoder = GenericSinkTermEncoder(lookup_preset=options.lookup_preset) + return SparqlStream(encoder, options) + + +def serialize_jelly_sparql( + variables: Iterable[str], + rows: Iterable[Sequence[Node | None]], + out: IO[bytes], + options: SparqlOptions | None = None, + *, + links: Sequence[str] = (), +) -> None: + """ + Write a solution sequence (the result of a SELECT query) in Jelly-SPARQL. + + If getting the rows fails midway (e.g., the query fails), the stream ends with + the error, so that readers know the results are incomplete, and the error is + raised. + + Args: + variables (Iterable[str]): names of the variables, in projection order. + rows (Iterable[Sequence[Node | None]]): the rows, with one value per + variable, None if the variable is unbound. + out (IO[bytes]): where to write the results. + options (SparqlOptions | None): options of the stream. + links (Sequence[str]): IRIs of links that describe the results. + + """ + stream = sparql_stream(options) + writer = FrameWriter(out, delimited=stream.options.delimited) + write_solutions(stream, variables, rows, writer, links) + + +def serialize_jelly_sparql_ask( + value: bool, # noqa: FBT001 + out: IO[bytes], + options: SparqlOptions | None = None, + *, + links: Sequence[str] = (), +) -> None: + """ + Write a boolean result (of an ASK query) in Jelly-SPARQL. + + Args: + value (bool): the result. + out (IO[bytes]): where to write it. + options (SparqlOptions | None): options of the stream. + links (Sequence[str]): IRIs of links that describe the result. + + """ + stream = sparql_stream(options) + FrameWriter(out, delimited=stream.options.delimited).write( + stream.ask(value, links=links) + ) diff --git a/pyjelly/integrations/rdflib/decode.py b/pyjelly/integrations/rdflib/decode.py index 68729c92..75597856 100644 --- a/pyjelly/integrations/rdflib/decode.py +++ b/pyjelly/integrations/rdflib/decode.py @@ -10,7 +10,7 @@ from collections.abc import Iterable from itertools import chain from typing import Any, Final -from typing_extensions import override +from typing_extensions import Never, override import rdflib from mypy_extensions import mypyc_attr @@ -73,6 +73,22 @@ def literal( def namespace_declaration(self, name: str, iri: rdflib.URIRef) -> Prefix: return Prefix(name, iri) + @override + def directional_literal(self, lex: str, language: str, direction: str) -> Never: + msg = ( + "rdflib does not support literals with a base direction (RDF 1.2). " + "Read the data with the generic API instead." + ) + raise NotImplementedError(msg) + + @override + def quoted_triple(self, terms: Iterable[Any]) -> Never: + msg = ( + "rdflib does not support triple terms (RDF 1.2) and quoted triples " + "(RDF-star). Read the data with the generic API instead." + ) + raise NotImplementedError(msg) + @mypyc_attr(allow_interpreted_subclasses=True) class RDFLibTriplesAdapter(RDFLibAdapter): diff --git a/pyjelly/integrations/rdflib/sparql.py b/pyjelly/integrations/rdflib/sparql.py new file mode 100644 index 00000000..f110bf90 --- /dev/null +++ b/pyjelly/integrations/rdflib/sparql.py @@ -0,0 +1,293 @@ +""" +Jelly-SPARQL for rdflib: SPARQL query results (rdflib.query.Result) in Jelly. + +With pyjelly installed, rdflib reads and writes them in the "jelly" format: + + result = graph.query("SELECT ?s ?o WHERE { ?s ?p ?o }") + result.serialize("results.jellys", format="jelly") + result = Result.parse(open("results.jellys", "rb"), format="jelly") +""" + +from __future__ import annotations + +from collections.abc import Iterable, Iterator, Mapping +from pathlib import Path +from typing import IO, Any +from typing_extensions import override + +from rdflib.query import Result, ResultParser, ResultSerializer +from rdflib.term import Identifier, Variable + +from pyjelly.errors import JellyConformanceError, JellyIncompleteResultsError +from pyjelly.integrations.rdflib.decode import RDFLibAdapter +from pyjelly.integrations.rdflib.encode import RDFLibTermEncoder +from pyjelly.options import SparqlOptions +from pyjelly.parse.decode import Adapter, ParserOptions +from pyjelly.parse.sparql import ( + AdapterFactory, + SparqlResultSet, + read_result_set, + read_result_sets, +) +from pyjelly.serialize.sparql import FrameWriter, SparqlStream, write_solutions + + +class JellySparqlResult(Result): + """ + SPARQL query results read from Jelly-SPARQL: an rdflib Result, with more. + + Attributes: + links: IRIs of links that describe the results, like head.link of SPARQL + JSON results. + truncated: whether the stream ended without saying that the results are + complete, so some may be missing. + + """ + + def __init__(self, type_: str) -> None: + super().__init__(type_) + self.links: list[str] = [] + self.truncated = False + + +def _adapter_factory(*, validate_iris: bool) -> AdapterFactory: + def make(options: ParserOptions) -> Adapter: + return RDFLibAdapter(options, validate_iris=validate_iris) + + return make + + +def _to_result(result_set: SparqlResultSet) -> JellySparqlResult: + """Read a result set whole, into an rdflib Result.""" + if result_set.boolean is not None: + result = JellySparqlResult("ASK") + result.askAnswer = result_set.boolean + else: + result = JellySparqlResult("SELECT") + result.vars = [Variable(name) for name in result_set.variables] + bindings: list[Mapping[Variable, Identifier]] = [] + result.bindings = bindings + variables = result.vars or [] + try: + # Unbound variables are left out, as rdflib does + bindings.extend( + { + variable: value + for variable, value in zip(variables, row, strict=True) + if value is not None + } + for row in result_set + ) + except JellyIncompleteResultsError as error: + result.links = result_set.links + error.result = result + raise + result.links = result_set.links + result.truncated = result_set.truncated + return result + + +def parse_jelly_sparql( + inp: IO[bytes], *, validate_iris: bool = False +) -> JellySparqlResult: + """ + Read SPARQL query results from Jelly-SPARQL. + + Args: + inp (IO[bytes]): the stream, delimited or not. It must have one result set + (the FLAT stream type, the default). + validate_iris (bool): whether rdflib checks the IRIs. An invalid IRI is + still read, rdflib only logs a warning. Checking slows down parsing. + + Returns: + JellySparqlResult: the results, an rdflib Result. + + Raises: + JellyIncompleteResultsError: if the producer said that the results are + incomplete, e.g., because the query failed midway. The results read + are in its `result`. + + """ + adapter_factory = _adapter_factory(validate_iris=validate_iris) + return _to_result(read_result_set(inp, adapter_factory)) + + +def parse_jelly_sparql_results( + inp: IO[bytes], *, validate_iris: bool = False +) -> Iterator[JellySparqlResult]: + """ + Read a sequence of SPARQL query results from Jelly-SPARQL. + + A stream of the PUNCTUATED type has any number of result sets, one of the FLAT + type has one. + + Args: + inp (IO[bytes]): the stream, delimited. + validate_iris (bool): whether rdflib checks the IRIs, see + parse_jelly_sparql(). + + Yields: + JellySparqlResult: the results, an rdflib Result for each result set. + + Raises: + JellyIncompleteResultsError: if the producer said that a result set is + incomplete. The results after it can still be read. + + """ + adapter_factory = _adapter_factory(validate_iris=validate_iris) + for result_set in read_result_sets(inp, adapter_factory): + yield _to_result(result_set) + + +def sparql_stream(options: SparqlOptions | None = None) -> SparqlStream: + """Return a stream that writes Jelly-SPARQL with terms of rdflib.""" + if options is None: + options = SparqlOptions() + encoder = RDFLibTermEncoder(lookup_preset=options.lookup_preset) + return SparqlStream(encoder, options) + + +def _write_result(stream: SparqlStream, result: Result, writer: FrameWriter) -> None: + """Write one result set: the result of a SELECT or ASK query.""" + links = getattr(result, "links", ()) + if result.type == "ASK": + writer.write(stream.ask(bool(result.askAnswer), links=links)) + return + if result.type != "SELECT": + msg = ( + f"Jelly-SPARQL is for the results of SELECT and ASK queries, not " + f"{result.type}. Write graphs in Jelly-RDF instead." + ) + raise JellyConformanceError(msg) + variables = result.vars or [] + rows = ( + [binding.get(variable) for variable in variables] + for binding in _bindings(result) + ) + write_solutions(stream, variables, rows, writer, links) + + +def _bindings(result: Result) -> Iterator[Mapping[Variable, Identifier]]: + """ + Return the bindings of a result, as they are made. + + Result.bindings would make all of them before any is written, and if making + one fails, nothing would be written at all. They are still kept in the result, + as Result.__iter__() does. + """ + made = getattr(result, "_genbindings", None) + if made is None: + yield from result.bindings + return + result._genbindings = None + kept = result._bindings + for binding in made: + kept.append(binding) + yield binding + + +def serialize_jelly_sparql( + result: Result, out: IO[bytes], options: SparqlOptions | None = None +) -> None: + """ + Write SPARQL query results (of a SELECT or ASK query) in Jelly-SPARQL. + + If getting the results fails midway (e.g., the query fails), the stream ends + with the error, so that readers know the results are incomplete, and the + error is raised. + + Args: + result (Result): the results. + out (IO[bytes]): where to write them. + options (SparqlOptions | None): options of the stream. + + """ + stream = sparql_stream(options) + _write_result(stream, result, FrameWriter(out, delimited=stream.options.delimited)) + + +def serialize_jelly_sparql_results( + results: Iterable[Result], out: IO[bytes], options: SparqlOptions | None = None +) -> None: + """ + Write a sequence of SPARQL query results in one Jelly-SPARQL stream. + + The stream is of the PUNCTUATED type: each result is a result set of its own. + + Args: + results (Iterable[Result]): the results. + out (IO[bytes]): where to write them. + options (SparqlOptions | None): options of the stream, which must be + punctuated. Defaults to SparqlOptions(punctuated=True). + + """ + if options is None: + options = SparqlOptions(punctuated=True) + if not options.punctuated: + msg = "a sequence of results needs SparqlOptions(punctuated=True)" + raise JellyConformanceError(msg) + stream = sparql_stream(options) + writer = FrameWriter(out, delimited=True) + for result in results: + _write_result(stream, result, writer) + + +def _input_stream(source: IO[bytes] | str | Path) -> IO[bytes]: + if isinstance(source, (str, Path)): + return Path(source).open("rb") + return source + + +class RDFLibJellySparqlParser(ResultParser): + """Reads SPARQL query results in Jelly-SPARQL, for rdflib's Result.parse().""" + + @override + def parse( + self, + source: IO[bytes] | str | Path, + content_type: str | None = None, + *, + validate_iris: bool = False, + **unused: Any, + ) -> Result: + """ + Read the results. + + Args: + source (IO[bytes] | str | Path): the stream, or the path of a file. + content_type (str | None): unused. + validate_iris (bool): whether rdflib checks the IRIs, see + parse_jelly_sparql(). + **unused (Any): unused arguments of Result.parse(). + + """ + inp = _input_stream(source) + if inp is source: + return parse_jelly_sparql(inp, validate_iris=validate_iris) + with inp: + return parse_jelly_sparql(inp, validate_iris=validate_iris) + + +class RDFLibJellySparqlSerializer(ResultSerializer): + """Writes SPARQL query results in Jelly-SPARQL, for rdflib's Result.serialize().""" + + @override + def serialize( + self, + stream: IO[bytes], + encoding: str = "utf-8", + *, + options: SparqlOptions | None = None, + **unused: Any, + ) -> None: + """ + Write the results. + + Args: + stream (IO[bytes]): where to write them. + encoding (str): unused, Jelly is binary. + options (SparqlOptions | None): options of the stream. + **unused (Any): unused arguments of Result.serialize(). + + """ + serialize_jelly_sparql(self.result, stream, options) diff --git a/pyjelly/options.py b/pyjelly/options.py index c6bd34aa..29a79b44 100644 --- a/pyjelly/options.py +++ b/pyjelly/options.py @@ -51,6 +51,23 @@ MIMETYPES = ("application/x-jelly-rdf",) +#: The media type of Jelly-SPARQL +SPARQL_MIMETYPE: Final = "application/x-jelly-sparql" +#: The file extension of Jelly-SPARQL +SPARQL_EXTENSION: Final = ".jellys" + +# Jelly-SPARQL has version tags of its own: 1 is Jelly-SPARQL 1.0 +SPARQL_VERSION_1_0: Final[int] = 1 +SPARQL_MAX_VERSION: Final[int] = SPARQL_VERSION_1_0 + +#: How many values (rows times variables) a writer puts in a frame of results +DEFAULT_SPARQL_FRAME_VALUES: Final[int] = 4096 +#: How many rows a reader accepts in one frame of results +MAX_SPARQL_FRAME_ROWS: Final[int] = 1 << 20 +#: How many values a reader accepts in one frame of results. It makes room for all +#: of them at once. +MAX_SPARQL_FRAME_VALUES: Final[int] = 1 << 24 + def is_row_layout(version: int) -> bool: """ @@ -93,6 +110,13 @@ def small(cls) -> Self: return cls(max_names=128, max_prefixes=32, max_datatypes=32) +#: The lookup sizes of Jelly-SPARQL streams. As in Jelly-JVM, larger than those of +#: Jelly-RDF: a row of results tends to have more distinct terms than a statement. +SPARQL_LOOKUP_PRESET: Final = LookupPreset( + max_names=8192, max_prefixes=1024, max_datatypes=64 +) + + @dataclass(frozen=True) class StreamTypes: physical_type: jelly.PhysicalStreamType = jelly.PHYSICAL_STREAM_TYPE_UNSPECIFIED @@ -218,3 +242,58 @@ def validate_type_compatibility( logical_type_name = jelly.LogicalStreamType.Name(logical_type) msg = f"{physical_type_name} is not compatible with {logical_type_name}" raise JellyAssertionError(msg) + + +@dataclass(frozen=True) +class SparqlOptions: + """ + Options of a Jelly-SPARQL stream that is written. + + Attributes: + lookup_preset: the sizes of the lookups. The name lookup must have at + least 128 entries. + rdf_version: the version of RDF whose terms may occur in the stream. + RDF 1.1 does not allow base directions or triple terms, RDF 1.2 Basic + does not allow triple terms. + punctuated: whether the stream is a sequence of result sets (the + PUNCTUATED stream type), rather than one result set. + frame_size: how many values (rows times variables) a frame has, at most. + A frame may end earlier, when the lookups are full. + delimited: whether the frames are delimited. A non-delimited stream has + one frame. + stream_name: name of the stream. + + """ + + lookup_preset: LookupPreset = SPARQL_LOOKUP_PRESET + rdf_version: jelly.RdfVersion = jelly.RDF_VERSION_UNSPECIFIED + punctuated: bool = False + frame_size: int = DEFAULT_SPARQL_FRAME_VALUES + delimited: bool = True + stream_name: str = "" + + def __post_init__(self) -> None: + max_names = self.lookup_preset.max_names + if max_names < MIN_COLUMN_NAME_LOOKUP_SIZE: + msg = ( + f"the name lookup size is {max_names}, but in Jelly-SPARQL it must " + f"be at least {MIN_COLUMN_NAME_LOOKUP_SIZE}" + ) + raise JellyConformanceError(msg) + if self.frame_size < 1: + msg = f"the frame size must be at least 1, not {self.frame_size}" + raise JellyConformanceError(msg) + if self.punctuated and not self.delimited: + msg = "a PUNCTUATED stream must be delimited" + raise JellyConformanceError(msg) + + def rows_per_frame(self, variables: int) -> int: + """ + Return how many rows a frame of a result set has, at most. + + >>> SparqlOptions().rows_per_frame(4) + 1024 + >>> SparqlOptions(frame_size=10).rows_per_frame(0) + 10 + """ + return max(1, self.frame_size // max(1, variables)) diff --git a/pyjelly/parse/columns.py b/pyjelly/parse/columns.py index 0135d41b..0315632c 100644 --- a/pyjelly/parse/columns.py +++ b/pyjelly/parse/columns.py @@ -96,6 +96,11 @@ def _check_kinds(kinds: bytes, value_count: int, position: str) -> None: raise JellyConformanceError(msg) +def _too_many_cells(position: str) -> JellyConformanceError: + msg = f"the {position} column has more cells than the frame has rows" + return JellyConformanceError(msg) + + def _kinds_mismatch(position: str, kind: int) -> JellyConformanceError: msg = ( f"the kinds of the {position} column refer to more {_KIND_NAMES[kind]} " @@ -104,6 +109,54 @@ def _kinds_mismatch(position: str, kind: int) -> JellyConformanceError: return JellyConformanceError(msg) +def _layout_cells( + values: list[Any], layouts: list[int], rows: int, position: str +) -> list[Any]: + """Return the cells that the layouts of a column give, in order.""" + cells: list[Any] = [] + index = 0 + k = 0 + while k < len(layouts): + token = layouts[k] + k += 1 + length = token & _LENGTH_MASK + if length == _LENGTH_MASK: + if k == len(layouts): + msg = f"corrupt layout of the {position} column: a length is cut off" + raise JellyConformanceError(msg) + length += layouts[k] + k += 1 + skip = token >> _SKIP_SHIFT + if skip: + if index + skip > len(values): + msg = ( + f"corrupt layout of the {position} column: it skips past its values" + ) + raise JellyConformanceError(msg) + cells += values[index : index + skip] + index += skip + if token & _UNBOUND: + term = None + count = length + 1 + elif index < len(values): + term = values[index] + index += 1 + count = length + 2 + else: + msg = ( + f"corrupt layout of the {position} column: a repeated value is past " + "its values" + ) + raise JellyConformanceError(msg) + # Checked before the cells are made, as a run can claim any length + if len(cells) + count > rows: + raise _too_many_cells(position) + cells += [term] * count + if index < len(values): + cells += values[index:] + return cells + + class _Runs: """ The cells of a column, in runs. @@ -238,6 +291,9 @@ def __init__(self, decoder: Decoder) -> None: self.options_read = False # Statements of the current RDF Message so far, in this and earlier frames self.rows_in_message = 0 + #: Put before the labels of blank nodes, to tell apart the blank nodes of + #: result sets that use the same labels (see pyjelly.parse.sparql) + self.bnode_scope = "" def iter_frame(self, frame: jelly.RdfStreamFrame) -> Iterator[Any]: """ @@ -353,15 +409,19 @@ def message_lengths(self, batch: jelly.RdfColumnBatch, rows: int) -> list[int]: return lengths def ingest_entries(self, batch: jelly.RdfColumnBatch) -> None: + self.apply_entries(batch.names, batch.prefixes, batch.datatypes) + + def apply_entries(self, names: Any, prefixes: Any, datatypes: Any) -> None: + """Apply the packed lookup entries of a frame.""" decoder = self.decoder - _apply_entries(batch.names, decoder.names) - _apply_entries(batch.prefixes, decoder.prefixes) - for entry in batch.datatypes: + _apply_entries(names, decoder.names) + _apply_entries(prefixes, decoder.prefixes) + for entry in datatypes: for value in entry.values: if value in (LANG_STRING_DATATYPE_IRI, DIR_LANG_STRING_DATATYPE_IRI): msg = f"the datatype lookup cannot have {value}" raise JellyConformanceError(msg) - _apply_entries(batch.datatypes, decoder.datatypes) + _apply_entries(datatypes, decoder.datatypes) def namespaces(self, batch: jelly.RdfColumnBatch) -> list[Any]: adapter = self.decoder.adapter @@ -459,6 +519,23 @@ def runs( runs.add(None, rows - runs.cells) return runs + def cells(self, column: jelly.RdfColumn, rows: int, position: str) -> list[Any]: + """ + Decode a column of Jelly-SPARQL into its cells, in order. + + The column can have terms of any type, and unbound cells, which are None. + """ + values = self.run_values(column, _ANY, position) + layouts: list[int] = column.layouts[:] + # Without layouts, each value takes one cell + cells = _layout_cells(values, layouts, rows, position) if layouts else values + if len(cells) > rows: + raise _too_many_cells(position) + if len(cells) < rows: + # The rest are unbound + cells += [None] * (rows - len(cells)) + return cells + def run_values( self, column: jelly.RdfColumn, allowed: int, position: str ) -> list[Any]: @@ -541,27 +618,41 @@ def iris(self, column: jelly.RdfColumn) -> list[Any]: prefix_index = prefix_ids[0] if prefix_count == 1 else 0 name_index = 0 terms: list[Any] = [] - for i in range(count): - if prefix_count > 1 and prefix_ids[i]: - prefix_index = prefix_ids[i] - name_index = name_ids[i] or name_index + 1 - name = names[name_index - 1] - prefix = prefixes[prefix_index - 1] if prefix_index else "" - if name is None or prefix is None: - msg = ( - f"an IRI refers to name {name_index} or prefix {prefix_index}, " - "which is not set" - ) - raise JellyConformanceError(msg) - cached = cache[name_index] - if cached is not None and cached.name is name and cached.prefix is prefix: - terms.append(cached.term) - else: - terms.append(decoder.new_iri(name_index, name, prefix)) + try: + for i in range(count): + if prefix_count > 1 and prefix_ids[i]: + prefix_index = prefix_ids[i] + name_index = name_ids[i] or name_index + 1 + name = names[name_index - 1] + prefix = prefixes[prefix_index - 1] if prefix_index else "" + if name is None or prefix is None: + msg = ( + f"an IRI refers to name {name_index} or prefix " + f"{prefix_index}, which is not set" + ) + raise JellyConformanceError(msg) + cached = cache[name_index] + if ( + cached is not None + and cached.name is name + and cached.prefix is prefix + ): + terms.append(cached.term) + else: + terms.append(decoder.new_iri(name_index, name, prefix)) + except IndexError: + msg = ( + f"an IRI refers to name {name_index} or prefix {prefix_index}, " + "which is outside the lookups" + ) + raise JellyConformanceError(msg) from None return terms def bnodes(self, column: jelly.RdfColumn) -> list[Any]: adapter = self.decoder.adapter + scope = self.bnode_scope + if scope: + return [adapter.bnode(scope + label) for label in column.bnodes] return [adapter.bnode(label) for label in column.bnodes] def literals(self, column: jelly.RdfColumn) -> list[Any]: @@ -605,7 +696,14 @@ def literal_kind(self, kind: int, langtags: Any, directions: Any) -> _LiteralKin if not kind: return _LiteralKind(decoder, None, None, None) if kind & 1: - datatype = decoder.datatypes.at((kind + 1) >> 1) + index = (kind + 1) >> 1 + if index > decoder.datatypes.lookup_size: + msg = ( + f"a literal kind refers to datatype {index}, which is outside the " + "datatype lookup" + ) + raise JellyConformanceError(msg) + datatype = decoder.datatypes.at(index) return _LiteralKind(decoder, None, datatype, None) index = (kind >> 1) - 1 if index >= len(langtags): @@ -646,7 +744,7 @@ def triple_term(self, triple: jelly.RdfTripleTerm, iris: IriSequence) -> Any: if kind == "s_iri": s = iris.term(triple.s_iri) elif kind == "s_bnode": - s = adapter.bnode(triple.s_bnode) + s = adapter.bnode(self.bnode_scope + triple.s_bnode) else: msg = "a triple term has no subject" raise JellyConformanceError(msg) @@ -658,7 +756,7 @@ def triple_term(self, triple: jelly.RdfTripleTerm, iris: IriSequence) -> Any: if kind == "o_iri": o = iris.term(triple.o_iri) elif kind == "o_bnode": - o = adapter.bnode(triple.o_bnode) + o = adapter.bnode(self.bnode_scope + triple.o_bnode) elif kind == "o_literal": o = self.triple_term_literal(triple.o_literal) elif kind == "o_triple_term": diff --git a/pyjelly/parse/ioutils.py b/pyjelly/parse/ioutils.py index 7186abed..c409f0af 100644 --- a/pyjelly/parse/ioutils.py +++ b/pyjelly/parse/ioutils.py @@ -69,13 +69,19 @@ def single_byte_tags(descriptor: Descriptor) -> frozenset[int]: ) """Tag bytes a non-delimited frame can start with (currently 0x0A, 0x12, 0x7A).""" +SPARQL_FRAME_TAGS: Final = single_byte_tags( + cast("Descriptor", jelly.SparqlResultsFrame.DESCRIPTOR) +) +"""Tag bytes a non-delimited frame of Jelly-SPARQL can start with.""" + -def hint_bytes_needed(first_byte: int) -> int: +def hint_bytes_needed(first_byte: int, tags: frozenset[int] = FRAME_TAGS) -> int: """ Return how many bytes `delimited_jelly_hint` needs to decide. Args: first_byte (int): first byte of the stream + tags (frozenset[int]): tag bytes a frame can start with Returns: int: number of bytes to read from the start of the stream, at most 128 @@ -87,17 +93,18 @@ def hint_bytes_needed(first_byte: int) -> int: 11 """ - if first_byte in FRAME_TAGS: + if first_byte in tags: return first_byte + 1 return 1 -def _is_frame_fields(data: bytes) -> bool: +def _is_frame_fields(data: bytes, tags: frozenset[int]) -> bool: """ Check whether `data` is exactly a sequence of fields of a stream frame. Args: data (bytes): candidate frame body + tags (frozenset[int]): tag bytes of the frame's fields Returns: bool: True if the bytes make up whole fields and nothing is left over @@ -108,7 +115,7 @@ def _is_frame_fields(data: bytes) -> bool: while pos < end: tag = data[pos] pos += 1 - if tag not in FRAME_TAGS: + if tag not in tags: return False wire_type = tag & _WIRE_TYPE_MASK if wire_type == _WIRE_VARINT: @@ -138,7 +145,7 @@ def _is_frame_fields(data: bytes) -> bool: return pos == end -def delimited_jelly_hint(header: bytes) -> bool: +def delimited_jelly_hint(header: bytes, tags: frozenset[int] = FRAME_TAGS) -> bool: """ Detect whether a Jelly stream is delimited, from its first bytes. @@ -157,6 +164,8 @@ def delimited_jelly_hint(header: bytes) -> bool: Args: header (bytes): first bytes of the stream + tags (frozenset[int]): tag bytes a frame can start with, FRAME_TAGS for + Jelly-RDF and SPARQL_FRAME_TAGS for Jelly-SPARQL Returns: bool: True if the stream is delimited @@ -199,13 +208,13 @@ def delimited_jelly_hint(header: bytes) -> bool: if not header: return False first = header[0] - if first not in FRAME_TAGS: + if first not in tags: # Not a frame tag, so it is the size of the first frame. This includes # 0, an empty frame: a stream of only empty frames is delimited too. return True if len(header) < first + 1: return False - return _is_frame_fields(header[1 : first + 1]) + return _is_frame_fields(header[1 : first + 1], tags) def _read_exactly(inp: IO[bytes], size: int) -> bytes: @@ -231,7 +240,7 @@ def _read_exactly(inp: IO[bytes], size: int) -> bytes: return b"".join(chunks) -def read_hint_header(inp: IO[bytes]) -> bytes: +def read_hint_header(inp: IO[bytes], tags: frozenset[int] = FRAME_TAGS) -> bytes: """ Read just enough of the start of the stream to detect delimiting. @@ -239,6 +248,7 @@ def read_hint_header(inp: IO[bytes]) -> bytes: Args: inp (IO[bytes]): stream to read from + tags (frozenset[int]): tag bytes a frame can start with Returns: bytes: bytes read from the start of the stream @@ -247,7 +257,7 @@ def read_hint_header(inp: IO[bytes]) -> bytes: header = inp.read(1) if not header: return header - remaining = hint_bytes_needed(header[0]) - 1 + remaining = hint_bytes_needed(header[0], tags) - 1 if remaining: header += _read_exactly(inp, remaining) return header @@ -274,6 +284,34 @@ def read(self, size: int = -1) -> bytes: return taken +def detect_delimited( + inp: IO[bytes], tags: frozenset[int] = FRAME_TAGS +) -> tuple[IO[bytes], bool]: + """ + Detect whether a stream is delimited, from its first bytes. + + Args: + inp (IO[bytes]): the stream + tags (frozenset[int]): tag bytes a frame can start with + + Returns: + tuple[IO[bytes], bool]: the stream to read from its start, and whether it + is delimited + + """ + if inp.seekable(): + header = read_hint_header(inp, tags) + inp.seek(-len(header), os.SEEK_CUR) + else: + # Input may not be seekable (e.g. a network stream) -- then we need to buffer + # it and put the bytes we looked at back in front of it. + # See also: https://github.com/Jelly-RDF/pyjelly/issues/298 + buffered = io.BufferedReader(inp) # type: ignore[arg-type, type-var, unused-ignore] + header = read_hint_header(buffered, tags) + inp = _StreamWithHeader(header, buffered) # type: ignore[assignment] + return inp, delimited_jelly_hint(header, tags) + + def frame_iterator(inp: IO[bytes]) -> Generator[jelly.RdfStreamFrame]: # protobuf annotates `input_bytes` as io.BytesIO, but any IO[bytes] works. while frame := parse_length_prefixed(jelly.RdfStreamFrame, inp): # type: ignore[arg-type] @@ -311,17 +349,7 @@ def get_options_and_frames( stream types, lookup presets and other stream options """ - if inp.seekable(): - header = read_hint_header(inp) - inp.seek(-len(header), os.SEEK_CUR) - else: - # Input may not be seekable (e.g. a network stream) -- then we need to buffer - # it and put the bytes we looked at back in front of it. - # See also: https://github.com/Jelly-RDF/pyjelly/issues/298 - buffered = io.BufferedReader(inp) # type: ignore[arg-type, type-var, unused-ignore] - header = read_hint_header(buffered) - inp = _StreamWithHeader(header, buffered) # type: ignore[assignment] - is_delimited = delimited_jelly_hint(header) + inp, is_delimited = detect_delimited(inp) if is_delimited: first_frame = None diff --git a/pyjelly/parse/lookup.py b/pyjelly/parse/lookup.py index 17dd573c..94cc56fe 100644 --- a/pyjelly/parse/lookup.py +++ b/pyjelly/parse/lookup.py @@ -33,7 +33,12 @@ def assign_entry(self, index: int, value: str) -> None: previous_index = self.last_assigned_index if index == 0: index = previous_index + 1 - assert index > 0 + if index > self.lookup_size: + msg = ( + f"a lookup entry has the index {index}, " + f"but the lookup has {self.lookup_size} entries" + ) + raise JellyConformanceError(msg) self.data[index - 1] = value self.last_assigned_index = index diff --git a/pyjelly/parse/sparql.py b/pyjelly/parse/sparql.py new file mode 100644 index 00000000..e5d87a61 --- /dev/null +++ b/pyjelly/parse/sparql.py @@ -0,0 +1,566 @@ +""" +Reading of Jelly-SPARQL: SPARQL query results in frames, one column per variable. + +SparqlDecoder decodes the frames one by one, and read_result_sets() puts their rows +together into result sets. +""" + +from __future__ import annotations + +from collections.abc import Callable, Iterator +from typing import IO, Any, Final + +from google.protobuf.proto import parse, parse_length_prefixed + +from pyjelly import jelly +from pyjelly.errors import JellyConformanceError, JellyIncompleteResultsError +from pyjelly.options import ( + MAX_DATATYPE_LOOKUP_SIZE, + MAX_NAME_LOOKUP_SIZE, + MAX_PREFIX_LOOKUP_SIZE, + MAX_SPARQL_FRAME_ROWS, + MAX_SPARQL_FRAME_VALUES, + MIN_COLUMN_NAME_LOOKUP_SIZE, + SPARQL_MAX_VERSION, + VERSION_1_2, + LookupPreset, + StreamParameters, + StreamTypes, +) +from pyjelly.parse.columns import MAX_ROWS, ColumnDecoder +from pyjelly.parse.decode import Adapter, Decoder, ParserOptions +from pyjelly.parse.ioutils import SPARQL_FRAME_TAGS, detect_delimited + +#: The metadata key of the links of a result set (head.link in SPARQL JSON results) +LINK_KEY: Final = "link" + +AdapterFactory = Callable[[ParserOptions], Adapter] + +# Methods called through the message classes rather than on messages: upb looks +# up any attribute of a message as a field name first +_FRAME_HAS_FIELD: Final = jelly.SparqlResultsFrame.HasField + +_STREAM_TYPES: Final = ( + jelly.SPARQL_STREAM_TYPE_FLAT, + jelly.SPARQL_STREAM_TYPE_PUNCTUATED, +) +_RDF_VERSIONS: Final = frozenset(jelly.RdfVersion.values()) + + +def check_sparql_options(options: jelly.SparqlResultsOptions) -> None: + """ + Check that the parser can read a Jelly-SPARQL stream with these options. + + Raises: + JellyConformanceError: if it can't. + + """ + version = options.version + if not 1 <= version <= SPARQL_MAX_VERSION: + msg = ( + f"unsupported Jelly-SPARQL version {version}, " + f"expected 1 to {SPARQL_MAX_VERSION}" + ) + raise JellyConformanceError(msg) + if options.stream_type not in _STREAM_TYPES: + msg = f"unknown stream type: {options.stream_type}" + raise JellyConformanceError(msg) + if options.rdf_version not in _RDF_VERSIONS: + msg = f"unknown RDF version: {options.rdf_version}" + raise JellyConformanceError(msg) + _check_size( + "name", + options.max_name_table_size, + MIN_COLUMN_NAME_LOOKUP_SIZE, + MAX_NAME_LOOKUP_SIZE, + ) + _check_size("prefix", options.max_prefix_table_size, 0, MAX_PREFIX_LOOKUP_SIZE) + _check_size( + "datatype", options.max_datatype_table_size, 0, MAX_DATATYPE_LOOKUP_SIZE + ) + + +def _check_size(lookup: str, size: int, smallest: int, largest: int) -> None: + if not smallest <= size <= largest: + msg = ( + f"the {lookup} lookup size is {size}, " + f"but it must be from {smallest} to {largest}" + ) + raise JellyConformanceError(msg) + + +def sparql_parser_options(options: jelly.SparqlResultsOptions) -> ParserOptions: + """ + Return the options that the adapter of a Jelly-SPARQL stream is made with. + + The terms are decoded as in a Jelly-RDF 1.2 stream with the same lookups and + RDF version. The stream types are unspecified. + """ + return ParserOptions( + stream_types=StreamTypes(), + lookup_preset=LookupPreset( + max_names=options.max_name_table_size, + max_prefixes=options.max_prefix_table_size, + max_datatypes=options.max_datatype_table_size, + ), + params=StreamParameters( + version=VERSION_1_2, + stream_name=options.stream_name, + rdf_version=options.rdf_version, + ), + ) + + +def decode_links(value: bytes) -> list[str] | None: + r""" + Return the IRIs of the link metadata key, or None if it is not valid UTF-8. + + >>> decode_links(b"http://example.org/a\nhttp://example.org/b") + ['http://example.org/a', 'http://example.org/b'] + >>> decode_links(b"") + [] + >>> decode_links(b"\xff") is None + True + """ + try: + text = value.decode() + except UnicodeDecodeError: + return None + return text.split("\n") if text else [] + + +class DecodedFrame: + """What one frame of a Jelly-SPARQL stream holds.""" + + def __init__(self) -> None: + #: Whether the frame starts a result set + self.starts = False + #: The variables of the result set, if the frame starts a solution sequence + self.variables: list[str] = [] + #: The boolean result, if the frame has one + self.boolean: bool | None = None + #: The rows of the frame: one value per variable, None if it is unbound + self.rows: list[tuple[Any, ...]] = [] + #: The links describing the result set, if the frame has them + self.links: list[str] | None = None + #: The trailer: None if the frame has none, else "" if the result set is + #: complete, or why it is not + self.trailer: str | None = None + + +class SparqlDecoder: + """ + Decodes the frames of a Jelly-SPARQL stream, in order. + + Args: + adapter_factory (AdapterFactory): makes the adapter that makes the terms, + given the options of the stream. + max_rows (int): the most rows a frame can have. + max_values (int): the most values (rows times variables) a frame can have. + Each frame is decoded at once, so this limits the memory it can take. + + """ + + def __init__( + self, + adapter_factory: AdapterFactory, + *, + max_rows: int = MAX_SPARQL_FRAME_ROWS, + max_values: int = MAX_SPARQL_FRAME_VALUES, + ) -> None: + self.adapter_factory = adapter_factory + self.max_rows = min(max_rows, MAX_ROWS) + self.max_values = max_values + #: The stream options, once read + self.options: jelly.SparqlResultsOptions | None = None + self.decoder: Decoder | None = None + self.columns: ColumnDecoder | None = None + #: Whether the stream is a sequence of result sets + self.punctuated = False + # The variables of the result set, as the first header declares them, and + # whether a header is in effect. Repeated stream options end its effect. + self.variables: list[str] | None = None + self.header_in_effect = False + # Whether the next frame starts a result set, whether the result set is a + # boolean result, and whether a trailer was read since the stream options + self.result_set_start = True + self.boolean_read = False + self.trailer_read = False + # How many result sets were started + self.result_sets = 0 + + def decode(self, frame: jelly.SparqlResultsFrame) -> DecodedFrame: + """Decode a frame.""" + out = DecodedFrame() + has_options = _FRAME_HAS_FIELD(frame, "options") + if has_options: + self.read_options(frame.options) + elif self.options is None: + msg = "the stream does not start with the stream options" + raise JellyConformanceError(msg) + elif self.trailer_read and not self.punctuated: + msg = ( + "a frame after the trailer must have the stream options: the trailer " + "ends the result set" + ) + raise JellyConformanceError(msg) + if self.result_set_start: + self.start_result_set() + out.starts = True + if self.boolean_read: + msg = "no frame of the result set can follow its boolean result" + raise JellyConformanceError(msg) + if _FRAME_HAS_FIELD(frame, "ask_result"): + self.read_boolean(frame, out) + else: + self.read_solutions(frame, has_options=has_options, out=out) + links = frame.metadata.get(LINK_KEY) + if links is not None: + out.links = decode_links(links) + if _FRAME_HAS_FIELD(frame, "trailer"): + out.trailer = frame.trailer.error + self.trailer_read = True + self.result_set_start = self.punctuated + else: + self.result_set_start = False + return out + + def read_options(self, options: jelly.SparqlResultsOptions) -> None: + """Read the stream options, which empty the lookups if they are repeated.""" + check_sparql_options(options) + punctuated = options.stream_type == jelly.SPARQL_STREAM_TYPE_PUNCTUATED + parser_options = sparql_parser_options(options) + current = self.options + if current is None: + decoder = Decoder(self.adapter_factory(parser_options)) + self.decoder = decoder + self.columns = decoder.column_decoder + else: + if options.stream_type != current.stream_type: + msg = "the stream type must be the same in all stream options" + raise JellyConformanceError(msg) + if punctuated and not self.result_set_start: + msg = ( + "in a PUNCTUATED stream, the stream options can only be repeated " + "in the first frame of a result set" + ) + raise JellyConformanceError(msg) + current_decoder = self.decoder + assert current_decoder is not None + current_decoder.adapter.options = parser_options + current_decoder.new_lookups() + self.header_in_effect = False + self.options = options + self.punctuated = punctuated + if not punctuated: + self.trailer_read = False + + def start_result_set(self) -> None: + """Forget the result set before, in a PUNCTUATED stream.""" + self.variables = None + self.header_in_effect = False + self.boolean_read = False + self.trailer_read = False + if self.result_sets: + # Blank node labels are scoped to a result set + columns = self.columns + assert columns is not None + columns.bnode_scope = f"r{self.result_sets}-" + self.result_sets += 1 + + def read_boolean(self, frame: jelly.SparqlResultsFrame, out: DecodedFrame) -> None: + if not out.starts: + msg = "a boolean result can only be in the first frame of a result set" + raise JellyConformanceError(msg) + if frame.row_count or frame.variables or frame.columns: + msg = "a frame with a boolean result cannot have variables, rows or columns" + raise JellyConformanceError(msg) + self.boolean_read = True + out.boolean = frame.ask_result.value + # The next result set of a PUNCTUATED stream may use the entries + columns = self.columns + assert columns is not None + columns.apply_entries(frame.names, frame.prefixes, frame.datatypes) + + def read_solutions( + self, frame: jelly.SparqlResultsFrame, *, has_options: bool, out: DecodedFrame + ) -> None: + variables = self.read_variables(frame, has_options=has_options) + if out.starts: + out.variables = variables + rows = frame.row_count + if rows > MAX_ROWS: + msg = f"a frame can have at most {MAX_ROWS} rows, not {rows}" + raise JellyConformanceError(msg) + if rows > self.max_rows: + msg = f"the frame has {rows} rows, more than the {self.max_rows} allowed" + raise JellyConformanceError(msg) + if rows * len(variables) > self.max_values: + msg = ( + f"the frame has {rows} rows of {len(variables)} variables, more than " + f"the {self.max_values} values allowed" + ) + raise JellyConformanceError(msg) + + columns = self.columns + assert columns is not None + # All lookup entries come first, so that the columns can refer to them + columns.apply_entries(frame.names, frame.prefixes, frame.datatypes) + column_count = len(frame.columns) + if not variables: + if column_count: + msg = "a result set with no variables cannot have columns" + raise JellyConformanceError(msg) + out.rows = [()] * rows + elif column_count == len(variables): + cells = [ + columns.cells(column, rows, "?" + name) + for column, name in zip(frame.columns, variables, strict=True) + ] + out.rows = list(zip(*cells, strict=True)) + elif column_count or rows: + msg = ( + f"the frame has {column_count} columns, but the result set has " + f"{len(variables)} variables" + ) + raise JellyConformanceError(msg) + + def read_variables( + self, frame: jelly.SparqlResultsFrame, *, has_options: bool + ) -> list[str]: + """Read the header of the frame, if any, and return the variables.""" + if frame.variables: + if not has_options and not self.result_set_start: + msg = ( + "the variables can only be declared in the first frame of a " + "result set, or in a frame with the stream options" + ) + raise JellyConformanceError(msg) + self.read_header(list(frame.variables)) + elif not self.header_in_effect: + # No header next to the options, or at the start of a result set, + # declares zero variables + if self.variables: + msg = ( + "the frame repeats the stream options, so it must repeat the " + "variables too" + ) + raise JellyConformanceError(msg) + self.read_header([]) + variables = self.variables + assert variables is not None + return variables + + def read_header(self, names: list[str]) -> None: + for name in names: + if not name: + msg = "variable names cannot be empty" + raise JellyConformanceError(msg) + if self.variables is None: + self.variables = names + elif names != self.variables: + msg = ( + "the repeated variables must be the same as the first ones: " + f"{names} instead of {self.variables}" + ) + raise JellyConformanceError(msg) + self.header_in_effect = True + + +def sparql_frames(inp: IO[bytes]) -> Iterator[jelly.SparqlResultsFrame]: + """ + Return the frames of a Jelly-SPARQL stream, delimited or not. + + Raises: + JellyConformanceError: if the stream is empty. + + """ + inp, delimited = detect_delimited(inp, SPARQL_FRAME_TAGS) + if not delimited: + data = inp.read() + if not data: + msg = "the stream is empty, but it must have at least one frame" + raise JellyConformanceError(msg) + return iter((parse(jelly.SparqlResultsFrame, data),)) + return _delimited_frames(inp) + + +def _delimited_frames(inp: IO[bytes]) -> Iterator[jelly.SparqlResultsFrame]: + while True: + # protobuf annotates `input_bytes` as io.BytesIO, but any IO[bytes] works. + frame = parse_length_prefixed(jelly.SparqlResultsFrame, inp) # type: ignore[arg-type] + if frame is None: + return + yield frame + + +class ResultSetReader: + """Decodes the frames of a stream, one at a time, for its result sets.""" + + def __init__( + self, frames: Iterator[jelly.SparqlResultsFrame], decoder: SparqlDecoder + ) -> None: + self.frames = frames + self.decoder = decoder + # A frame that was read, but not yet taken + self.pending: DecodedFrame | None = None + + def next(self) -> DecodedFrame | None: + """Return the next frame, or None at the end of the stream.""" + frame = self.pending + if frame is not None: + self.pending = None + return frame + message = next(self.frames, None) + if message is None: + return None + return self.decoder.decode(message) + + +class SparqlResultSet: + """ + One result set of a Jelly-SPARQL stream: a solution sequence or a boolean result. + + The rows of a solution sequence are read as they are iterated, so they can be + iterated only once. Each row is a tuple, with one value per variable, or None + if the variable is unbound. Rows not yet iterated when the next result set is + read are kept in memory. + + If the producer ended the result set with an error, iterating it raises + JellyIncompleteResultsError after the last row. + """ + + def __init__(self, reader: ResultSetReader, first: DecodedFrame) -> None: + self.reader = reader + self.first: DecodedFrame | None = first + #: The variables, in projection order. Empty for a boolean result. + self.variables: list[str] = first.variables + #: The boolean result (of an ASK query), None for a solution sequence + self.boolean: bool | None = first.boolean + #: IRIs of links that describe the result set (head.link in SPARQL JSON) + self.links: list[str] = first.links or [] + #: Why the result set is incomplete, if the producer said so + self.error = "" + #: Whether the stream ended without a trailer after the result set: the + #: result set may be cut short. Known once all rows were read. + self.truncated = False + self._rows = self._read() + # The rows kept when the next result set was read before they were + self._kept: list[tuple[Any, ...]] | None = None + + @property + def is_boolean(self) -> bool: + """Whether this is a boolean result (of an ASK query).""" + return self.boolean is not None + + def __iter__(self) -> Iterator[tuple[Any, ...]]: + kept = self._kept + if kept is not None: + self._kept = None + yield from kept + else: + yield from self._rows + if self.error: + msg = f"the result set is incomplete: {self.error}" + raise JellyIncompleteResultsError(msg, self.error) + + def _read(self) -> Iterator[tuple[Any, ...]]: + """Yield the rows, and note the trailers, until the result set ends.""" + reader = self.reader + frame = self.first + self.first = None + ended = False + while frame is not None: + if frame.trailer is not None: + ended = True + if frame.trailer: + # An error stays even if the result set goes on, in a stream + # made by concatenating streams + self.error = self.error or frame.trailer + else: + # In a FLAT stream, rows can follow a trailer if the stream options + # are repeated (concatenated streams): it no longer ends the stream + ended = False + if frame.links is not None and not self.links: + self.links = frame.links + yield from frame.rows + if ended and reader.decoder.punctuated: + return + frame = reader.next() + if frame is not None and frame.starts: + # The next result set of a PUNCTUATED stream + reader.pending = frame + return + self.truncated = not ended + + def keep(self) -> None: + """Read the rest of the result set, and keep its rows for later.""" + rows = list(self._rows) + if self._kept is None: + self._kept = rows + else: + self._kept += rows + + +def read_result_sets( + inp: IO[bytes], + adapter_factory: AdapterFactory, + *, + max_rows: int = MAX_SPARQL_FRAME_ROWS, + max_values: int = MAX_SPARQL_FRAME_VALUES, +) -> Iterator[SparqlResultSet]: + """ + Read the result sets of a Jelly-SPARQL stream. + + A stream of the FLAT type (the default) has one result set, one of the + PUNCTUATED type has any number of them. Each result set is read when the one + before was read whole: moving on to the next result set reads the rest of the + rows, and keeps them in memory. + + Args: + inp (IO[bytes]): the stream, delimited or not. + adapter_factory (AdapterFactory): makes the adapter that makes the terms, + given the options of the stream. + max_rows (int): the most rows a frame can have. + max_values (int): the most values (rows times variables) a frame can have. + + Yields: + SparqlResultSet: the result sets, in order. + + """ + decoder = SparqlDecoder(adapter_factory, max_rows=max_rows, max_values=max_values) + reader = ResultSetReader(sparql_frames(inp), decoder) + while (first := reader.next()) is not None: + result_set = SparqlResultSet(reader, first) + yield result_set + result_set.keep() + + +def read_result_set( + inp: IO[bytes], + adapter_factory: AdapterFactory, + *, + max_rows: int = MAX_SPARQL_FRAME_ROWS, + max_values: int = MAX_SPARQL_FRAME_VALUES, +) -> SparqlResultSet: + """ + Read the result set of a Jelly-SPARQL stream of the FLAT type. + + The first frame is read right away, the rows as they are iterated. To read a + stream of the PUNCTUATED type, use read_result_sets(). + + Raises: + JellyConformanceError: if the stream is of the PUNCTUATED type. + + """ + result_sets = read_result_sets( + inp, adapter_factory, max_rows=max_rows, max_values=max_values + ) + result_set = next(result_sets) + if result_set.reader.decoder.punctuated: + msg = ( + "the stream is a sequence of result sets (PUNCTUATED). Read all of them " + "with parse_jelly_sparql_results(), or read_result_sets()" + ) + raise JellyConformanceError(msg) + return result_set diff --git a/pyjelly/serialize/encode.py b/pyjelly/serialize/encode.py index e958c4c2..f0dea44a 100644 --- a/pyjelly/serialize/encode.py +++ b/pyjelly/serialize/encode.py @@ -157,7 +157,7 @@ def new_epoch(self) -> None: self.prefixes.lookup.new_epoch() self.datatypes.lookup.new_epoch() - def new_frame(self, iris: int) -> None: + def new_frame(self, iris: int, literals: int = 1) -> None: """ Start encoding a new frame in the column layout. @@ -167,12 +167,13 @@ def new_frame(self, iris: int) -> None: Args: iris (int): the most IRIs that one statement can have. + literals (int): the most literals that one statement can have. In a + statement, only the object can be a literal. """ self.names.lookup.new_epoch(_budget(self.names, iris)) self.prefixes.lookup.new_epoch(_budget(self.prefixes, iris)) - # Only the object can be a literal - self.datatypes.lookup.new_epoch(_budget(self.datatypes, 1)) + self.datatypes.lookup.new_epoch(_budget(self.datatypes, literals)) self.triple_prefix_index = -1 self.triple_name_index = 0 diff --git a/pyjelly/serialize/ioutils.py b/pyjelly/serialize/ioutils.py index 1180df6e..7af210e8 100644 --- a/pyjelly/serialize/ioutils.py +++ b/pyjelly/serialize/ioutils.py @@ -1,14 +1,34 @@ from typing import IO -from google.protobuf.proto import serialize_length_prefixed +from google.protobuf.message import Message -from pyjelly import jelly +_VARINT_PAYLOAD = 0x7F +_VARINT_CONTINUATION = 0x80 -def write_delimited(frame: jelly.RdfStreamFrame, output_stream: IO[bytes]) -> None: - # protobuf annotates `output` as io.BytesIO, but any IO[bytes] works at runtime. - serialize_length_prefixed(frame, output_stream) # type: ignore[arg-type] +def varint(value: int) -> bytes: + r""" + Return a non-negative integer as a protobuf varint. + >>> varint(300) + b'\xac\x02' + """ + out = bytearray() + while value > _VARINT_PAYLOAD: + out.append((value & _VARINT_PAYLOAD) | _VARINT_CONTINUATION) + value >>= 7 + out.append(value) + return bytes(out) -def write_single(frame: jelly.RdfStreamFrame, output_stream: IO[bytes]) -> None: + +def write_delimited(frame: Message, output_stream: IO[bytes]) -> None: + """Write a frame, after its size (the delimited variant of Jelly).""" + # Not protobuf's serialize_length_prefixed(), which needs write() to return the + # number of bytes written. Not every stream does, e.g., rdflib's for results. + data = frame.SerializeToString() + output_stream.write(varint(len(data))) + output_stream.write(data) + + +def write_single(frame: Message, output_stream: IO[bytes]) -> None: output_stream.write(frame.SerializeToString(deterministic=True)) diff --git a/pyjelly/serialize/lookup.py b/pyjelly/serialize/lookup.py index 2853025a..9b1e27c1 100644 --- a/pyjelly/serialize/lookup.py +++ b/pyjelly/serialize/lookup.py @@ -139,8 +139,8 @@ def victim(self) -> int: return index index = 1 if index == size else index + 1 msg = ( - f"a lookup of size {size} is too small to encode a single statement: " - "the statement uses every one of its entries" + f"a lookup of size {size} is too small to encode a single statement or " + "row of results: it uses every one of its entries" ) raise JellyConformanceError(msg) diff --git a/pyjelly/serialize/sparql.py b/pyjelly/serialize/sparql.py new file mode 100644 index 00000000..010117ba --- /dev/null +++ b/pyjelly/serialize/sparql.py @@ -0,0 +1,445 @@ +""" +Writing of Jelly-SPARQL: SPARQL query results in frames, one column per variable. + +The columns are those of Jelly-RDF 1.2 (see pyjelly.serialize.columns), in which +any cell may be unbound. +""" + +from __future__ import annotations + +from collections.abc import Iterable, Iterator, Sequence +from contextlib import suppress +from typing import IO, Final + +from pyjelly import jelly +from pyjelly.errors import JellyConformanceError +from pyjelly.options import SPARQL_VERSION_1_0, SparqlOptions +from pyjelly.serialize.columns import MAX_ROWS, Column +from pyjelly.serialize.encode import EncodedTerm, TermEncoder +from pyjelly.serialize.ioutils import write_delimited, write_single + +#: The metadata key of the links of a result set (head.link in SPARQL JSON results) +LINK_KEY: Final = "link" + +# A column of results can have terms of any type +_ANY: Final = 0b1111 + + +def links_value(links: Sequence[str]) -> bytes: + r""" + Return the value of the link metadata key: the IRIs, separated by LF. + + >>> links_value(["http://example.org/a", "http://example.org/b"]) + b'http://example.org/a\nhttp://example.org/b' + """ + for link in links: + if "\n" in link: + msg = f"a link cannot contain the LF character, as no IRI can: {link!r}" + raise JellyConformanceError(msg) + return "\n".join(links).encode() + + +def variable_names(variables: Iterable[str]) -> list[str]: + """ + Return the names of variables as written: without a leading "?" or "$". + + >>> variable_names(["?s", "$p", "o"]) + ['s', 'p', 'o'] + """ + names = [] + for variable in variables: + name = str(variable) + if name[:1] in ("?", "$"): + name = name[1:] + if not name: + msg = "variable names cannot be empty" + raise JellyConformanceError(msg) + names.append(name) + return names + + +def _failed() -> JellyConformanceError: + msg = ( + "a row failed to encode, so the stream can only be ended, with an error: " + "end(error)" + ) + return JellyConformanceError(msg) + + +class VariableColumn(Column): + """The cells of one variable in the frame being built.""" + + def __init__(self) -> None: + super().__init__(_ANY, "variable") + # The IRI sequence of the column's triple terms: the indices of the last IRI, + # while another column's triple terms are encoded (see TermEncoder) + self.triple_prefix_index = -1 + self.triple_name_index = 0 + # The most IRIs a triple term of the column had, so far in the result set + self.max_triple_iris = 1 + + +class SparqlStream: + """ + Writes SPARQL query results as the frames of a Jelly-SPARQL stream. + + A result set is a solution sequence (the result of a SELECT query) or a boolean + result (ASK). Start a solution sequence with start(), add its rows with row(), + and end it with end(). row() returns a frame when one is full, end() returns the + last frame of the result set. A boolean result is one frame, made by ask(). + + A stream has one result set, or with SparqlOptions(punctuated=True), any number + of them. + + Args: + encoder (TermEncoder): encoder of the terms, with the lookup sizes of the + options. + options (SparqlOptions | None): options of the stream. + + """ + + def __init__( + self, encoder: TermEncoder, options: SparqlOptions | None = None + ) -> None: + if options is None: + options = SparqlOptions() + if encoder.lookup_preset != options.lookup_preset: + msg = "the encoder must have the lookup sizes of the options" + raise ValueError(msg) + self.encoder = encoder + self.options = options + encoder.column_layout = True + encoder.rdf_version = options.rdf_version + self.encoded = EncodedTerm() + #: The variables of the result set being written, None between result sets + self.variables: list[str] | None = None + self.columns: list[VariableColumn] = [] + self.row_count = 0 + self.rows_per_frame = 1 + self.links: list[str] = [] + # Whether the stream options and the header are yet to be written + self.options_pending = True + self.header_pending = False + # Whether nothing more can be written: the result set of a FLAT stream ended, + # or a row failed to encode + self.ended = False + self.failed = False + # The column whose IRI sequence of triple terms the encoder has, if any + self.triple_owner: VariableColumn | None = None + + def start(self, variables: Iterable[str], links: Sequence[str] = ()) -> None: + """ + Start a solution sequence. + + Args: + variables (Iterable[str]): names of the variables, in projection order. A + leading "?" or "$" is left out. + links (Sequence[str]): IRIs of links describing the result set, like + head.link of SPARQL JSON results. + + """ + self._check_can_start() + names = variable_names(variables) + self.variables = names + self.columns = [VariableColumn() for _ in names] + self.rows_per_frame = self.options.rows_per_frame(len(names)) + self.header_pending = True + self.links = list(links) + self.new_frame() + + def _check_can_start(self) -> None: + if self.failed: + raise _failed() + if self.ended: + msg = ( + "the stream has ended. A FLAT stream has one result set, use " + "SparqlOptions(punctuated=True) to write more" + ) + raise JellyConformanceError(msg) + if self.variables is not None: + msg = "the result set has not ended yet, end it with end()" + raise JellyConformanceError(msg) + + def new_frame(self) -> None: + columns = self.columns + iris = len(columns) + for column in columns: + # A cell of a triple term can need that many IRIs + iris += column.max_triple_iris - 1 + column.triple_prefix_index = -1 + column.triple_name_index = 0 + self.encoder.new_frame(iris, len(columns)) + self.triple_owner = None + self.row_count = 0 + + def row(self, values: Sequence[object | None]) -> jelly.SparqlResultsFrame | None: + """ + Add a row (solution) to the result set. + + Args: + values (Sequence[object | None]): the values of the variables, None for + an unbound variable. + + Returns: + jelly.SparqlResultsFrame | None: a frame, if one is full. + + """ + columns = self.columns + if self.variables is None or self.failed: + if self.failed: + raise _failed() + msg = "start a result set with start() before adding rows" + raise JellyConformanceError(msg) + count = len(values) + if count != len(columns): + msg = f"expected {len(columns)} values in the row, got {count}" + raise JellyConformanceError(msg) + frame = None + if self.row_count and ( + self.row_count >= MAX_ROWS or not self.encoder.has_room() + ): + frame = self.full_frame() + try: + for i in range(count): + self.add(columns[i], values[i]) + except BaseException: + # The frame is half-written: it can't be completed + self.failed = True + raise + self.row_count += 1 + if self.row_count >= self.rows_per_frame and self.options.delimited: + return self.end_frame() + return frame + + def rows( + self, rows: Iterable[Sequence[object | None]] + ) -> Iterator[jelly.SparqlResultsFrame]: + """ + Add rows to the result set, see row(). + + Yields: + jelly.SparqlResultsFrame: full frames. + + """ + for values in rows: + frame = self.row(values) + if frame is not None: + yield frame + + def full_frame(self) -> jelly.SparqlResultsFrame: + """End the frame, which has no room for another row.""" + if not self.options.delimited: + msg = ( + "the results do not fit in one frame: the lookups can't hold all of " + "their terms. A non-delimited stream can only have one frame. Write " + "a delimited stream, or use larger lookups." + ) + raise JellyConformanceError(msg) + return self.end_frame() + + def add(self, column: VariableColumn, term: object | None) -> None: + """Add a value to its column, None for an unbound one.""" + if term is None: + if not column.run_unbound: + column.end_run() + column.run_unbound = True + column.run_length += 1 + return + if ( + column.run_length + and not column.run_unbound + and self.encoder.same_term(column.run_term, term) + ): + column.run_length += 1 + return + column.end_run() + if self.triple_owner is not None and self.triple_owner is not column: + self._switch_triple_iris(column) + encoded = self.encoded + self.encoder.encode_spo(term, encoded) + column.add_value(encoded) + column.run_term = term + column.run_length = 1 + if encoded.triple_term is not None: + self._triple_term_added(column) + + def _switch_triple_iris(self, column: VariableColumn) -> None: + """Give the encoder the IRI sequence of a column's triple terms.""" + owner = self.triple_owner + assert owner is not None + encoder = self.encoder + owner.triple_prefix_index = encoder.triple_prefix_index + owner.triple_name_index = encoder.triple_name_index + encoder.triple_prefix_index = column.triple_prefix_index + encoder.triple_name_index = column.triple_name_index + self.triple_owner = column + + def _triple_term_added(self, column: VariableColumn) -> None: + self.encoded.triple_term = None + if self.triple_owner is None: + self.triple_owner = column + iris = self.encoder.triple_iris + if iris > column.max_triple_iris: + # From now on, each row may need this many IRIs for the column + self.encoder.reserve_iris(iris - column.max_triple_iris) + column.max_triple_iris = iris + + def end_frame(self) -> jelly.SparqlResultsFrame: + """End the current frame of the result set, and return it.""" + if self.variables is None or self.failed: + if self.failed: + raise _failed() + msg = "there is no result set to end a frame of" + raise JellyConformanceError(msg) + frame = jelly.SparqlResultsFrame() + self.write_frame(frame) + return frame + + def write_frame(self, frame: jelly.SparqlResultsFrame) -> None: + """Write the frame's options, header, lookup entries and rows.""" + self.write_start(frame) + encoder = self.encoder + row_count = self.row_count + frame.row_count = row_count + encoder.names.packed.write(frame.names) + encoder.prefixes.packed.write(frame.prefixes) + encoder.datatypes.packed.write(frame.datatypes) + # A frame with no rows needs no columns + if row_count: + for column in self.columns: + column.write(frame.columns.add()) + self.new_frame() + + def write_start(self, frame: jelly.SparqlResultsFrame) -> None: + """Write the stream options and the header, if the frame needs them.""" + if self.options_pending: + frame.options.CopyFrom(self.stream_options()) + self.options_pending = False + if self.header_pending: + assert self.variables is not None + frame.variables.extend(self.variables) + self.header_pending = False + if self.links: + frame.metadata[LINK_KEY] = links_value(self.links) + + def end(self, error: str = "") -> jelly.SparqlResultsFrame: + """ + End the result set, and return its last frame. + + Args: + error (str): why the result set is incomplete, if it is. Readers then + know that some results are missing. + + Returns: + jelly.SparqlResultsFrame: the last frame, with the trailer. + + """ + frame = jelly.SparqlResultsFrame() + if self.failed: + if not error or self.variables is None: + raise _failed() + # The rows of the frame are dropped, with the lookup entries they added, + # so nothing can follow + self.write_start(frame) + self.ended = True + elif self.variables is None: + msg = "there is no result set to end, start one with start()" + raise JellyConformanceError(msg) + else: + self.write_frame(frame) + if not self.options.punctuated: + self.ended = True + frame.trailer.SetInParent() + frame.trailer.error = error + self.variables = None + return frame + + def ask(self, value: bool, links: Sequence[str] = ()) -> jelly.SparqlResultsFrame: # noqa: FBT001 + """ + Write a boolean result (of an ASK query), and return its frame. + + Args: + value (bool): the result. + links (Sequence[str]): IRIs of links describing the result. + + Returns: + jelly.SparqlResultsFrame: the only frame of the result set. + + """ + self._check_can_start() + frame = jelly.SparqlResultsFrame() + self.write_start(frame) + frame.ask_result.value = value + frame.trailer.SetInParent() + if links: + frame.metadata[LINK_KEY] = links_value(links) + if not self.options.punctuated: + self.ended = True + return frame + + def stream_options(self) -> jelly.SparqlResultsOptions: + options = self.options + lookup_preset = options.lookup_preset + return jelly.SparqlResultsOptions( + stream_name=options.stream_name, + stream_type=( + jelly.SPARQL_STREAM_TYPE_PUNCTUATED + if options.punctuated + else jelly.SPARQL_STREAM_TYPE_FLAT + ), + rdf_version=options.rdf_version, + max_name_table_size=lookup_preset.max_names, + max_prefix_table_size=lookup_preset.max_prefixes, + max_datatype_table_size=lookup_preset.max_datatypes, + version=SPARQL_VERSION_1_0, + ) + + +class FrameWriter: + """Writes the frames of a stream to a file: delimited, or as its only frame.""" + + def __init__(self, out: IO[bytes], *, delimited: bool) -> None: + self.out = out + self.delimited = delimited + self.written = False + + def write(self, frame: jelly.SparqlResultsFrame) -> None: + if self.delimited: + write_delimited(frame, self.out) + return + if self.written: + msg = ( + "the results do not fit in one frame. A non-delimited stream can " + "only have one frame. Write a delimited stream, or use larger lookups." + ) + raise JellyConformanceError(msg) + write_single(frame, self.out) + self.written = True + + +def write_solutions( + stream: SparqlStream, + variables: Iterable[str], + rows: Iterable[Sequence[object | None]], + writer: FrameWriter, + links: Sequence[str] = (), +) -> None: + """ + Write a solution sequence (the result of a SELECT query) as a result set. + + If getting the rows fails midway (e.g., the query fails), the result set ends + with the error, so that readers know it is incomplete, and the error is + raised. + """ + stream.start(variables, links) + try: + for values in rows: + frame = stream.row(values) + if frame is not None: + writer.write(frame) + except Exception as error: + # If the frame that says so can still be written + with suppress(Exception): + writer.write(stream.end(str(error) or type(error).__name__)) + raise + writer.write(stream.end()) diff --git a/pyproject.toml b/pyproject.toml index d63a7353..a3dbeb61 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,7 +37,9 @@ include = [ "pyjelly/parse/lookup.py", "pyjelly/parse/ioutils.py", "pyjelly/parse/columns.py", + "pyjelly/parse/sparql.py", "pyjelly/serialize/columns.py", + "pyjelly/serialize/sparql.py", "pyjelly/serialize/streams.py", "pyjelly/serialize/flows.py", "pyjelly/parse/decode.py", @@ -74,7 +76,7 @@ archs = ["x86_64", "aarch64"] name = "pyjelly" version = "0.0.0" readme = "README.md" -description = "Jelly-RDF implementation for Python" +description = "Jelly-RDF and Jelly-SPARQL implementation for Python" authors = [ { name = "NeverBlink et al.", email = "contact@neverblink.eu" } ] @@ -124,6 +126,14 @@ jelly = "pyjelly.integrations.rdflib.parse:RDFLibJellyParser" jelly = "pyjelly.integrations.rdflib.serialize:RDFLibJellySerializer" "application/x-jelly-rdf" = "pyjelly.integrations.rdflib.serialize:RDFLibJellySerializer" +[project.entry-points."rdf.plugins.resultparser"] +jelly = "pyjelly.integrations.rdflib.sparql:RDFLibJellySparqlParser" +"application/x-jelly-sparql" = "pyjelly.integrations.rdflib.sparql:RDFLibJellySparqlParser" + +[project.entry-points."rdf.plugins.resultserializer"] +jelly = "pyjelly.integrations.rdflib.sparql:RDFLibJellySparqlSerializer" +"application/x-jelly-sparql" = "pyjelly.integrations.rdflib.sparql:RDFLibJellySparqlSerializer" + [dependency-groups] dev = ["mypy>=2.4", "pre-commit>=4.2.0", "ruff>=0.11.8"] # grpc doesn't support PyPy at the time of writing, see grpc/grpc#4221 diff --git a/tests/integration_tests/test_generic/test_sparql.py b/tests/integration_tests/test_generic/test_sparql.py new file mode 100644 index 00000000..bd247421 --- /dev/null +++ b/tests/integration_tests/test_generic/test_sparql.py @@ -0,0 +1,209 @@ +"""Jelly-SPARQL with the generic API.""" + +from __future__ import annotations + +import io +from collections.abc import Iterator + +import pytest + +from pyjelly import jelly +from pyjelly.errors import JellyConformanceError, JellyIncompleteResultsError +from pyjelly.integrations.generic.generic_sink import ( + IRI, + BlankNode, + Literal, + Node, + Triple, +) +from pyjelly.integrations.generic.sparql import ( + parse_jelly_sparql, + parse_jelly_sparql_results, + serialize_jelly_sparql, + serialize_jelly_sparql_ask, + sparql_stream, +) +from pyjelly.options import LookupPreset, SparqlOptions +from pyjelly.parse.sparql import sparql_frames +from pyjelly.serialize.sparql import FrameWriter, write_solutions + +EX = "http://example.org/" +XSD_INTEGER = "http://www.w3.org/2001/XMLSchema#integer" + +A = IRI(EX + "a") +P = IRI(EX + "p") + +ROWS: list[tuple[Node | None, ...]] = [ + (A, Literal("1", datatype=XSD_INTEGER), None), + (A, Literal("hello", "en"), Literal("مرحبا", "ar", direction="rtl")), + (BlankNode("b"), None, Triple(A, P, Literal("x"))), + (None, None, None), + ( + BlankNode("b"), + Literal("hello", "en"), + Triple(BlankNode("c"), P, Triple(A, P, A)), + ), +] + + +def _write( + rows: list[tuple[Node | None, ...]], options: SparqlOptions | None = None +) -> io.BytesIO: + out = io.BytesIO() + serialize_jelly_sparql(["s", "o", "t"], rows, out, options) + out.seek(0) + return out + + +def test_round_trip() -> None: + results = parse_jelly_sparql(_write(ROWS)) + assert results.variables == ["s", "o", "t"] + assert not results.is_boolean + assert list(results) == ROWS + assert not results.truncated + assert not results.error + + +@pytest.mark.parametrize("frame_size", [1, 3, 7, 1000]) +def test_frames(frame_size: int) -> None: + out = _write(ROWS * 20, SparqlOptions(frame_size=frame_size)) + frames = list(sparql_frames(out)) + rows_per_frame = max(1, frame_size // 3) + assert all(frame.row_count <= rows_per_frame for frame in frames) + out.seek(0) + assert list(parse_jelly_sparql(out)) == ROWS * 20 + + +def test_rdf_version() -> None: + options = SparqlOptions(rdf_version=jelly.RDF_VERSION_1_2_BASIC) + with pytest.raises(JellyConformanceError, match=r"RDF 1\.2 Basic"): + _write(ROWS, options) + options = SparqlOptions(rdf_version=jelly.RDF_VERSION_1_1) + with pytest.raises(JellyConformanceError, match=r"RDF 1\.1"): + _write(ROWS[1:2], options) + + +def test_triple_terms_in_several_columns() -> None: + # Each column has an IRI sequence of its own in its triple terms + t1 = Triple(IRI(EX + "s1"), P, IRI(EX + "o1")) + t2 = Triple(IRI(EX + "s2"), IRI(EX + "q"), Triple(A, P, IRI(EX + "o2"))) + rows: list[tuple[Node | None, ...]] = [ + (t1, t2, IRI(EX + "z")), + (t2, t1, t1), + (t1, None, t2), + ] * 50 + options = SparqlOptions( + lookup_preset=LookupPreset(max_names=128, max_prefixes=4, max_datatypes=4), + frame_size=9, + ) + assert list(parse_jelly_sparql(_write(rows, options))) == rows + + +def test_lookups_full() -> None: + rows: list[tuple[Node | None, ...]] = [ + (IRI(f"{EX}{i}"), Literal(str(i), datatype=f"{EX}type{i % 40}"), None) + for i in range(2000) + ] + options = SparqlOptions( + lookup_preset=LookupPreset(max_names=128, max_prefixes=2, max_datatypes=16) + ) + out = _write(rows, options) + frames = list(sparql_frames(out)) + assert len(frames) > 2000 // 128 + out.seek(0) + assert list(parse_jelly_sparql(out)) == rows + + +def test_ask() -> None: + out = io.BytesIO() + serialize_jelly_sparql_ask(True, out, links=[EX + "doc"]) # noqa: FBT003 + out.seek(0) + results = parse_jelly_sparql(out) + assert results.is_boolean + assert results.boolean is True + assert results.variables == [] + assert results.links == [EX + "doc"] + assert list(results) == [] + + +def test_non_delimited() -> None: + out = _write(ROWS * 10, SparqlOptions(delimited=False)) + (frame,) = sparql_frames(out) + assert frame.row_count == len(ROWS) * 10 + out.seek(0) + assert list(parse_jelly_sparql(out)) == ROWS * 10 + + +def test_non_delimited_too_large() -> None: + rows: list[tuple[Node | None, ...]] = [ + (IRI(f"{EX}{i}"), None, None) for i in range(200) + ] + options = SparqlOptions( + lookup_preset=LookupPreset(max_names=128, max_prefixes=0, max_datatypes=0), + delimited=False, + ) + with pytest.raises(JellyConformanceError, match="one frame"): + _write(rows, options) + + +def _failing_rows() -> Iterator[tuple[Node | None, ...]]: + yield from ROWS[:2] + msg = "timeout" + raise TimeoutError(msg) + + +def test_incomplete() -> None: + out = io.BytesIO() + with pytest.raises(TimeoutError): + serialize_jelly_sparql(["s", "o", "t"], _failing_rows(), out) + out.seek(0) + results = parse_jelly_sparql(out) + rows: list[tuple[Node | None, ...]] = [] + with pytest.raises(JellyIncompleteResultsError, match="timeout"): + rows.extend(results) + # The rows before the error are read + assert rows == ROWS[:2] + assert results.error == "timeout" + + +def test_sequence_of_result_sets() -> None: + options = SparqlOptions(punctuated=True) + stream = sparql_stream(options) + out = io.BytesIO() + writer = FrameWriter(out, delimited=True) + write_solutions(stream, ["s", "o", "t"], ROWS, writer) + writer.write(stream.ask(False)) # noqa: FBT003 + write_solutions(stream, ["x"], [(BlankNode("b"),)], writer, links=[EX + "l"]) + out.seek(0) + first, second, third = parse_jelly_sparql_results(out) + assert list(first) == ROWS + assert second.boolean is False + assert third.variables == ["x"] + assert third.links == [EX + "l"] + (row,) = list(third) + # Blank node labels are scoped to a result set + assert row[0] != ROWS[2][0] + + +def test_rows_are_kept_for_later() -> None: + stream = sparql_stream(SparqlOptions(punctuated=True, frame_size=3)) + out = io.BytesIO() + writer = FrameWriter(out, delimited=True) + write_solutions(stream, ["s", "o", "t"], ROWS, writer) + write_solutions(stream, ["s", "o", "t"], ROWS[:1], writer) + out.seek(0) + first, second = parse_jelly_sparql_results(out) + assert list(second) == ROWS[:1] + assert list(first) == ROWS + + +def test_flat_reader_rejects_punctuated() -> None: + out = io.BytesIO() + serialize_jelly_sparql_ask( + True, # noqa: FBT003 + out, + SparqlOptions(punctuated=True), + ) + out.seek(0) + with pytest.raises(JellyConformanceError, match="parse_jelly_sparql_results"): + parse_jelly_sparql(out) diff --git a/tests/integration_tests/test_rdflib/test_sparql.py b/tests/integration_tests/test_rdflib/test_sparql.py new file mode 100644 index 00000000..0ac05bd9 --- /dev/null +++ b/tests/integration_tests/test_rdflib/test_sparql.py @@ -0,0 +1,309 @@ +"""Jelly-SPARQL with rdflib: query results written and read through the plugins.""" + +from __future__ import annotations + +import io +import threading +from collections.abc import Iterator +from http.server import BaseHTTPRequestHandler, HTTPServer +from pathlib import Path +from typing import Any + +import pytest +from rdflib import BNode, Graph, Literal, URIRef, Variable +from rdflib.plugins.stores.sparqlstore import SPARQLStore +from rdflib.query import Result, ResultRow + +from pyjelly import jelly +from pyjelly.errors import JellyConformanceError, JellyIncompleteResultsError +from pyjelly.integrations.rdflib.sparql import ( + JellySparqlResult, + parse_jelly_sparql, + parse_jelly_sparql_results, + serialize_jelly_sparql, + serialize_jelly_sparql_results, +) +from pyjelly.options import LookupPreset, SparqlOptions +from pyjelly.parse.sparql import sparql_frames + +EX = "http://example.org/" + +DATA = f""" +@prefix ex: <{EX}> . +ex:a ex:name "Alice" ; ex:age 42 ; ex:knows ex:b, _:c . +ex:b ex:name "Bob"@en . +_:c ex:name "Carol" . +""" + +SELECT = """ +SELECT ?person ?name ?age WHERE { + ?person ?name . + OPTIONAL { ?person ?age } +} +ORDER BY ?name +""" + + +@pytest.fixture +def graph() -> Graph: + return Graph().parse(data=DATA, format="turtle") + + +def _round_trip(result: Result, **kwargs: Any) -> JellySparqlResult: + data = result.serialize(format="jelly", **kwargs) + assert data is not None + parsed = Result.parse(io.BytesIO(data), format="jelly") + assert isinstance(parsed, JellySparqlResult) + return parsed + + +def test_select_round_trip(graph: Graph) -> None: + result = graph.query(SELECT) + parsed = _round_trip(result) + assert parsed.type == "SELECT" + assert parsed.vars == [Variable("person"), Variable("name"), Variable("age")] + assert parsed == result + assert not parsed.truncated + rows = {row.name: row for row in parsed if isinstance(row, ResultRow)} + assert rows[Literal("Alice")].age == Literal(42) + assert rows[Literal("Bob", lang="en")].person == URIRef(EX + "b") + assert isinstance(rows[Literal("Carol")].person, BNode) + # Unbound variables are left out of the bindings, as in rdflib + assert sum(Variable("age") in binding for binding in parsed.bindings) == 1 + + +@pytest.mark.parametrize("query", ["ASK { ?s ?p 42 }", "ASK { ?s ?p 43 }"]) +def test_ask_round_trip(graph: Graph, query: str) -> None: + result = graph.query(query) + parsed = _round_trip(result) + assert parsed.type == "ASK" + assert parsed.askAnswer is result.askAnswer + assert bool(parsed) is bool(result) + + +def test_media_type(graph: Graph) -> None: + result = graph.query(SELECT) + data = result.serialize(format="application/x-jelly-sparql") + assert data is not None + parsed = Result.parse( + io.BytesIO(data), content_type="application/x-jelly-sparql; charset=utf-8" + ) + assert parsed == result + + +def test_files(graph: Graph, tmp_path: Path) -> None: + result = graph.query(SELECT) + path = tmp_path / "results.jellys" + result.serialize(str(path), format="jelly") + # Paths work too, but rdflib's types say only streams do + assert Result.parse(str(path), format="jelly") == result # type: ignore[arg-type] + assert Result.parse(path, format="jelly") == result # type: ignore[arg-type] + with path.open("rb") as file: + assert parse_jelly_sparql(file) == result + + +def test_empty_and_zero_variable_results(graph: Graph) -> None: + empty = graph.query("SELECT ?s WHERE { ?s ?p 'nothing' }") + parsed = _round_trip(empty) + assert parsed.vars == [Variable("s")] + assert len(parsed) == 0 + # One solution that binds no variables + one = graph.query("SELECT * WHERE {}") + parsed = _round_trip(one) + assert parsed.vars == [] + assert parsed.bindings == [{}] + + +def test_options(graph: Graph) -> None: + options = SparqlOptions( + lookup_preset=LookupPreset(max_names=128, max_prefixes=0, max_datatypes=8), + frame_size=2, + stream_name="people", + ) + result = graph.query(SELECT) + data = result.serialize(format="jelly", options=options) + assert data is not None + frames = list(sparql_frames(io.BytesIO(data))) + # 3 rows of 3 variables in frames of at most 2 values: one row per frame + assert [frame.row_count for frame in frames] == [1, 1, 1, 0] + assert frames[0].options.stream_name == "people" + assert frames[0].options.max_prefix_table_size == 0 + assert Result.parse(io.BytesIO(data), format="jelly") == result + + +def test_non_delimited(graph: Graph) -> None: + result = graph.query(SELECT) + data = result.serialize(format="jelly", options=SparqlOptions(delimited=False)) + assert data is not None + (frame,) = sparql_frames(io.BytesIO(data)) + assert frame.row_count == len(result) + assert frame.HasField("trailer") + assert Result.parse(io.BytesIO(data), format="jelly") == result + + +def test_links() -> None: + result = JellySparqlResult("SELECT") + result.vars = [Variable("x")] + result.bindings = [{Variable("x"): URIRef(EX + "a")}] + result.links = [EX + "about"] + parsed = _round_trip(result) + assert parsed.links == [EX + "about"] + + +def _failing_bindings() -> Iterator[dict[Variable, Any]]: + yield {Variable("x"): URIRef(EX + "a")} + msg = "the endpoint went away" + raise RuntimeError(msg) + + +def test_error_midway_ends_the_stream_with_it() -> None: + result = Result("SELECT") + result.vars = [Variable("x")] + result.bindings = _failing_bindings() + out = io.BytesIO() + with pytest.raises(RuntimeError, match="went away"): + serialize_jelly_sparql(result, out) + out.seek(0) + with pytest.raises(JellyIncompleteResultsError, match="went away") as info: + Result.parse(out, format="jelly") + assert info.value.error == "the endpoint went away" + partial = info.value.result + assert isinstance(partial, JellySparqlResult) + assert partial.bindings == [{Variable("x"): URIRef(EX + "a")}] + + +def test_truncated(graph: Graph) -> None: + result = graph.query(SELECT) + data = result.serialize(format="jelly", options=SparqlOptions(frame_size=3)) + assert data is not None + # The stream without its last frame, the one with the trailer + frames = list(sparql_frames(io.BytesIO(data))) + cut = data[: -(frames[-1].ByteSize() + 1)] + parsed = Result.parse(io.BytesIO(cut), format="jelly") + assert parsed.truncated + assert len(parsed) == len(result) + + +def test_sequence_of_results(graph: Graph) -> None: + results = [graph.query(SELECT), graph.query("ASK { ?s ?p ?o }")] + out = io.BytesIO() + serialize_jelly_sparql_results(results, out) + out.seek(0) + parsed = list(parse_jelly_sparql_results(out)) + assert parsed == results + out.seek(0) + with pytest.raises(JellyConformanceError, match="PUNCTUATED"): + parse_jelly_sparql(out) + + +def test_sequence_needs_punctuated_options(graph: Graph) -> None: + with pytest.raises(JellyConformanceError, match="punctuated"): + serialize_jelly_sparql_results( + [graph.query(SELECT)], io.BytesIO(), SparqlOptions() + ) + + +def test_graph_results_are_not_jelly_sparql(graph: Graph) -> None: + result = graph.query("CONSTRUCT WHERE { ?s ?p ?o }") + with pytest.raises(JellyConformanceError, match="CONSTRUCT"): + serialize_jelly_sparql(result, io.BytesIO()) + + +def test_rdf_12_terms_are_not_supported() -> None: + from pyjelly.integrations.generic.generic_sink import IRI, Triple # noqa: PLC0415 + from pyjelly.integrations.generic.sparql import ( # noqa: PLC0415 + serialize_jelly_sparql as generic_serialize, + ) + + out = io.BytesIO() + iri = IRI(EX + "a") + generic_serialize(["t"], [(Triple(iri, iri, iri),)], out) + out.seek(0) + with pytest.raises(NotImplementedError, match="triple terms"): + Result.parse(out, format="jelly") + + +class _Endpoint(BaseHTTPRequestHandler): + """A SPARQL endpoint that answers every query with the same Jelly-SPARQL.""" + + body = b"" + accept = "" + + def do_GET(self) -> None: + type(self).accept = self.headers["Accept"] + self.send_response(200) + self.send_header("Content-Type", "application/x-jelly-sparql") + self.end_headers() + self.wfile.write(self.body) + + def log_message(self, *args: object) -> None: + pass + + +def test_sparql_store(graph: Graph) -> None: + result = graph.query(SELECT) + body = result.serialize(format="jelly") + assert body is not None + _Endpoint.body = body + server = HTTPServer(("127.0.0.1", 0), _Endpoint) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + store = SPARQLStore( + f"http://127.0.0.1:{server.server_port}/sparql", + returnFormat="application/x-jelly-sparql", + ) + remote = Graph(store=store).query(SELECT) + finally: + server.shutdown() + assert _Endpoint.accept == "application/x-jelly-sparql" + assert remote == result + + +def test_validate_iris_is_passed_on(caplog: pytest.LogCaptureFixture) -> None: + result = Result("SELECT") + result.vars = [Variable("x")] + result.bindings = [{Variable("x"): URIRef(EX + "a b")}] + data = result.serialize(format="jelly") + assert data is not None + caplog.clear() + Result.parse(io.BytesIO(data), format="jelly") + assert not caplog.records + Result.parse(io.BytesIO(data), format="jelly", validate_iris=True) + assert "does not look like a valid URI" in caplog.text + + +def test_frames_of_many_terms() -> None: + # More distinct IRIs than the name lookup has: frames end early + options = SparqlOptions( + lookup_preset=LookupPreset(max_names=128, max_prefixes=8, max_datatypes=8) + ) + result = Result("SELECT") + result.vars = [Variable("s"), Variable("o")] + result.bindings = [ + {Variable("s"): URIRef(f"{EX}s{i}"), Variable("o"): Literal(i)} + for i in range(1000) + ] + out = io.BytesIO() + serialize_jelly_sparql(result, out, options) + out.seek(0) + frames = list(sparql_frames(out)) + max_names = options.lookup_preset.max_names + assert len(frames) > len(result) // max_names + assert all(frame.row_count <= max_names for frame in frames) + out.seek(0) + assert Result.parse(out, format="jelly") == result + + +def test_stream_options_are_written_once(graph: Graph) -> None: + data = graph.query(SELECT).serialize( + format="jelly", options=SparqlOptions(frame_size=3) + ) + assert data is not None + frames = list(sparql_frames(io.BytesIO(data))) + has_options = [frame.HasField("options") for frame in frames] + assert has_options == [True] + [False] * (len(frames) - 1) + assert frames[0].options.version == 1 + assert frames[0].options.stream_type == jelly.SPARQL_STREAM_TYPE_FLAT + assert list(frames[0].variables) == ["person", "name", "age"] + assert all(not frame.variables for frame in frames[1:]) diff --git a/tests/unit_tests/test_parse/test_lookup_decoder.py b/tests/unit_tests/test_parse/test_lookup_decoder.py index 9a306c96..ab1b0568 100644 --- a/tests/unit_tests/test_parse/test_lookup_decoder.py +++ b/tests/unit_tests/test_parse/test_lookup_decoder.py @@ -24,3 +24,10 @@ def test_at_invalid_index() -> None: decoder.at(2) assert "invalid resolved index 2" in str(excinfo.value) assert decoder.last_reused_index == 2 + + +def test_assign_entry_outside_lookup() -> None: + decoder = LookupDecoder(lookup_size=2) + decoder.assign_entry(2, "b") + with pytest.raises(JellyConformanceError, match="the lookup has 2 entries"): + decoder.assign_entry(0, "c") diff --git a/tests/unit_tests/test_parse/test_sparql.py b/tests/unit_tests/test_parse/test_sparql.py new file mode 100644 index 00000000..20a77ba3 --- /dev/null +++ b/tests/unit_tests/test_parse/test_sparql.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +import io + +import pytest + +from pyjelly import jelly +from pyjelly.errors import JellyConformanceError +from pyjelly.integrations.generic.decode import GenericStatementSinkAdapter +from pyjelly.integrations.generic.generic_sink import IRI +from pyjelly.parse.sparql import SparqlDecoder, read_result_set +from pyjelly.serialize.ioutils import write_delimited, write_single + +OPTIONS = jelly.SparqlResultsOptions(max_name_table_size=128, version=1) + + +def _decoder(**kwargs: int) -> SparqlDecoder: + return SparqlDecoder(GenericStatementSinkAdapter, **kwargs) + + +def _frame(**fields: object) -> jelly.SparqlResultsFrame: + return jelly.SparqlResultsFrame(options=OPTIONS, **fields) # type: ignore[arg-type] + + +def test_row_limit() -> None: + decoder = _decoder(max_rows=10) + with pytest.raises(JellyConformanceError, match="more than the 10 allowed"): + decoder.decode(_frame(variables=["x"], row_count=11)) + + +def test_value_limit() -> None: + # Checked before the cells are made + decoder = _decoder(max_values=100) + frame = _frame(variables=["x", "y"], row_count=51) + with pytest.raises(JellyConformanceError, match="100 values allowed"): + decoder.decode(frame) + + +def test_zero_variables() -> None: + decoded = _decoder().decode(_frame(row_count=3)) + assert decoded.variables == [] + assert decoded.rows == [(), (), ()] + + +def test_column_padded_with_unbound_cells() -> None: + column = jelly.RdfColumn(name_ids=[1]) + frame = _frame( + variables=["x"], + row_count=3, + names=[jelly.RdfLookupEntryPacked(values=["http://example.org/a"])], + columns=[column], + ) + decoded = _decoder().decode(frame) + assert decoded.rows == [(IRI("http://example.org/a"),), (None,), (None,)] + + +def test_links_not_utf8_are_ignored() -> None: + decoded = _decoder().decode(_frame(metadata={"link": b"\xff"})) + assert decoded.links is None + + +@pytest.mark.parametrize("delimited", [True, False]) +def test_read_delimited_or_not(*, delimited: bool) -> None: + out = io.BytesIO() + frame = _frame(variables=["x"], trailer=jelly.SparqlResultsTrailer()) + if delimited: + write_delimited(frame, out) + else: + write_single(frame, out) + out.seek(0) + results = read_result_set(out, GenericStatementSinkAdapter) + assert results.variables == ["x"] + assert list(results) == [] + assert not results.truncated diff --git a/tests/unit_tests/test_serialize/test_sparql.py b/tests/unit_tests/test_serialize/test_sparql.py new file mode 100644 index 00000000..a268e2ff --- /dev/null +++ b/tests/unit_tests/test_serialize/test_sparql.py @@ -0,0 +1,147 @@ +from __future__ import annotations + +import pytest + +from pyjelly import jelly +from pyjelly.errors import JellyConformanceError +from pyjelly.integrations.generic.generic_sink import IRI, Literal +from pyjelly.integrations.generic.serialize import GenericSinkTermEncoder +from pyjelly.integrations.generic.sparql import sparql_stream +from pyjelly.options import LookupPreset, SparqlOptions +from pyjelly.serialize.sparql import SparqlStream, links_value + +A = IRI("http://example.org/a") + + +def test_variable_names() -> None: + stream = sparql_stream() + stream.start(["?s", "$p", "o"]) + frame = stream.end() + assert list(frame.variables) == ["s", "p", "o"] + + +@pytest.mark.parametrize("name", ["", "?", "$"]) +def test_empty_variable_name(name: str) -> None: + with pytest.raises(JellyConformanceError, match="cannot be empty"): + sparql_stream().start([name]) + + +def test_first_frame() -> None: + stream = sparql_stream(SparqlOptions(stream_name="s")) + stream.start(["x"], links=["http://example.org/l1", "http://example.org/l2"]) + stream.row([A]) + frame = stream.end() + assert frame.options == jelly.SparqlResultsOptions( + stream_name="s", + max_name_table_size=8192, + max_prefix_table_size=1024, + max_datatype_table_size=64, + version=1, + ) + assert frame.metadata["link"] == b"http://example.org/l1\nhttp://example.org/l2" + assert frame.row_count == 1 + assert frame.HasField("trailer") + assert not frame.trailer.error + + +def test_frame_without_rows_has_no_columns() -> None: + stream = sparql_stream() + stream.start(["x", "y"]) + frame = stream.end() + assert frame.row_count == 0 + assert not frame.columns + + +def test_unbound_column_is_an_empty_message() -> None: + stream = sparql_stream() + stream.start(["x", "y"]) + stream.row([A, None]) + frame = stream.end() + assert len(frame.columns) == 2 + assert frame.columns[1] == jelly.RdfColumn() + + +def test_row_length() -> None: + stream = sparql_stream() + stream.start(["x", "y"]) + with pytest.raises(JellyConformanceError, match="expected 2 values"): + stream.row([A]) + + +def test_order_of_calls() -> None: + stream = sparql_stream() + with pytest.raises(JellyConformanceError, match="start a result set"): + stream.row([A]) + with pytest.raises(JellyConformanceError, match="no result set to end"): + stream.end() + stream.start(["x"]) + with pytest.raises(JellyConformanceError, match="not ended yet"): + stream.start(["x"]) + with pytest.raises(JellyConformanceError, match="not ended yet"): + stream.ask(True) # noqa: FBT003 + stream.end() + # A FLAT stream has one result set + with pytest.raises(JellyConformanceError, match="punctuated=True"): + stream.start(["x"]) + with pytest.raises(JellyConformanceError, match="punctuated=True"): + stream.ask(True) # noqa: FBT003 + + +def test_failed_row() -> None: + stream = sparql_stream( + SparqlOptions(lookup_preset=LookupPreset(max_names=128, max_datatypes=0)) + ) + stream.start(["x"]) + stream.row([A]) + with pytest.raises(JellyConformanceError, match="datatype lookup"): + stream.row([Literal("1", datatype="http://example.org/type")]) + with pytest.raises(JellyConformanceError, match="failed to encode"): + stream.row([A]) + with pytest.raises(JellyConformanceError, match="failed to encode"): + stream.end() + frame = stream.end("the row failed") + # The rows of the frame are dropped + assert frame.row_count == 0 + assert frame.HasField("options") + assert list(frame.variables) == ["x"] + assert frame.trailer.error == "the row failed" + with pytest.raises(JellyConformanceError, match="failed to encode"): + stream.end("again") + + +def test_punctuated() -> None: + stream = sparql_stream(SparqlOptions(punctuated=True)) + first = stream.ask(True) # noqa: FBT003 + assert first.HasField("options") + assert first.options.stream_type == jelly.SPARQL_STREAM_TYPE_PUNCTUATED + stream.start(["x"]) + stream.row([A]) + second = stream.end() + assert not second.HasField("options") + assert list(second.variables) == ["x"] + third = stream.ask(False) # noqa: FBT003 + assert third.HasField("trailer") + assert not third.ask_result.value + + +def test_punctuated_must_be_delimited() -> None: + with pytest.raises(JellyConformanceError, match="delimited"): + SparqlOptions(punctuated=True, delimited=False) + + +def test_options_checks() -> None: + with pytest.raises(JellyConformanceError, match="at least 128"): + SparqlOptions(lookup_preset=LookupPreset(max_names=64)) + with pytest.raises(JellyConformanceError, match="frame size"): + SparqlOptions(frame_size=0) + + +def test_encoder_lookups_must_match() -> None: + encoder = GenericSinkTermEncoder(lookup_preset=LookupPreset()) + with pytest.raises(ValueError, match="lookup sizes"): + SparqlStream(encoder, SparqlOptions()) + + +def test_links_with_lf() -> None: + with pytest.raises(JellyConformanceError, match="LF"): + links_value(["http://example.org/a\nb"]) From 4dee0403377ced612904f33961ba0e84e336fad1 Mon Sep 17 00:00:00 2001 From: Ostrzyciel Date: Mon, 26 Oct 2026 17:52:35 +0100 Subject: [PATCH 03/24] Add the Jelly-SPARQL conformance tests Run the tests of jelly-protobuf's test/sparql with the rdflib integration and the generic API, like the Jelly-RDF ones. rdflib skips the RDF 1.2 categories, as it has no triple terms or base directions. The parse tests compare with the expected SPARQL JSON results, the serialize tests check the options, the trailer, and that the output reads back as the input. The conformance report script writes separate reports for Jelly-SPARQL. --- .github/workflows/_build-wheels.yml | 2 + .github/workflows/ci.yml | 2 + pyproject.toml | 7 +- .../generate_conformance_report.py | 29 +++- tests/conformance_tests/sparql | 1 + tests/conformance_tests/test_rdf/_common.py | 13 +- .../test_sparql/test_parse.py | 84 +++++++++ .../test_sparql/test_serialize.py | 130 ++++++++++++++ tests/meta.py | 4 + tests/utils/sparql_compare.py | 160 ++++++++++++++++++ 10 files changed, 421 insertions(+), 11 deletions(-) create mode 120000 tests/conformance_tests/sparql create mode 100644 tests/conformance_tests/test_sparql/test_parse.py create mode 100644 tests/conformance_tests/test_sparql/test_serialize.py create mode 100644 tests/utils/sparql_compare.py diff --git a/.github/workflows/_build-wheels.yml b/.github/workflows/_build-wheels.yml index 3505aaeb..f2941450 100644 --- a/.github/workflows/_build-wheels.yml +++ b/.github/workflows/_build-wheels.yml @@ -55,9 +55,11 @@ jobs: cp -r submodules/protobuf/proto pyjelly/_proto rm -rf tests/conformance_tests/rdf rm -rf tests/conformance_tests/grpc + rm -rf tests/conformance_tests/sparql mkdir -p tests/conformance_tests cp -r submodules/protobuf/test/rdf tests/conformance_tests/rdf cp -r submodules/protobuf/test/grpc tests/conformance_tests/grpc + cp -r submodules/protobuf/test/sparql tests/conformance_tests/sparql - uses: astral-sh/setup-uv@v7 with: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 22a229fa..bb37c449 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,9 +68,11 @@ jobs: rm -r pyjelly/_proto rm -r tests/conformance_tests/rdf rm -r tests/conformance_tests/grpc + rm -r tests/conformance_tests/sparql cp -r submodules/protobuf/proto pyjelly/_proto cp -r submodules/protobuf/test/rdf tests/conformance_tests/rdf cp -r submodules/protobuf/test/grpc tests/conformance_tests/grpc + cp -r submodules/protobuf/test/sparql tests/conformance_tests/sparql - uses: astral-sh/setup-uv@v7 with: diff --git a/pyproject.toml b/pyproject.toml index a3dbeb61..732f59b5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -187,7 +187,8 @@ exclude = [ ".*_pb2(_grpc)?\\.pyi?", "examples/", "submodules/", - "tests/conformance_tests/rdf/" + "tests/conformance_tests/rdf/", + "tests/conformance_tests/sparql/" ] [[tool.mypy.overrides]] @@ -200,10 +201,10 @@ extra-standard-library = ["typing_extensions"] [tool.pytest.ini_options] pythonpath = "." -addopts = ["--import-mode=importlib", "--doctest-modules", "--ignore-glob=docs/examples/**", "--ignore-glob=examples/**", "--ignore-glob=submodules/**", "--ignore-glob=tests/conformance_tests/rdf/**"] +addopts = ["--import-mode=importlib", "--doctest-modules", "--ignore-glob=docs/examples/**", "--ignore-glob=examples/**", "--ignore-glob=submodules/**", "--ignore-glob=tests/conformance_tests/rdf/**", "--ignore-glob=tests/conformance_tests/sparql/**"] [tool.ruff] -extend-exclude = ["*{_pb2,_pb2_grpc}.{py,pyi}", "submodules", "tests/conformance_tests/rdf"] +extend-exclude = ["*{_pb2,_pb2_grpc}.{py,pyi}", "submodules", "tests/conformance_tests/rdf", "tests/conformance_tests/sparql"] [tool.setuptools] packages = ["pyjelly"] diff --git a/tests/conformance_tests/generate_conformance_report.py b/tests/conformance_tests/generate_conformance_report.py index 548bbd62..4fc9c1b9 100644 --- a/tests/conformance_tests/generate_conformance_report.py +++ b/tests/conformance_tests/generate_conformance_report.py @@ -78,6 +78,8 @@ def __init__( self, path_to_rdflib_report: str, path_to_generic_report: str, + path_to_rdflib_sparql_report: str, + path_to_generic_sparql_report: str, ) -> None: """ Initialize ConformanceReportPlugin. @@ -85,10 +87,16 @@ def __init__( Args: path_to_rdflib_report (str): report path for RDFLib integration path_to_generic_report (str): report path for pyjelly with Generic API + path_to_rdflib_sparql_report (str): report path of the Jelly-SPARQL tests + of the RDFLib integration + path_to_generic_sparql_report (str): report path of the Jelly-SPARQL + tests of pyjelly with Generic API """ self.path_to_rdflib_report = path_to_rdflib_report self.path_to_generic_report = path_to_generic_report + self.path_to_rdflib_sparql_report = path_to_rdflib_sparql_report + self.path_to_generic_sparql_report = path_to_generic_sparql_report self._results: defaultdict[str, list[Result]] = defaultdict(list) self._case_by_id: dict[str, object | None] = defaultdict(list) @@ -263,10 +271,19 @@ def pytest_sessionfinish(self) -> None: Builds RDF Graph using RDFLib and serializes to Turtle. """ - g_generic = self._initialize_graph(implementation="generic") - g_rdflib = self._initialize_graph(implementation="RDFLib") + rdf_graphs = ( + self._initialize_graph(implementation="generic"), + self._initialize_graph(implementation="RDFLib"), + ) + sparql_graphs = ( + self._initialize_graph(implementation="generic"), + self._initialize_graph(implementation="RDFLib"), + ) for k, v in self._results.items(): + # The tests of Jelly-SPARQL have reports of their own + is_sparql = "/tests/sparql/" in str(k) + g_generic, g_rdflib = sparql_graphs if is_sparql else rdf_graphs if len(v) == 1: r = v[0] if ( @@ -311,8 +328,10 @@ def pytest_sessionfinish(self) -> None: g_rdflib, r.outcome, k, ASSERTOR, IMPL ) - g_rdflib.serialize(self.path_to_rdflib_report, format="turtle") - g_generic.serialize(self.path_to_generic_report, format="turtle") + rdf_graphs[1].serialize(self.path_to_rdflib_report, format="turtle") + rdf_graphs[0].serialize(self.path_to_generic_report, format="turtle") + sparql_graphs[1].serialize(self.path_to_rdflib_sparql_report, format="turtle") + sparql_graphs[0].serialize(self.path_to_generic_sparql_report, format="turtle") def main() -> int: @@ -321,6 +340,8 @@ def main() -> int: conformance_report_plugin = ConformanceReportPlugin( path_to_rdflib_report="pyjelly RDFLib.ttl", path_to_generic_report="pyjelly Generic API.ttl", + path_to_rdflib_sparql_report="pyjelly RDFLib SPARQL.ttl", + path_to_generic_sparql_report="pyjelly Generic API SPARQL.ttl", ) return pytest.main(args=[tests_dir], plugins=[conformance_report_plugin]) diff --git a/tests/conformance_tests/sparql b/tests/conformance_tests/sparql new file mode 120000 index 00000000..98943cec --- /dev/null +++ b/tests/conformance_tests/sparql @@ -0,0 +1 @@ +../../submodules/protobuf/test/sparql \ No newline at end of file diff --git a/tests/conformance_tests/test_rdf/_common.py b/tests/conformance_tests/test_rdf/_common.py index e791aa78..c21e8ad5 100644 --- a/tests/conformance_tests/test_rdf/_common.py +++ b/tests/conformance_tests/test_rdf/_common.py @@ -10,7 +10,8 @@ MF = Namespace("http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#") # Requirements of the tests that the rdflib integration can't pass: rdflib has no -# generalized statements, quoted triples, triple terms, or base directions +# generalized statements, quoted triples, triple terms, or base directions. The same +# for Jelly-SPARQL. NOT_RDFLIB = { JELLYT.requirementRdfStar, JELLYT.requirementGeneralizedRdf, @@ -24,7 +25,7 @@ class ConformanceCase: uri: str name: str positive: bool - #: The suite of the test: "v1_1" or "v1_2" + #: The suite of the test: "v1_1" or "v1_2", or "sparql" suite: str actions: list[Path] results: list[Path] @@ -52,18 +53,22 @@ def graphs(self) -> bool: def messages(self) -> bool: return JELLYT.requirementMessages in self.requirements + @property + def punctuated(self) -> bool: + return JELLYT.requirementPunctuated in self.requirements + @property def options(self) -> Path | None: """The stream options of a to_jelly test.""" for path in self.actions: - if path.name == "stream_options.jelly": + if path.stem == "stream_options": return path return None @property def inputs(self) -> list[Path]: """The input files, without the stream options.""" - return [path for path in self.actions if path.name != "stream_options.jelly"] + return [path for path in self.actions if path.stem != "stream_options"] def load_cases(manifest_path: Path) -> list[ConformanceCase]: diff --git a/tests/conformance_tests/test_sparql/test_parse.py b/tests/conformance_tests/test_sparql/test_parse.py new file mode 100644 index 00000000..9bf3d321 --- /dev/null +++ b/tests/conformance_tests/test_sparql/test_parse.py @@ -0,0 +1,84 @@ +"""The "from Jelly" conformance tests of Jelly-SPARQL.""" + +from __future__ import annotations + +from typing import IO, Any + +import pytest +from rdflib.query import Result + +from pyjelly.integrations.generic.sparql import ( + parse_jelly_sparql as generic_parse_jelly_sparql, +) +from pyjelly.integrations.generic.sparql import ( + parse_jelly_sparql_results as generic_parse_jelly_sparql_results, +) +from pyjelly.integrations.rdflib.sparql import parse_jelly_sparql_results +from tests.conformance_tests.test_rdf._common import ConformanceCase, load_cases +from tests.meta import SPARQL_FROM_JELLY_MANIFEST +from tests.utils.sparql_compare import ( + Results, + assert_same_result_sets, + generic_results, + rdflib_results, + read_srj, +) + +CASES = load_cases(SPARQL_FROM_JELLY_MANIFEST) + + +def _params(*, positive: bool, rdflib: bool = False) -> list[Any]: + return [ + pytest.param(case, id=case.id) + for case in CASES + if case.positive == positive and (case.rdflib or not rdflib) + ] + + +def _expected(case: ConformanceCase) -> list[Results]: + return [read_srj(path) for path in case.results] + + +@pytest.mark.parametrize("case", _params(positive=True, rdflib=True)) +def test_rdflib_parses_positive(case: ConformanceCase) -> None: + with case.actions[0].open("rb") as file: + if case.punctuated: + results = [rdflib_results(r) for r in parse_jelly_sparql_results(file)] + else: + results = [rdflib_results(Result.parse(file, format="jelly"))] + assert_same_result_sets(_expected(case), results, hint=case.id) + + +@pytest.mark.parametrize("case", _params(positive=True)) +def test_generic_parses_positive(case: ConformanceCase) -> None: + with case.actions[0].open("rb") as file: + if case.punctuated: + results = [ + generic_results(r) for r in generic_parse_jelly_sparql_results(file) + ] + else: + results = [generic_results(generic_parse_jelly_sparql(file))] + assert_same_result_sets(_expected(case), results, hint=case.id) + + +@pytest.mark.parametrize("case", _params(positive=False, rdflib=True)) +def test_rdflib_parsing_fails_negative(case: ConformanceCase) -> None: + with ( + pytest.raises(Exception), # noqa: B017,PT011 any error will do + case.actions[0].open("rb") as file, + ): + list(parse_jelly_sparql_results(file)) + + +def _read_all(file: IO[bytes]) -> None: + for result_set in generic_parse_jelly_sparql_results(file): + list(result_set) + + +@pytest.mark.parametrize("case", _params(positive=False)) +def test_generic_parsing_fails_negative(case: ConformanceCase) -> None: + with ( + pytest.raises(Exception), # noqa: B017,PT011 any error will do + case.actions[0].open("rb") as file, + ): + _read_all(file) diff --git a/tests/conformance_tests/test_sparql/test_serialize.py b/tests/conformance_tests/test_sparql/test_serialize.py new file mode 100644 index 00000000..6ecd1403 --- /dev/null +++ b/tests/conformance_tests/test_sparql/test_serialize.py @@ -0,0 +1,130 @@ +"""The "to Jelly" conformance tests of Jelly-SPARQL.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest +from rdflib.query import Result + +from pyjelly import jelly +from pyjelly.integrations.generic.sparql import ( + parse_jelly_sparql_results, + sparql_stream, +) +from pyjelly.integrations.rdflib.sparql import ( + serialize_jelly_sparql, + serialize_jelly_sparql_results, +) +from pyjelly.options import LookupPreset, SparqlOptions +from pyjelly.parse.sparql import sparql_frames +from pyjelly.serialize.sparql import FrameWriter, write_solutions +from tests.conformance_tests.test_rdf._common import ConformanceCase, load_cases +from tests.meta import SPARQL_TO_JELLY_MANIFEST, TEST_OUTPUTS_DIR +from tests.utils.sparql_compare import ( + assert_same_result_sets, + generic_results, + generic_rows, + read_srj, +) + +CASES = load_cases(SPARQL_TO_JELLY_MANIFEST) + + +def _params(*, positive: bool, rdflib: bool = False) -> list[Any]: + return [ + pytest.param(case, id=case.id) + for case in CASES + if case.positive == positive and (case.rdflib or not rdflib) + ] + + +def _output(case: ConformanceCase) -> Path: + return TEST_OUTPUTS_DIR / f"{case.id}.jellys" + + +def _stream_options(case: ConformanceCase) -> jelly.SparqlResultsOptions: + assert case.options + with case.options.open("rb") as file: + return next(sparql_frames(file)).options + + +def _options(case: ConformanceCase) -> SparqlOptions: + """Return the options of a test, as written in its stream options file.""" + options = _stream_options(case) + return SparqlOptions( + lookup_preset=LookupPreset( + max_names=options.max_name_table_size, + max_prefixes=options.max_prefix_table_size, + max_datatypes=options.max_datatype_table_size, + ), + rdf_version=options.rdf_version, + punctuated=options.stream_type == jelly.SPARQL_STREAM_TYPE_PUNCTUATED, + stream_name=options.stream_name, + ) + + +def _write_generic(case: ConformanceCase) -> None: + stream = sparql_stream(_options(case)) + with _output(case).open("wb") as file: + writer = FrameWriter(file, delimited=True) + for path in case.inputs: + results = read_srj(path) + if results.boolean is not None: + writer.write(stream.ask(results.boolean)) + else: + write_solutions( + stream, results.variables, generic_rows(results), writer + ) + + +def _write_rdflib(case: ConformanceCase) -> None: + results = [] + for path in case.inputs: + with path.open("rb") as file: + results.append(Result.parse(file, format="json")) + options = _options(case) + with _output(case).open("wb") as file: + if options.punctuated: + serialize_jelly_sparql_results(results, file, options) + else: + (result,) = results + serialize_jelly_sparql(result, file, options) + + +def _check(case: ConformanceCase) -> None: + output = _output(case) + with output.open("rb") as file: + frames = list(sparql_frames(file)) + assert frames[0].options == _stream_options(case) + assert frames[-1].HasField("trailer") + assert not frames[-1].trailer.error + with output.open("rb") as file: + actual = [generic_results(r) for r in parse_jelly_sparql_results(file)] + expected = [read_srj(path) for path in case.inputs] + assert_same_result_sets(expected, actual, hint=case.id) + + +@pytest.mark.parametrize("case", _params(positive=True, rdflib=True)) +def test_serializes_rdflib_positive(case: ConformanceCase) -> None: + _write_rdflib(case) + _check(case) + + +@pytest.mark.parametrize("case", _params(positive=True)) +def test_serializes_generic_positive(case: ConformanceCase) -> None: + _write_generic(case) + _check(case) + + +@pytest.mark.parametrize("case", _params(positive=False, rdflib=True)) +def test_serializing_fails_rdflib_negative(case: ConformanceCase) -> None: + with pytest.raises(Exception): # noqa: B017,PT011 any error will do + _write_rdflib(case) + + +@pytest.mark.parametrize("case", _params(positive=False)) +def test_serializing_fails_generic_negative(case: ConformanceCase) -> None: + with pytest.raises(Exception): # noqa: B017,PT011 any error will do + _write_generic(case) diff --git a/tests/meta.py b/tests/meta.py index 9b0b1004..bf1352b8 100644 --- a/tests/meta.py +++ b/tests/meta.py @@ -16,3 +16,7 @@ RDF_TO_JELLY_MANIFESTS = [ RDF_TESTS_DIR / suite / "to_jelly" / "manifest.ttl" for suite in RDF_TEST_SUITES ] + +SPARQL_TESTS_DIR = CONFORMANCE_TESTS_DIR / "sparql" +SPARQL_FROM_JELLY_MANIFEST = SPARQL_TESTS_DIR / "from_jelly" / "manifest.ttl" +SPARQL_TO_JELLY_MANIFEST = SPARQL_TESTS_DIR / "to_jelly" / "manifest.ttl" diff --git a/tests/utils/sparql_compare.py b/tests/utils/sparql_compare.py new file mode 100644 index 00000000..3cd0298b --- /dev/null +++ b/tests/utils/sparql_compare.py @@ -0,0 +1,160 @@ +""" +Reading SPARQL JSON results and comparing results, for Jelly-SPARQL tests. + +Two results are equivalent if both are boolean results with the same value, or if +they have the same variables and the same rows in the same order, with a +one-to-one mapping between their blank nodes. Links are not compared. +""" + +from __future__ import annotations + +import json +from collections.abc import Iterable +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +from rdflib.query import Result + +from pyjelly.integrations.generic.generic_sink import ( + IRI, + BlankNode, + Literal, + Node, + Triple, +) +from pyjelly.parse.sparql import SparqlResultSet +from tests.utils.rdf_compare import ( + Term, + _BlankNodes, + _literal, + generic_term, + rdflib_term, +) + +Row = tuple[Term | None, ...] + + +@dataclass +class Results: + """Results, with their terms as in rdf_compare.""" + + variables: list[str] = field(default_factory=list) + rows: list[Row] = field(default_factory=list) + boolean: bool | None = None + + +def _json_term(value: dict[str, Any]) -> Term: + kind = value["type"] + if kind == "uri": + return ("iri", value["value"]) + if kind == "bnode": + return ("bnode", value["value"]) + if kind in ("literal", "typed-literal"): + return _literal( + value["value"], + value.get("xml:lang"), + value.get("datatype"), + value.get("its:dir"), + ) + if kind == "triple": + triple = value["value"] + return ( + "triple", + _json_term(triple["subject"]), + _json_term(triple["predicate"]), + _json_term(triple["object"]), + ) + msg = f"unknown term type: {kind}" + raise ValueError(msg) + + +def read_srj(path: Path) -> Results: + """Read a file of SPARQL JSON results.""" + data = json.loads(path.read_text("utf-8")) + if "boolean" in data: + return Results(boolean=bool(data["boolean"])) + variables: list[str] = data["head"]["vars"] + rows = [ + tuple(_json_term(b[name]) if name in b else None for name in variables) + for b in data["results"]["bindings"] + ] + return Results(variables, rows) + + +def generic_node(term: Term) -> Node: + """Return the term of the generic API for a term.""" + kind = term[0] + if kind == "iri": + return IRI(term[1]) + if kind == "bnode": + return BlankNode(term[1]) + if kind == "literal": + return Literal(term[1], term[2], term[3], term[4]) + if kind == "triple": + return Triple(*map(generic_node, term[1:])) + msg = f"unknown term: {term!r}" + raise ValueError(msg) + + +def generic_rows(results: Results) -> list[tuple[Node | None, ...]]: + """Return the rows with terms of the generic API.""" + return [ + tuple(None if term is None else generic_node(term) for term in row) + for row in results.rows + ] + + +def generic_results(result_set: SparqlResultSet) -> Results: + """Read a result set of the generic API whole.""" + rows = [ + tuple(None if value is None else generic_term(value) for value in row) + for row in result_set + ] + return Results(result_set.variables, rows, result_set.boolean) + + +def rdflib_results(result: Result) -> Results: + if result.type == "ASK": + return Results(boolean=bool(result.askAnswer)) + variables = result.vars or [] + rows = [ + tuple( + None if binding.get(v) is None else rdflib_term(binding[v]) + for v in variables + ) + for binding in result.bindings + ] + return Results([str(v) for v in variables], rows) + + +def assert_same_results(expected: Results, actual: Results, hint: str = "") -> None: + """Check that two results are equivalent (see the module).""" + assert actual.boolean == expected.boolean, ( + f"{hint}: expected the boolean result {expected.boolean}, got {actual.boolean}" + ) + assert actual.variables == expected.variables, ( + f"{hint}: expected the variables {expected.variables}, got {actual.variables}" + ) + assert len(actual.rows) == len(expected.rows), ( + f"{hint}: expected {len(expected.rows)} rows, got {len(actual.rows)}" + ) + blank_nodes = _BlankNodes() + for index, (e, a) in enumerate(zip(expected.rows, actual.rows, strict=True)): + same = len(e) == len(a) and all( + x is y if x is None or y is None else blank_nodes.same(x, y) + for x, y in zip(e, a, strict=True) + ) + assert same, f"{hint}: row {index} differs: expected {e}, got {a}" + + +def assert_same_result_sets( + expected: Iterable[Results], actual: Iterable[Results], hint: str = "" +) -> None: + expected = list(expected) + actual = list(actual) + assert len(actual) == len(expected), ( + f"{hint}: expected {len(expected)} result sets, got {len(actual)}" + ) + for index, (e, a) in enumerate(zip(expected, actual, strict=True)): + assert_same_results(e, a, hint=f"{hint}: result set {index}") From a9c1d93ed45f78b3fbed4289ff8ed1a519a781cf Mon Sep 17 00:00:00 2001 From: Ostrzyciel Date: Mon, 26 Oct 2026 17:55:52 +0100 Subject: [PATCH 04/24] Read rdflib query results lazily, if asked parse_jelly_sparql(lazy=True), also through Result.parse(), reads the rows as the result is iterated, like rdflib does with the results of its own queries. --- pyjelly/integrations/rdflib/sparql.py | 66 +++++++++++-------- .../test_rdflib/test_sparql.py | 39 +++++++++++ 2 files changed, 79 insertions(+), 26 deletions(-) diff --git a/pyjelly/integrations/rdflib/sparql.py b/pyjelly/integrations/rdflib/sparql.py index f110bf90..3aa3edee 100644 --- a/pyjelly/integrations/rdflib/sparql.py +++ b/pyjelly/integrations/rdflib/sparql.py @@ -57,38 +57,50 @@ def make(options: ParserOptions) -> Adapter: return make -def _to_result(result_set: SparqlResultSet) -> JellySparqlResult: - """Read a result set whole, into an rdflib Result.""" +def _to_result(result_set: SparqlResultSet, *, lazy: bool) -> JellySparqlResult: + """Return a result set as an rdflib Result.""" if result_set.boolean is not None: result = JellySparqlResult("ASK") result.askAnswer = result_set.boolean else: result = JellySparqlResult("SELECT") result.vars = [Variable(name) for name in result_set.variables] + rows = _read_bindings(result_set, result) + if lazy and result_set.boolean is None: + # rdflib makes the bindings as the result is iterated, as for its own + # query results + result.bindings = rows + return result bindings: list[Mapping[Variable, Identifier]] = [] result.bindings = bindings + # If the results are incomplete, the result has those read before the error + bindings.extend(rows) + return result + + +def _read_bindings( + result_set: SparqlResultSet, result: JellySparqlResult +) -> Iterator[Mapping[Variable, Identifier]]: + """Yield the bindings of the rows, and when they end, what the stream said.""" variables = result.vars or [] try: - # Unbound variables are left out, as rdflib does - bindings.extend( - { + for row in result_set: + # Unbound variables are left out, as rdflib does + yield { variable: value for variable, value in zip(variables, row, strict=True) if value is not None } - for row in result_set - ) except JellyIncompleteResultsError as error: - result.links = result_set.links error.result = result raise - result.links = result_set.links - result.truncated = result_set.truncated - return result + finally: + result.links = result_set.links + result.truncated = result_set.truncated def parse_jelly_sparql( - inp: IO[bytes], *, validate_iris: bool = False + inp: IO[bytes], *, validate_iris: bool = False, lazy: bool = False ) -> JellySparqlResult: """ Read SPARQL query results from Jelly-SPARQL. @@ -98,6 +110,9 @@ def parse_jelly_sparql( (the FLAT stream type, the default). validate_iris (bool): whether rdflib checks the IRIs. An invalid IRI is still read, rdflib only logs a warning. Checking slows down parsing. + lazy (bool): whether to read the rows as the result is iterated, rather + than all of them now. The stream must then stay open until the result + was iterated, and the error of incomplete results is raised then. Returns: JellySparqlResult: the results, an rdflib Result. @@ -109,7 +124,7 @@ def parse_jelly_sparql( """ adapter_factory = _adapter_factory(validate_iris=validate_iris) - return _to_result(read_result_set(inp, adapter_factory)) + return _to_result(read_result_set(inp, adapter_factory), lazy=lazy) def parse_jelly_sparql_results( @@ -136,7 +151,7 @@ def parse_jelly_sparql_results( """ adapter_factory = _adapter_factory(validate_iris=validate_iris) for result_set in read_result_sets(inp, adapter_factory): - yield _to_result(result_set) + yield _to_result(result_set, lazy=False) def sparql_stream(options: SparqlOptions | None = None) -> SparqlStream: @@ -162,12 +177,12 @@ def _write_result(stream: SparqlStream, result: Result, writer: FrameWriter) -> variables = result.vars or [] rows = ( [binding.get(variable) for variable in variables] - for binding in _bindings(result) + for binding in _made_bindings(result) ) write_solutions(stream, variables, rows, writer, links) -def _bindings(result: Result) -> Iterator[Mapping[Variable, Identifier]]: +def _made_bindings(result: Result) -> Iterator[Mapping[Variable, Identifier]]: """ Return the bindings of a result, as they are made. @@ -232,12 +247,6 @@ def serialize_jelly_sparql_results( _write_result(stream, result, writer) -def _input_stream(source: IO[bytes] | str | Path) -> IO[bytes]: - if isinstance(source, (str, Path)): - return Path(source).open("rb") - return source - - class RDFLibJellySparqlParser(ResultParser): """Reads SPARQL query results in Jelly-SPARQL, for rdflib's Result.parse().""" @@ -248,6 +257,7 @@ def parse( content_type: str | None = None, *, validate_iris: bool = False, + lazy: bool = False, **unused: Any, ) -> Result: """ @@ -258,13 +268,17 @@ def parse( content_type (str | None): unused. validate_iris (bool): whether rdflib checks the IRIs, see parse_jelly_sparql(). + lazy (bool): whether to read the rows as the result is iterated, see + parse_jelly_sparql(). Only for streams, not paths. **unused (Any): unused arguments of Result.parse(). """ - inp = _input_stream(source) - if inp is source: - return parse_jelly_sparql(inp, validate_iris=validate_iris) - with inp: + if not isinstance(source, (str, Path)): + return parse_jelly_sparql(source, validate_iris=validate_iris, lazy=lazy) + if lazy: + msg = "lazy reading needs a stream that stays open, not a path" + raise ValueError(msg) + with Path(source).open("rb") as inp: return parse_jelly_sparql(inp, validate_iris=validate_iris) diff --git a/tests/integration_tests/test_rdflib/test_sparql.py b/tests/integration_tests/test_rdflib/test_sparql.py index 0ac05bd9..b461a095 100644 --- a/tests/integration_tests/test_rdflib/test_sparql.py +++ b/tests/integration_tests/test_rdflib/test_sparql.py @@ -307,3 +307,42 @@ def test_stream_options_are_written_once(graph: Graph) -> None: assert frames[0].options.stream_type == jelly.SPARQL_STREAM_TYPE_FLAT assert list(frames[0].variables) == ["person", "name", "age"] assert all(not frame.variables for frame in frames[1:]) + + +def test_lazy(graph: Graph) -> None: + result = graph.query(SELECT) + data = result.serialize(format="jelly", options=SparqlOptions(frame_size=3)) + assert data is not None + inp = io.BytesIO(data) + parsed = Result.parse(inp, format="jelly", lazy=True) + # Only the first frame was read + assert inp.tell() < len(data) + assert parsed.vars == result.vars + assert len(list(parsed)) == len(result) + assert inp.tell() == len(data) + assert parsed == result + assert isinstance(parsed, JellySparqlResult) + assert not parsed.truncated + + +def test_lazy_incomplete() -> None: + result = Result("SELECT") + result.vars = [Variable("x")] + result.bindings = _failing_bindings() + out = io.BytesIO() + with pytest.raises(RuntimeError): + serialize_jelly_sparql(result, out) + out.seek(0) + parsed = parse_jelly_sparql(out, lazy=True) + rows: list[object] = [] + with pytest.raises(JellyIncompleteResultsError): # noqa: PT012 + # Not rows.extend(parsed): it would call len(parsed), which reads all rows + for row in parsed: + rows.append(row) # noqa: PERF402 + assert len(rows) == 1 + + +def test_lazy_needs_a_stream(tmp_path: Path) -> None: + path = tmp_path / "results.jellys" + with pytest.raises(ValueError, match="stream"): + Result.parse(path, format="jelly", lazy=True) # type: ignore[arg-type] From c624b3ee05e9325d6e1bab269b63ab2df5f4f866 Mon Sep 17 00:00:00 2001 From: Ostrzyciel Date: Mon, 26 Oct 2026 17:57:13 +0100 Subject: [PATCH 05/24] Document Jelly-SPARQL A page on SPARQL query results: reading and writing them with rdflib (including from a SPARQL endpoint), lazy reading, incomplete results, options, sequences of results, and the generic API. The examples run in the tests, like the others. --- README.md | 1 + docs/index.md | 2 + docs/overview.md | 7 +- docs/sparql.md | 133 ++++++++++++++++++++++ examples/generic/10_sparql_results.py | 23 ++++ examples/rdflib/09_sparql_results.py | 32 ++++++ examples/rdflib/10_sparql_results_lazy.py | 33 ++++++ mkdocs.yml | 1 + 8 files changed, 231 insertions(+), 1 deletion(-) create mode 100644 docs/sparql.md create mode 100644 examples/generic/10_sparql_results.py create mode 100644 examples/rdflib/09_sparql_results.py create mode 100644 examples/rdflib/10_sparql_results_lazy.py diff --git a/README.md b/README.md index 53c8aa1b..a517c2ab 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ ## Features - **Fast reading and writing** of RDF knowledge graphs in the [Jelly format](http://w3id.org/jelly). +- Reading and writing **[SPARQL query results](https://w3id.org/jelly/pyjelly/dev/sparql)** in [Jelly-SPARQL](https://w3id.org/jelly/dev/specification/sparql), with rdflib or the generic API. - **Standalone [generic API](https://w3id.org/jelly/pyjelly/dev/generic-sink)** with no third-party dependencies, allowing for: - Serialization and parsing of statements to and from Jelly files. - Parsing and serializing streams of graphs and statements. diff --git a/docs/index.md b/docs/index.md index 194047f8..c9feb8b1 100644 --- a/docs/index.md +++ b/docs/index.md @@ -7,6 +7,7 @@ ## Features - **Fast reading and writing** of RDF knowledge graphs in the [Jelly format](http://w3id.org/jelly) +- **[SPARQL query results](sparql.md)** in [Jelly-SPARQL]({{ proto_link("specification/sparql/") }}) - **Seamless integration with [RDFLib](https://rdflib.readthedocs.io/)** - **Stream processing support** for large datasets or streams of many RDF graphs/datasets @@ -15,6 +16,7 @@ - **[Getting started](getting-started.md)** - **[Overview and supported features](overview.md)** - **[Generic interface (usage without RDFLib)](generic-sink.md)** +- **[SPARQL query results](sparql.md)** - **[API reference](api.md)** - **[Contributing](contributing/index.md)** – how to report issues, contribute code, and request features diff --git a/docs/overview.md b/docs/overview.md index c9e11f83..f1aa7bca 100644 --- a/docs/overview.md +++ b/docs/overview.md @@ -7,6 +7,7 @@ With Jelly, you can transmit both flat and structured streams of triples, quads, **pyjelly** is a Python implementation of the Jelly protocol. It provides: * Full support for reading and writing Jelly-encoded RDF data +* Reading and writing SPARQL query results in [Jelly-SPARQL](sparql.md) * Seamless integration with [RDFLib](https://rdflib.readthedocs.io/) (*"works just like Turtle"*) * Standalone generic API with no third-party dependencies [Generic API](generic-sink.md) * Support for all Jelly stream types @@ -39,9 +40,13 @@ A Jelly-RDF 1.2 stream is either one graph or dataset, or a sequence of [RDF Mes The generic API supports [RDF 1.2](https://www.w3.org/TR/rdf12-concepts/): triple terms, as a `Triple` in the object of a statement, and directional language-tagged strings, as `Literal("text", "en", direction="ltr")`. To declare the RDF version for a stream, set `rdf_version` in `StreamParameters`. RDFLib 7 doesn't support RDF 1.2 yet. +### SPARQL query results + +pyjelly reads and writes the results of `SELECT` and `ASK` queries in [Jelly-SPARQL]({{ proto_link("specification/sparql/") }}), with RDFLib's `Result.parse` and `Result.serialize`, or with the generic API. See [SPARQL query results](sparql.md). + ### Conformance to the Jelly specification -pyjelly is continuously tested for conformance to [the Jelly specification]({{ proto_link("specification/") }}). It passes the [conformance tests of Jelly-RDF 1.2]({{ proto_link("conformance/rdf-test-cases/") }}), and the [tests of Jelly-RDF 1.1]({{ proto_link("conformance/rdf-1-1-test-cases/") }}), except for writing the `GRAPHS` physical stream type. +pyjelly is continuously tested for conformance to [the Jelly specification]({{ proto_link("specification/") }}). It passes the [conformance tests of Jelly-RDF 1.2]({{ proto_link("conformance/rdf-test-cases/") }}), and the [tests of Jelly-RDF 1.1]({{ proto_link("conformance/rdf-1-1-test-cases/") }}), except for writing the `GRAPHS` physical stream type. It also passes the [conformance tests of Jelly-SPARQL]({{ proto_link("conformance/sparql-test-cases/") }}). See the [conformance tests]({{ git_link("tests/conformance_tests") }}) in the pyjelly repository. diff --git a/docs/sparql.md b/docs/sparql.md new file mode 100644 index 00000000..66d07e40 --- /dev/null +++ b/docs/sparql.md @@ -0,0 +1,133 @@ +# SPARQL query results + +pyjelly reads and writes SPARQL query results in [Jelly-SPARQL]({{ proto_link("specification/sparql/") }}): the results of `SELECT` queries (solution sequences) and `ASK` queries (boolean results). Like Jelly-RDF, it is a compact binary format that can be written and read as a stream. + +The media type of Jelly-SPARQL is `application/x-jelly-sparql`, and the file extension is `.jellys`. + +## Usage with RDFLib + +With pyjelly installed, RDFLib reads and writes query results in the `jelly` format, like it does `json` or `xml`: + +{{ code_example('rdflib/09_sparql_results.py') }} + +`Result.parse` returns a `JellySparqlResult`, which is an RDFLib `Result` with two more attributes: + +- `links` – the IRIs of links that describe the results, like `head.link` in [SPARQL JSON results](https://www.w3.org/TR/sparql12-results-json/). They are written too, if the result you write has them. +- `truncated` – `True` if the stream ended without saying that the results are complete, so some may be missing (e.g., the server closed the connection). + +RDFLib does not support RDF 1.2 yet, so triple terms and literals with a base direction can't be read into RDFLib. Use the [generic API](#generic-api) for them. + +### Querying a SPARQL endpoint + +RDFLib's `SPARQLStore` can ask a SPARQL endpoint for Jelly-SPARQL, if the endpoint offers it (e.g., Apache Jena Fuseki with the [Jelly-JVM plugin](https://w3id.org/jelly/jelly-jvm/dev/getting-started-plugins/)): + +```python +from rdflib import Graph +from rdflib.plugins.stores.sparqlstore import SPARQLStore + +store = SPARQLStore( + "http://localhost:3030/dataset/sparql", + returnFormat="application/x-jelly-sparql", +) +for row in Graph(store=store).query("SELECT * WHERE { ?s ?p ?o } LIMIT 10"): + print(row) +``` + +### Large results + +By default, `Result.parse` reads all rows at once. With `lazy=True`, the rows are read as the result is iterated, like RDFLib does with the results of its own queries. The stream must then stay open until you are done with the result. + +When writing, the rows of an RDFLib result are written as the query makes them, so the first frames are written before the query is done. + +{{ code_example('rdflib/10_sparql_results_lazy.py') }} + +!!! note + + Iterate a lazy result with a `for` loop. `list(result)` and `len(result)` read all of its rows first, as for any RDFLib result. + +### Incomplete results + +A query can fail after some of its results were already written: a timeout, or a broken backend. Jelly-SPARQL then ends the stream with an error, so that readers know that the results are incomplete. pyjelly does this for you: if getting a row fails while results are being written, the stream ends with the error, and the error is raised. + +When reading such results, pyjelly raises `JellyIncompleteResultsError` after the rows that were read. Its `error` attribute is the message of the writer, and its `result` attribute holds the results read before the error: + +```python +from pyjelly.errors import JellyIncompleteResultsError + +try: + result = Result.parse(file, format="jelly") +except JellyIncompleteResultsError as error: + print(f"The query failed: {error.error}") + partial_result = error.result +``` + +### Options + +To change how the results are written, pass `SparqlOptions` to `serialize`: + +```python +from pyjelly.options import LookupPreset, SparqlOptions + +options = SparqlOptions( + # Sizes of the lookups, see below + lookup_preset=LookupPreset(max_names=4096, max_prefixes=256, max_datatypes=32), + # How many values (rows times variables) a frame has, at most + frame_size=4096, +) +results.serialize(destination="results.jellys", format="jelly", options=options) +``` + +The default lookup sizes are larger than for Jelly-RDF (8192 names, 1024 prefixes and 64 datatypes), as in Jelly-JVM: a row of results tends to have more distinct terms than an RDF statement. Readers accept lookups of up to 16384 names, 4096 prefixes and 256 datatypes. + +With `delimited=False`, the results are written as one frame, which some transports (e.g., a message broker) need. Then the lookups must be large enough to hold all terms of the results. + +### Sequences of results + +A Jelly-SPARQL stream can also hold a sequence of results, e.g., the answers to a stream of queries, or to a query evaluated again and again. This is the `PUNCTUATED` stream type of Jelly-SPARQL. To write and read such a stream, use the functions in `pyjelly.integrations.rdflib.sparql`: + +```python +from pyjelly.integrations.rdflib.sparql import ( + parse_jelly_sparql_results, + serialize_jelly_sparql_results, +) + +with open("answers.jellys", "wb") as file: + serialize_jelly_sparql_results([result_1, result_2, ask_result], file) + +with open("answers.jellys", "rb") as file: + for result in parse_jelly_sparql_results(file): + print(result.type, len(result)) +``` + +The lookups are shared by the results of the stream, so the terms they have in common are written once. + +## Generic API + +The [generic API](generic-sink.md) reads and writes results with its own terms (`IRI`, `Literal`, `BlankNode`, and `Triple` for triple terms), so it supports all of RDF 1.2. A row is a tuple with one value per variable, or `None` if the variable is unbound: + +{{ code_example('generic/10_sparql_results.py') }} + +`parse_jelly_sparql` returns a `SparqlResultSet`, with the `variables`, the `boolean` result of an `ASK` query (`None` for a `SELECT` query), the `links`, and whether it is `truncated`. Its rows are read as you iterate it, so it can be iterated only once. `serialize_jelly_sparql_ask` writes the result of an `ASK` query. + +### Writing frame by frame + +To write results frame by frame, e.g., to send each frame as a message over Kafka, use `SparqlStream` directly. It works with the terms of RDFLib (`pyjelly.integrations.rdflib.sparql.sparql_stream()`) or of the generic API (`pyjelly.integrations.generic.sparql.sparql_stream()`): + +```python +from pyjelly.integrations.generic.sparql import sparql_stream + +stream = sparql_stream() +stream.start(["person", "name"]) +for row in rows: + # A frame, when it is full + if frame := stream.row(row): + send(frame.SerializeToString()) +# The last frame of the results, which says they are complete +send(stream.end().SerializeToString()) +``` + +If the results can't be completed, end them with `stream.end("why")` instead. + +## Conformance + +pyjelly passes the [conformance tests of Jelly-SPARQL]({{ proto_link("conformance/sparql-test-cases/") }}), with the generic API. The RDFLib integration passes all tests except those of RDF 1.2 terms. diff --git a/examples/generic/10_sparql_results.py b/examples/generic/10_sparql_results.py new file mode 100644 index 00000000..b500aad9 --- /dev/null +++ b/examples/generic/10_sparql_results.py @@ -0,0 +1,23 @@ +from pyjelly.integrations.generic.generic_sink import IRI, Literal +from pyjelly.integrations.generic.sparql import ( + parse_jelly_sparql, + serialize_jelly_sparql, +) + +alice = IRI("http://example.org/alice") +bob = IRI("http://example.org/bob") + +# The rows of the results: one value per variable, None if it is unbound +rows = [ + (alice, Literal("Alice"), bob), + (bob, Literal("Bob"), None), +] +with open("results.jellys", "wb") as file: + serialize_jelly_sparql(["person", "name", "knows"], rows, file) + +with open("results.jellys", "rb") as file: + results = parse_jelly_sparql(file) + print("Variables:", results.variables) + # The rows are read as they are iterated, so before the file is closed + for person, name, knows in results: + print(person, name, knows) diff --git a/examples/rdflib/09_sparql_results.py b/examples/rdflib/09_sparql_results.py new file mode 100644 index 00000000..1b799f5c --- /dev/null +++ b/examples/rdflib/09_sparql_results.py @@ -0,0 +1,32 @@ +from rdflib import Graph +from rdflib.query import Result + +graph = Graph() +graph.parse( + data=""" + @prefix foaf: . + foaf:name "Alice" ; foaf:knows . + foaf:name "Bob" . + """, + format="turtle", +) + +# Run a query and write its results in Jelly-SPARQL +results = graph.query(""" + PREFIX foaf: + SELECT ?person ?name WHERE { ?person foaf:name ?name } +""") +results.serialize(destination="results.jellys", format="jelly") + +# Read them back +with open("results.jellys", "rb") as file: + parsed = Result.parse(file, format="jelly") + +for row in parsed: + print(f"{row.person} is called {row.name}") + +# Results of ASK queries work the same way +answer = graph.query("ASK { ?person ?p 'Alice' }") +answer.serialize(destination="answer.jellys", format="jelly") +with open("answer.jellys", "rb") as file: + print("Is anyone called Alice?", Result.parse(file, format="jelly").askAnswer) diff --git a/examples/rdflib/10_sparql_results_lazy.py b/examples/rdflib/10_sparql_results_lazy.py new file mode 100644 index 00000000..91d41eee --- /dev/null +++ b/examples/rdflib/10_sparql_results_lazy.py @@ -0,0 +1,33 @@ +from rdflib import Literal, URIRef, Variable +from rdflib.query import Result + +from pyjelly.errors import JellyIncompleteResultsError +from pyjelly.integrations.rdflib.sparql import ( + parse_jelly_sparql, + serialize_jelly_sparql, +) + +# Many results, made one by one +results = Result("SELECT") +results.vars = [Variable("item"), Variable("number")] +results.bindings = ( + { + Variable("item"): URIRef(f"http://example.org/item/{i}"), + Variable("number"): Literal(i), + } + for i in range(100_000) +) +with open("many_results.jellys", "wb") as file: + serialize_jelly_sparql(results, file) + +# Read the rows as they are needed, not all at once +with open("many_results.jellys", "rb") as file: + parsed = parse_jelly_sparql(file, lazy=True) + try: + total = sum(row.number.toPython() for row in parsed) + except JellyIncompleteResultsError as error: + # The writer said that the results are incomplete, e.g., the query failed + print(f"Some results are missing: {error.error}") + raise + +print(f"The sum of the numbers is {total}") diff --git a/mkdocs.yml b/mkdocs.yml index 53fba597..4356d7b8 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -119,6 +119,7 @@ nav: - Getting started (RDFLib): "getting-started.md" - Overview and supported features: "overview.md" - Generic API (no RDFLib): "generic-sink.md" + - SPARQL query results: "sparql.md" - Neo4j: "rdflib-neo4j-integration.md" - NetworkX: "networkx-integration.md" - API reference: "api.md" From ed744823638a306d1bf44ac55a8438e85f79da6e Mon Sep 17 00:00:00 2001 From: Ostrzyciel Date: Mon, 26 Oct 2026 18:03:22 +0100 Subject: [PATCH 06/24] Add benchmarks of Jelly-SPARQL A "sparql" suite of the benchmarks (--suite sparql), as in Jelly-JVM: RiverBench datasets read as the results of SELECT * queries, and the synthetic result sets of Jelly-JVM's generator (ported). Jelly-SPARQL, through rdflib and the generic API, is compared with rdflib's JSON, XML and CSV results. Throughput counts cells (rows times variables). The measurement, processes and sweeps are those of the RDF benchmarks. --- benchmarks/README.md | 39 ++- benchmarks/__main__.py | 53 +++- benchmarks/measure.py | 31 ++- benchmarks/report.py | 27 +- benchmarks/sparql_data.py | 473 +++++++++++++++++++++++++++++++++++ benchmarks/sparql_methods.py | 179 +++++++++++++ benchmarks/suites.py | 45 ++++ 7 files changed, 818 insertions(+), 29 deletions(-) create mode 100644 benchmarks/sparql_data.py create mode 100644 benchmarks/sparql_methods.py create mode 100644 benchmarks/suites.py diff --git a/benchmarks/README.md b/benchmarks/README.md index af331887..8447d195 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -1,6 +1,6 @@ # Benchmarks -Throughput and size benchmarks of Jelly-RDF in pyjelly, compared with the RDF formats of rdflib. +Throughput and size benchmarks of Jelly-RDF in pyjelly, compared with the RDF formats of rdflib, and of Jelly-SPARQL, compared with rdflib's formats of SPARQL query results (see [Jelly-SPARQL](#jelly-sparql)). The method follows the [Jelly-JVM benchmarks](https://github.com/Jelly-RDF/jelly-jvm/tree/main/jmh) as closely as Python allows, so that the results can be read the same way. @@ -109,3 +109,40 @@ Before measuring, each case writes the data, reads it back and compares the stat - Before each iteration, the garbage collector runs, outside the timing. The data prepared before measuring is frozen with `gc.freeze()`, so the collector doesn't go through it again and again during the measurement. - A sweep runs every case once. There are 3 sweeps. The result is the median of the sweeps. Values whose sweeps differ by more than 10% are marked with `*`. - Averages over datasets are geometric means. A method that couldn't run on all datasets has the number of datasets it ran on next to its mean. + +## Jelly-SPARQL + +The `sparql` suite measures SPARQL query results instead of RDF statements, as in the Jelly-SPARQL benchmarks of Jelly-JVM. Pass `--suite sparql` to `run`, `case` and `size`: + +```bash +uv run python -m benchmarks run write --suite sparql +uv run python -m benchmarks run read --suite sparql +uv run python -m benchmarks size --suite sparql +``` + +Everything else (processes, iterations, sweeps, cores, the round-trip check) is the same as for RDF. + +### Data + +- The 13 RiverBench datasets, read as the results of `SELECT * WHERE { ?s ?p ?o }` for triples and `SELECT * WHERE { GRAPH ?g { ?s ?p ?o } }` for quads, with `?g` unbound in the default graph. The first 100,000 rows, as for RDF. +- Synthetic result sets, from the presets of Jelly-JVM's generator (`SparqlDataGen`), ported in `sparql_data.py`. Each brings out one property of the format: IRIs sorted or shuffled, repeated values, unbound cells, kinds of literals, columns that mix term types, RDF 1.2 terms, and the shape of the result set (`wide-5`, `wide-20`, `zero-vars`). `realistic-mixed` looks like the results of a typical query: an entity, a property, a label, an optional value, and a value of mixed types. As in Jelly-JVM, a preset has `--rows` rows, 100,000 by default. +- By default, `run` and `size` use all RiverBench datasets and the presets `realistic-mixed`, `wide-20`, `sparse-random-50`, `lit-lang` and `iri-shuffled`. Pass any presets with `--datasets`; their names are in `PRESETS` in `sparql_data.py`. + +### Methods + +| Method | Writes with | Reads with | +|---|---|---| +| `rdflib-srj` | rdflib's SPARQL JSON results: `Result.serialize(format="json")` | `Result.parse(format="json")` | +| `rdflib-srx` | rdflib's SPARQL XML results | `Result.parse(format="xml")` | +| `rdflib-csv` | rdflib's SPARQL CSV results | `Result.parse(format="csv")` | +| `rdflib-jelly` | Jelly-SPARQL through rdflib's plugin API: `Result.serialize(format="jelly")` | `Result.parse(format="jelly")` | +| `generic-jelly` | Jelly-SPARQL with pyjelly's generic API, from a list of rows | `parse_jelly_sparql` of the generic API | + +The rdflib methods write a `Result` whose bindings are a list of dicts, made before measuring. Reading makes a whole `Result`, with its bindings. Jelly uses pyjelly's default options: lookups of 8192 names, 1024 prefixes and 64 datatypes, and frames of 4096 values. rdflib can't hold RDF 1.2 terms, so its methods skip the presets that have them. + +CSV has no term types: every value that is not an IRI or a blank node is read back as a simple literal. The round-trip check then prints a warning, as in Jelly-JVM. + +### Throughput + +Throughput counts cells, that is, rows times variables, as in Jelly-JVM. A result set without variables counts one cell per row. + diff --git a/benchmarks/__main__.py b/benchmarks/__main__.py index c05b69e1..c12cfc8e 100644 --- a/benchmarks/__main__.py +++ b/benchmarks/__main__.py @@ -15,7 +15,7 @@ from pathlib import Path from benchmarks import data, measure, report -from benchmarks.methods import METHODS +from benchmarks.suites import SUITES RESULTS_DIR = Path(__file__).parent / "results" @@ -24,18 +24,36 @@ def _names(value: str) -> list[str]: return [name.strip() for name in value.split(",") if name.strip()] +def _add_suite(parser: argparse.ArgumentParser) -> None: + parser.add_argument( + "--suite", + choices=list(SUITES), + default="rdf", + help=( + "rdf: Jelly-RDF and RDF formats, sparql: Jelly-SPARQL and SPARQL " + "result formats (default: %(default)s)" + ), + ) + + def _add_selection(parser: argparse.ArgumentParser) -> None: + _add_suite(parser) parser.add_argument( "--datasets", type=_names, - default=list(data.DATASETS), - help="comma-separated RiverBench datasets (default: all 13)", + help=( + "comma-separated datasets (default: the 13 RiverBench datasets, and in " + "the sparql suite, also a few synthetic ones)" + ), ) parser.add_argument( "--methods", type=_names, - default=list(METHODS), - help=f"comma-separated methods (default: all of {', '.join(METHODS)})", + help=( + "comma-separated methods (default: all of the suite: " + + "; ".join(f"{s.name}: {', '.join(s.methods)}" for s in SUITES.values()) + + ")" + ), ) parser.add_argument( "--rows", @@ -69,6 +87,16 @@ def _add_timing(parser: argparse.ArgumentParser) -> None: ) +def _fill_selection(args: argparse.Namespace, parser: argparse.ArgumentParser) -> None: + """Choose the datasets and methods of the suite, unless given.""" + suite = SUITES[args.suite] + args.datasets = args.datasets or list(suite.default_datasets) + args.methods = args.methods or list(suite.methods) + for name in args.methods: + if name not in suite.methods: + parser.error(f"no method {name} in the {args.suite} suite") + + def _format_cpus(cpus: list[int]) -> str: """Format CPU cores like taskset does: "5-9,15".""" ranges: list[list[int]] = [] @@ -112,10 +140,13 @@ def _case(args: argparse.Namespace, parser: argparse.ArgumentParser) -> None: if sys.platform != "linux": parser.error("--cpus works only on Linux") os.sched_setaffinity(0, args.cpus) + if args.method not in SUITES[args.suite].methods: + parser.error(f"no method {args.method} in the {args.suite} suite") result = measure.run_case( args.task, args.dataset, args.method, + suite=args.suite, rows=args.rows, warmup=args.warmup, iterations=args.iterations, @@ -133,6 +164,7 @@ def _run(args: argparse.Namespace) -> None: settings = { name: getattr(args, name) for name in ( + "suite", "task", "datasets", "methods", @@ -156,9 +188,11 @@ def _run(args: argparse.Namespace) -> None: seconds=args.time, check=args.check, cpu_sets=args.cpus, + suite=args.suite, ) stamp = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S") - out = args.out or RESULTS_DIR / f"{stamp}-{args.task}.json" + prefix = "sparql-" if args.suite == "sparql" else "" + out = args.out or RESULTS_DIR / f"{stamp}-{prefix}{args.task}.json" measure.save(out, settings, results) print(f"\nSaved to {out}\n") print(report.throughput_table(args.task, results)) @@ -196,7 +230,8 @@ def main() -> None: ) case.add_argument("task", choices=["write", "read"]) case.add_argument("dataset") - case.add_argument("method", choices=list(METHODS)) + case.add_argument("method") + _add_suite(case) case.add_argument("--rows", type=int, default=data.DEFAULT_ROWS) case.add_argument( "--cpus", type=measure.parse_cpu_list, help="CPU cores to pin the case to" @@ -215,6 +250,8 @@ def main() -> None: show.add_argument("results", type=Path, nargs="+") args = parser.parse_args() + if args.command in ("run", "size"): + _fill_selection(args, parser) if args.command == "fetch": data.fetch(args.datasets) elif args.command == "case": @@ -222,7 +259,7 @@ def main() -> None: elif args.command == "run": _run(args) elif args.command == "size": - print(report.size_tables(args.datasets, args.methods, args.rows)) + print(report.size_tables(args.datasets, args.methods, args.rows, args.suite)) elif args.command == "report": for path in args.results: content = json.loads(path.read_text()) diff --git a/benchmarks/measure.py b/benchmarks/measure.py index 6a500cd8..11cf1ddd 100644 --- a/benchmarks/measure.py +++ b/benchmarks/measure.py @@ -30,8 +30,8 @@ from typing import Any import pyjelly.parse.decode -from benchmarks.data import load -from benchmarks.methods import METHODS, NullOutput, round_trip +from benchmarks.methods import NullOutput +from benchmarks.suites import SUITES from pyjelly.options import MAX_VERSION # The folder that has the `benchmarks` package, where the cases run. With a @@ -45,9 +45,10 @@ class CaseResult: dataset: str method: str #: Statements done in one operation: every statement the writer writes, or - #: every statement the reader found in the bytes + #: every statement the reader found in the bytes. In the sparql suite, cells + #: (rows times variables). statements: int = 0 - #: Throughput of each iteration, in statements per second + #: Throughput of each iteration, in statements (or cells) per second warmup: list[float] = field(default_factory=list) measured: list[float] = field(default_factory=list) #: Mean throughput of the measured iterations, None if the case didn't run @@ -61,6 +62,8 @@ class CaseResult: #: The CPU cores the case ran on cpus: list[int] | None = None sweep: int = 0 + #: "rdf" or "sparql", see suites.py + suite: str = "rdf" def _iteration( @@ -94,6 +97,7 @@ def run_case( # noqa: PLR0913 seconds: float, check: bool = True, profile: Path | None = None, + suite: str = "rdf", ) -> CaseResult: """ Run one case in this process. See the module docstring. @@ -101,11 +105,13 @@ def run_case( # noqa: PLR0913 With `check=False`, the round-trip check is skipped. Do that only when the code under test hasn't changed since the check last passed. """ - result = CaseResult(task, dataset, method_name, checked=check) + result = CaseResult(task, dataset, method_name, checked=check, suite=suite) if hasattr(os, "sched_getaffinity"): result.cpus = sorted(os.sched_getaffinity(0)) - method = METHODS[method_name] - data = load(dataset, rows) + benchmarks = SUITES[suite] + round_trip = benchmarks.round_trip + method = benchmarks.methods[method_name] + data = benchmarks.load(dataset, rows) if not method.supports(data): result.skipped = "the format can't hold this kind of statements" return result @@ -140,7 +146,7 @@ def operation() -> None: result.statements = method.read(written, quads=data.quads) def operation() -> int: - return method.read(written, quads=data.quads) + return int(method.read(written, quads=data.quads)) # Everything made so far lives until the end. Freezing it keeps the garbage # collector from going through it again and again during the measurement. @@ -222,6 +228,7 @@ def _run_case_process( # noqa: PLR0913 seconds: float, check: bool, cpus: Sequence[int] | None, + suite: str, ) -> CaseResult: """Run one case in a new process, pinned to `cpus` if given.""" command = [ @@ -232,6 +239,7 @@ def _run_case_process( # noqa: PLR0913 task, dataset, method, + f"--suite={suite}", f"--rows={rows}", f"--warmup={warmup}", f"--iterations={iterations}", @@ -250,7 +258,7 @@ def _run_case_process( # noqa: PLR0913 # The JSON is the last line, in case a library printed something last_line = process.stdout.strip().splitlines()[-1] return CaseResult(**json.loads(last_line)) - result = CaseResult(task, dataset, method, checked=check) + result = CaseResult(task, dataset, method, checked=check, suite=suite) result.skipped = f"the process failed with code {process.returncode}" return result @@ -311,6 +319,7 @@ def run_sweeps( # noqa: PLR0913 seconds: float, check: bool = True, cpu_sets: Sequence[Sequence[int]] = (), + suite: str = "rdf", ) -> list[CaseResult]: """ Run every case `sweeps` times, each time in a new process. @@ -340,9 +349,11 @@ def run_share(sweep: int, worker: int) -> None: seconds=seconds, check=check, cpus=sets[worker], + suite=suite, ) result.sweep = sweep - score = f"{result.score:,.0f} st/s" if result.score else result.skipped + unit = SUITES[suite].unit + score = f"{result.score:,.0f} {unit}/s" if result.score else result.skipped with lock: results.append(result) progress = f"[{len(results)}/{total}] sweep {sweep + 1}" diff --git a/benchmarks/report.py b/benchmarks/report.py index 8618cfed..7cd08547 100644 --- a/benchmarks/report.py +++ b/benchmarks/report.py @@ -9,9 +9,8 @@ from collections.abc import Callable, Sequence from typing import Any -from benchmarks.data import load from benchmarks.measure import CaseResult -from benchmarks.methods import METHODS +from benchmarks.suites import SUITES # A case is marked when its sweeps differ by more than this, relative to the median SPREAD_LIMIT = 0.10 @@ -53,16 +52,18 @@ def _geometric_means( def throughput_table(task: str, results: Sequence[CaseResult]) -> str: """ - Median throughput of each case over its sweeps, in thousands of statements/s. + Median throughput of each case over its sweeps, in thousands per second. A value marked with `*` has sweeps that differ by more than 10%. """ + unit = "statements" scores: dict[tuple[str, str], list[float]] = defaultdict(list) datasets: list[str] = [] methods: list[str] = [] for result in results: if result.task != task: continue + unit = SUITES[result.suite].unit if result.dataset not in datasets: datasets.append(result.dataset) if result.method not in methods: @@ -87,7 +88,7 @@ def throughput_table(task: str, results: Sequence[CaseResult]) -> str: rows.append( ["**geometric mean**", *_geometric_means(medians, methods, datasets, 1)] ) - title = f"### {task.capitalize()} throughput (thousands of statements per second)" + title = f"### {task.capitalize()} throughput (thousands of {unit} per second)" return f"{title}\n\n{_table(['dataset', *methods], rows)}" @@ -113,24 +114,30 @@ def zstd_compress(data: bytes) -> bytes: } -def size_tables(datasets: Sequence[str], methods: Sequence[str], rows: int) -> str: +def size_tables( + datasets: Sequence[str], methods: Sequence[str], rows: int, suite: str = "rdf" +) -> str: """ - Return tables of serialized sizes in bytes per statement. + Return tables of serialized sizes in bytes per statement (or cell). Always divided by the number of statements given to the writer, also when the writer dropped duplicates. """ + benchmarks = SUITES[suite] + unit = benchmarks.unit.removesuffix("s") compressors = _compressors() sizes: dict[str, dict[tuple[str, str], float]] = defaultdict(dict) for dataset in datasets: - data = load(dataset, rows) + data = benchmarks.load(dataset, rows) + # As many as the writer is given: statements, or cells + count = data.cells if suite == "sparql" else rows for name in methods: - method = METHODS[name] + method = benchmarks.methods[name] if not method.supports(data): continue written = method.write_bytes(data) for compression, compress in compressors.items(): - size = len(compress(written)) / rows + size = len(compress(written)) / count sizes[compression][dataset, name] = size print(f"sizes: {dataset} done", file=sys.stderr) @@ -147,7 +154,7 @@ def size_tables(datasets: Sequence[str], methods: Sequence[str], rows: int) -> s table_rows.append( ["**geometric mean**", *_geometric_means(values, methods, datasets, 2)] ) - title = f"### Size, {compression} (bytes per statement)" + title = f"### Size, {compression} (bytes per {unit})" tables.append(f"{title}\n\n{_table(['dataset', *methods], table_rows)}") return "\n\n".join(tables) diff --git a/benchmarks/sparql_data.py b/benchmarks/sparql_data.py new file mode 100644 index 00000000..aeb39342 --- /dev/null +++ b/benchmarks/sparql_data.py @@ -0,0 +1,473 @@ +""" +SPARQL query results for the Jelly-SPARQL benchmarks, as in Jelly-JVM. + +A dataset is either a RiverBench dataset, read as the results of +`SELECT * WHERE { ?s ?p ?o }` (triples) or `SELECT * WHERE { GRAPH ?g { ?s ?p ?o } }` +(quads, with ?g unbound in the default graph), or a synthetic result set made from +one of the presets of Jelly-JVM's generator (SparqlDataGen). Each preset brings +out one property of the format. +""" + +from __future__ import annotations + +import random +from dataclasses import dataclass, field, replace +from functools import cached_property +from typing import Any, ClassVar + +from rdflib import BNode, Literal, URIRef, Variable +from rdflib.graph import DATASET_DEFAULT_GRAPH_ID +from rdflib.query import Result + +from benchmarks import data as rdf_data +from pyjelly.integrations.generic import generic_sink as generic + +# --- The generator, ported from Jelly-JVM's SparqlDataGen --- + +IRI = "iri" +BNODE = "bnode" +PLAIN_LITERAL = "plain" +LANG_LITERAL = "lang" +DT_LITERAL = "typed" +DIR_LANG_LITERAL = "dirlang" +TRIPLE_TERM = "triple" + +# How the unbound cells of a column are spread: each cell on its own, every +# second cell, or blocks of `block_size` cells +RANDOM = "random" +ALTERNATING = "alternating" +BLOCKS = "blocks" + +_NAMESPACE_BASE = "https://example.org/ns" +_DATATYPE_BASE = "https://example.org/dt#" +_LANGS = ("en", "fr", "de", "es", "pl", "nl", "en-GB", "pt-BR") + + +@dataclass(frozen=True) +class ColumnSpec: + """One variable of a generated result set. The defaults: dense IRIs.""" + + kind: str = IRI + #: How many distinct values the column draws from + distinct_values: int = 1024 + #: How many IRI namespaces, datatypes, or language tags + namespaces: int = 1 + #: Length of the local name of IRIs, or of the lexical form of literals + value_length: int = 12 + #: How many of the cells are bound + bound_fraction: float = 1.0 + sparsity: str = RANDOM + block_size: int = 1 + #: Mean length of a run of one value in consecutive rows + run_length: float = 1.0 + #: Whether runs have random lengths, or exactly `run_length` + jitter_runs: bool = True + #: Whether values are drawn in order (as with ORDER BY) or at random + sorted: bool = False + #: How many of the bound cells are of another term type + mix_fraction: float = 0.0 + #: The first row with a value of the other term type + mix_start_row: int = 0 + + +@dataclass(frozen=True) +class ResultSetSpec: + name: str + columns: tuple[ColumnSpec, ...] + rows: int + seed: int = 0 + + @property + def variables(self) -> list[str]: + return [f"v{i}" for i in range(len(self.columns))] + + +def _pad(base: str, length: int) -> str: + return base if len(base) >= length else base + "x" * (length - len(base)) + + +def _iri(column: ColumnSpec, j: int, name: str) -> generic.IRI: + namespace = j % max(1, column.namespaces) + return generic.IRI( + f"{_NAMESPACE_BASE}{namespace}#{_pad(name, max(1, column.value_length))}" + ) + + +def _pool(column: ColumnSpec, kind: str) -> list[Any]: + """Return the distinct values a column draws from, as generic terms.""" + n = max(1, column.distinct_values) + length = max(1, column.value_length) + if kind == IRI: + return [_iri(column, j, f"term{j}") for j in range(n)] + if kind == BNODE: + return [generic.BlankNode(_pad(f"b{j}", min(length, 24))) for j in range(n)] + if kind == TRIPLE_TERM: + return [_triple_term(column, j) for j in range(n)] + return _literal_pool(column, kind) + + +def _literal_pool(column: ColumnSpec, kind: str) -> list[Any]: + n = max(1, column.distinct_values) + ns = max(1, column.namespaces) + length = max(1, column.value_length) + if kind == PLAIN_LITERAL: + return [generic.Literal(_pad(f"value {j} ", length)) for j in range(n)] + if kind == LANG_LITERAL: + langs = _LANGS[: min(ns, len(_LANGS))] + return [ + generic.Literal(_pad(f"value {j} ", length), langs[j % len(langs)]) + for j in range(n) + ] + if kind == DT_LITERAL: + return [ + generic.Literal(_pad(f"{j}", length), None, f"{_DATATYPE_BASE}d{j % ns}") + for j in range(n) + ] + assert kind == DIR_LANG_LITERAL + # Each "namespace" is one combination of a language tag and a direction + return [ + generic.Literal( + _pad(f"value {j} ", length), + _LANGS[(j % ns) // 2 % len(_LANGS)], + None, + "ltr" if j % ns % 2 == 0 else "rtl", + ) + for j in range(n) + ] + + +def _triple_term(column: ColumnSpec, j: int) -> generic.Triple: + """Return a triple term with fresh IRIs, some nested.""" + o: Any + if j % 3 == 0: + o = _iri(column, j, f"o{j}") + elif j % 3 == 1: + o = generic.Literal(f"value {j}", "en", None, "ltr" if j % 2 else "rtl") + else: + datatype = f"{_DATATYPE_BASE}d{j % max(1, column.namespaces)}" + o = generic.Triple( + generic.BlankNode(f"b{j}"), + _iri(column, j, f"q{j}"), + generic.Literal(f"{j}", None, datatype), + ) + return generic.Triple(_iri(column, j, f"s{j}"), _iri(column, j, f"p{j % 16}"), o) + + +def _generate_column(column: ColumnSpec, rows: int, rnd: random.Random) -> list[Any]: + pool = _pool(column, column.kind) + other_kind = PLAIN_LITERAL if column.kind == IRI else IRI + other_pool = _pool(column, other_kind) if column.mix_fraction > 0 else [] + cursor = 0 + + def next_value(row: int) -> Any: + nonlocal cursor + use_other = ( + other_pool + and row >= column.mix_start_row + and rnd.random() < column.mix_fraction + ) + values = other_pool if use_other else pool + if column.sorted: + value = values[cursor % len(values)] + cursor += 1 + return value + return values[rnd.randrange(len(values))] + + def run_length() -> int: + if column.run_length <= 1: + return 1 + if not column.jitter_runs: + return max(1, round(column.run_length)) + # Geometric, with the mean run_length + p = 1 / column.run_length + n = 1 + while rnd.random() > p and n < 4096: # noqa: PLR2004 + n += 1 + return n + + cells: list[Any] = [] + while len(cells) < rows: + row = len(cells) + if column.sparsity == ALTERNATING: + cells.append(None if row % 2 else next_value(row)) + elif rnd.random() >= column.bound_fraction: + block = max(1, column.block_size) if column.sparsity == BLOCKS else 1 + cells.extend([None] * min(block, rows - len(cells))) + else: + value = next_value(row) + cells.extend([value] * min(run_length(), rows - len(cells))) + return cells + + +def generate(spec: ResultSetSpec) -> list[tuple[Any, ...]]: + """Return the rows of a result set, with generic terms and None if unbound.""" + if not spec.columns: + return [()] * spec.rows + columns = [ + _generate_column(column, spec.rows, random.Random(spec.seed * 31 + i)) # noqa: S311 + for i, column in enumerate(spec.columns) + ] + return list(zip(*columns, strict=True)) + + +_N = 20_000 + +PRESETS: dict[str, ResultSetSpec] = { + spec.name: spec + for spec in ( + # IRIs: the lookups, and the prefix and name inference + ResultSetSpec("iri-sorted", (ColumnSpec(distinct_values=_N, sorted=True),), _N), + ResultSetSpec("iri-shuffled", (ColumnSpec(distinct_values=_N),), _N), + ResultSetSpec("iri-small-pool", (ColumnSpec(distinct_values=2048),), _N), + ResultSetSpec( + "iri-many-ns", (ColumnSpec(distinct_values=2048, namespaces=32),), _N + ), + # Repeated values: the runs of the layouts + ResultSetSpec( + "runs-2", + (ColumnSpec(distinct_values=2048, run_length=2, sorted=True),), + _N, + ), + ResultSetSpec( + "runs-8", + (ColumnSpec(distinct_values=2048, run_length=8, sorted=True),), + _N, + ), + ResultSetSpec( + "alternating-values", (ColumnSpec(distinct_values=2, sorted=True),), _N + ), + # Unbound cells + ResultSetSpec( + "sparse-random-10", + (ColumnSpec(distinct_values=2048, bound_fraction=0.1),), + _N, + ), + ResultSetSpec( + "sparse-random-50", + (ColumnSpec(distinct_values=2048, bound_fraction=0.5),), + _N, + ), + ResultSetSpec( + "sparse-random-90", + (ColumnSpec(distinct_values=2048, bound_fraction=0.9),), + _N, + ), + ResultSetSpec( + "sparse-blocks", + ( + ColumnSpec( + distinct_values=2048, + bound_fraction=0.5, + sparsity=BLOCKS, + block_size=16, + ), + ), + _N, + ), + ResultSetSpec( + "sparse-alternating", + (ColumnSpec(distinct_values=2048, sparsity=ALTERNATING),), + _N, + ), + # Literals + ResultSetSpec( + "lit-plain", (ColumnSpec(PLAIN_LITERAL, distinct_values=2048),), _N + ), + ResultSetSpec( + "lit-lang", + (ColumnSpec(LANG_LITERAL, distinct_values=2048, namespaces=8),), + _N, + ), + ResultSetSpec( + "lit-lang-one", + (ColumnSpec(LANG_LITERAL, distinct_values=2048, namespaces=1),), + _N, + ), + ResultSetSpec( + "lit-typed", + (ColumnSpec(DT_LITERAL, distinct_values=2048, namespaces=8),), + _N, + ), + ResultSetSpec( + "lit-long", + (ColumnSpec(PLAIN_LITERAL, distinct_values=512, value_length=4096),), + 2_000, + ), + ResultSetSpec("bnodes", (ColumnSpec(BNODE, distinct_values=2048),), _N), + # Columns that mix term types + ResultSetSpec( + "poly-half", (ColumnSpec(distinct_values=2048, mix_fraction=0.5),), _N + ), + ResultSetSpec( + "poly-late-switch", + ( + ColumnSpec( + distinct_values=2048, mix_fraction=0.0005, mix_start_row=5_000 + ), + ), + _N, + ), + # RDF 1.2 terms + ResultSetSpec( + "lit-dirlang", + (ColumnSpec(DIR_LANG_LITERAL, distinct_values=2048, namespaces=4),), + _N, + ), + ResultSetSpec( + "triple-terms", + ( + ColumnSpec(TRIPLE_TERM, distinct_values=_N, namespaces=4), + ColumnSpec(distinct_values=2048), + ), + _N, + ), + # The shape of the result set + ResultSetSpec("wide-5", (ColumnSpec(distinct_values=2048),) * 5, _N), + ResultSetSpec("wide-20", (ColumnSpec(distinct_values=2048),) * 20, _N // 2), + ResultSetSpec("single-row", (ColumnSpec(distinct_values=8),) * 3, 1), + ResultSetSpec("zero-vars", (), _N), + # Entity, property, label, optional value, and a value of mixed types + ResultSetSpec( + "realistic-mixed", + ( + ColumnSpec( + distinct_values=4096, run_length=3, sorted=True, value_length=8 + ), + ColumnSpec(distinct_values=32, value_length=10), + ColumnSpec( + LANG_LITERAL, distinct_values=4096, namespaces=4, value_length=24 + ), + ColumnSpec(PLAIN_LITERAL, distinct_values=1024, bound_fraction=0.4), + ColumnSpec(distinct_values=2048, mix_fraction=0.3), + ), + _N, + ), + ) +} + +#: The datasets of a run, unless chosen: all RiverBench datasets, and the presets +#: that look most like the results of real queries +DEFAULT_DATASETS = ( + *rdf_data.DATASETS, + "realistic-mixed", + "wide-20", + "sparse-random-50", + "lit-lang", + "iri-shuffled", +) + +DATASETS = (*rdf_data.DATASETS, *PRESETS) + + +def _resized(spec: ResultSetSpec, rows: int) -> ResultSetSpec: + """Change the row count of a preset, but not its shape.""" + factor = rows / spec.rows + return replace( + spec, + rows=rows, + columns=tuple( + replace(column, mix_start_row=int(column.mix_start_row * factor)) + for column in spec.columns + ), + ) + + +# --- The results, in the terms of each library --- + + +def _rdflib_term(term: Any) -> Any: + if term is None: + return None + if isinstance(term, generic.IRI): + return URIRef(term._iri) + if isinstance(term, generic.BlankNode): + return BNode(term._id) + if isinstance(term, generic.Literal): + if term._direction: + msg = "rdflib has no literals with a base direction" + raise TypeError(msg) + # As pyjelly reads them: the lexical form stays as it is + return Literal( + term._lex, lang=term._langtag, datatype=term._datatype, normalize=False + ) + msg = "rdflib has no triple terms" + raise TypeError(msg) + + +@dataclass +class SparqlBenchData: + """ + A result set: its variables and rows, with generic terms and rdflib terms. + + `cells` counts what the throughput is measured in: rows times variables, and + rows for a result set without variables. + """ + + #: As in the data of RDF, which the benchmark code also reads + quads: ClassVar[bool] = False + + name: str + variables: list[str] + generic: list[tuple[Any, ...]] = field(repr=False) + #: The rows with rdflib terms, if not made from the generic ones + rdflib_rows: list[tuple[Any, ...]] | None = field(default=None, repr=False) + + @property + def cells(self) -> int: + return len(self.generic) * max(1, len(self.variables)) + + @cached_property + def rdf_12(self) -> bool: + """Whether the results have terms of RDF 1.2, which rdflib has not.""" + return any( + isinstance(term, generic.Triple) + or (isinstance(term, generic.Literal) and term._direction) + for row in self.generic + for term in row + ) + + @cached_property + def rdflib(self) -> Result: + """The results as an rdflib Result, with the bindings in a list.""" + result = Result("SELECT") + variables = [Variable(name) for name in self.variables] + result.vars = variables + rows = self.rdflib_rows + if rows is None: + rows = [tuple(map(_rdflib_term, row)) for row in self.generic] + result.bindings = [ + { + variable: term + for variable, term in zip(variables, row, strict=True) + if term is not None + } + for row in rows + ] + return result + + +def load(dataset: str, rows: int = rdf_data.DEFAULT_ROWS) -> SparqlBenchData: + """Return the first `rows` rows of a dataset.""" + if dataset in PRESETS: + spec = _resized(PRESETS[dataset], rows) + return SparqlBenchData(dataset, spec.variables, generate(spec)) + statements = rdf_data.load(dataset, rows) + if not statements.quads: + return SparqlBenchData( + dataset, + ["s", "p", "o"], + [tuple(statement) for statement in statements.generic], + [tuple(statement) for statement in statements.rdflib], + ) + return SparqlBenchData( + dataset, + ["s", "p", "o", "g"], + [ + (s, p, o, None if g is generic.DefaultGraph else g) + for s, p, o, g in statements.generic + ], + [ + (s, p, o, None if g == DATASET_DEFAULT_GRAPH_ID else g) + for s, p, o, g in statements.rdflib + ], + ) diff --git a/benchmarks/sparql_methods.py b/benchmarks/sparql_methods.py new file mode 100644 index 00000000..40acc67e --- /dev/null +++ b/benchmarks/sparql_methods.py @@ -0,0 +1,179 @@ +""" +Ways of writing and reading SPARQL query results, each through one library's API. + +Each method writes the rows of a result set to a stream, and reads them back from +bytes in memory. The throughput counts cells: rows times variables. +""" + +from __future__ import annotations + +import io +from abc import ABC, abstractmethod +from collections import Counter +from typing import IO, Any +from typing_extensions import override + +from rdflib import BNode, Literal, URIRef +from rdflib.query import Result + +from benchmarks.sparql_data import SparqlBenchData +from pyjelly.integrations.generic import generic_sink +from pyjelly.integrations.generic import sparql as generic_sparql + + +class SparqlMethod(ABC): + name: str + + def supports(self, data: SparqlBenchData) -> bool: # noqa: ARG002 + return True + + @abstractmethod + def rows(self, data: SparqlBenchData) -> list[tuple[Any, ...]]: + """Return the rows in this method's term model, None if unbound.""" + + def prepare(self, data: SparqlBenchData) -> None: + """Make everything the writer starts from, so that it isn't measured.""" + self.rows(data) + + def write_count(self, data: SparqlBenchData) -> int: + """Return how many cells `write` writes.""" + return data.cells + + @abstractmethod + def write(self, data: SparqlBenchData, out: IO[bytes]) -> None: + """Write the result set to `out`.""" + + @abstractmethod + def read(self, data: bytes, *, quads: bool = False) -> int: + """ + Read the rows, do nothing with them, and return how many cells there were. + + `quads` is not used: it is there to read like the methods of RDF. + """ + + @abstractmethod + def read_all(self, data: bytes) -> list[tuple[Any, ...]]: + """Read the rows and return them, for checking.""" + + def write_bytes(self, data: SparqlBenchData) -> bytes: + out = io.BytesIO() + self.write(data, out) + return out.getvalue() + + +def _cells(rows: int, variables: int) -> int: + return rows * max(1, variables) + + +class RdflibResults(SparqlMethod): + """A result format of rdflib, written and read with Result.serialize and parse.""" + + def __init__(self, name: str, rdflib_format: str) -> None: + self.name = name + self.format = rdflib_format + + @override + def supports(self, data: SparqlBenchData) -> bool: + return not data.rdf_12 + + @override + def rows(self, data: SparqlBenchData) -> list[tuple[Any, ...]]: + result = data.rdflib + variables = result.vars or [] + return [tuple(b.get(v) for v in variables) for b in result.bindings] + + @override + def write(self, data: SparqlBenchData, out: IO[bytes]) -> None: + data.rdflib.serialize(destination=out, format=self.format) + + def _parse(self, data: bytes) -> Result: + return Result.parse(io.BytesIO(data), format=self.format) + + @override + def read(self, data: bytes, *, quads: bool = False) -> int: + result = self._parse(data) + return _cells(len(result.bindings), len(result.vars or [])) + + @override + def read_all(self, data: bytes) -> list[tuple[Any, ...]]: + result = self._parse(data) + variables = result.vars or [] + return [tuple(b.get(v) for v in variables) for b in result.bindings] + + +class GenericJelly(SparqlMethod): + """Jelly-SPARQL with pyjelly's generic API, with its own terms.""" + + name = "generic-jelly" + + @override + def rows(self, data: SparqlBenchData) -> list[tuple[Any, ...]]: + return data.generic + + @override + def write(self, data: SparqlBenchData, out: IO[bytes]) -> None: + generic_sparql.serialize_jelly_sparql(data.variables, data.generic, out) + + @override + def read(self, data: bytes, *, quads: bool = False) -> int: + results = generic_sparql.parse_jelly_sparql(io.BytesIO(data)) + rows = 0 + for _ in results: + rows += 1 + return _cells(rows, len(results.variables)) + + @override + def read_all(self, data: bytes) -> list[tuple[Any, ...]]: + return list(generic_sparql.parse_jelly_sparql(io.BytesIO(data))) + + +SPARQL_METHODS: dict[str, SparqlMethod] = { + method.name: method + for method in ( + RdflibResults("rdflib-srj", "json"), + RdflibResults("rdflib-srx", "xml"), + RdflibResults("rdflib-csv", "csv"), + # Jelly through rdflib's plugin API: Result.serialize and Result.parse + RdflibResults("rdflib-jelly", "jelly"), + GenericJelly(), + ) +} + + +def _normalize(term: Any) -> Any: + """Make terms comparable. Blank node labels are not compared.""" + if term is None: + return None + if isinstance(term, (BNode, generic_sink.BlankNode)): + return "_:" + if isinstance(term, (URIRef, Literal)): + return term.n3() + return str(term) + + +def sparql_round_trip( + method: SparqlMethod, data: SparqlBenchData +) -> tuple[bytes, list[str]]: + """ + Write the result set with the method, read it back, and compare the rows. + + Returns the written bytes, and a list of problems found (empty if none). + Raises the reader's exception if the method can't read its own output. + """ + written = method.write_bytes(data) + expected = [tuple(map(_normalize, row)) for row in method.rows(data)] + actual = [tuple(map(_normalize, row)) for row in method.read_all(written)] + problems = [] + if method.read(written) != method.write_count(data): + problems.append("read() counts a different number of cells than written") + missing = Counter(expected) - Counter(actual) + extra = Counter(actual) - Counter(expected) + if missing or extra: + problems.append( + f"{sum(missing.values())} rows missing, {sum(extra.values())} unexpected; " + f"e.g. missing {next(iter(missing), None)}, " + f"unexpected {next(iter(extra), None)}" + ) + elif expected != actual: + problems.append("the rows are in another order") + return written, problems diff --git a/benchmarks/suites.py b/benchmarks/suites.py new file mode 100644 index 00000000..f0e5bf2f --- /dev/null +++ b/benchmarks/suites.py @@ -0,0 +1,45 @@ +"""The suites of benchmarks: Jelly-RDF and Jelly-SPARQL, each with other formats.""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from typing import Any + +from benchmarks import data, sparql_data +from benchmarks.methods import METHODS, round_trip +from benchmarks.sparql_methods import SPARQL_METHODS, sparql_round_trip + + +@dataclass(frozen=True) +class Suite: + name: str + #: What the throughput counts + unit: str + datasets: tuple[str, ...] + default_datasets: tuple[str, ...] + methods: Mapping[str, Any] + load: Callable[[str, int], Any] + round_trip: Callable[[Any, Any], tuple[bytes, list[str]]] + + +SUITES = { + "rdf": Suite( + "rdf", + "statements", + data.DATASETS, + data.DATASETS, + METHODS, + data.load, + round_trip, + ), + "sparql": Suite( + "sparql", + "cells", + sparql_data.DATASETS, + sparql_data.DEFAULT_DATASETS, + SPARQL_METHODS, + sparql_data.load, + sparql_round_trip, + ), +} From 4320ad2047c68f9151e029c8543106348389e3c6 Mon Sep 17 00:00:00 2001 From: Ostrzyciel Date: Mon, 26 Oct 2026 18:13:17 +0100 Subject: [PATCH 07/24] Fix what a review of the Jelly-SPARQL API found Bugs: - Results are read and written as they come over a connection: a buffered input (e.g. from socket.makefile()) is no longer wrapped in a second buffer, which waited for 8 KiB, and each frame is flushed. - Writing an rdflib result that was partly iterated wrote only the rest. - Bindings keyed by plain strings were written as unbound. - An incomplete boolean result read with the generic API looked complete. - Lazy rdflib results had no links until iterated. - Results read from an incomplete or truncated stream were written again as complete. API changes: - parse/serialize_jelly_sparql_results are now ..._sequence, and the generic API can write sequences too. In a sequence, an incomplete rdflib result sets its `error` instead of ending the sequence. - A truncated stream issues JellyTruncatedResultsWarning, and a frame cut short or input that isn't Jelly-SPARQL raises a clear error. - SparqlOptions.frame_size is now frame_values, as it counts values. - Unknown keyword arguments of Result.parse/serialize raise TypeError. - Rows can be iterated once, an iteration stopped early can go on, and reading from a closed input says so. Bytes are accepted as input. - Trailers name the type of the error that stopped the results. --- docs/sparql.md | 35 ++-- pyjelly/errors.py | 9 + pyjelly/integrations/generic/sparql.py | 66 +++++-- pyjelly/integrations/rdflib/sparql.py | 156 ++++++++++++----- pyjelly/options.py | 23 +-- pyjelly/parse/ioutils.py | 11 +- pyjelly/parse/sparql.py | 144 ++++++++++++---- pyjelly/serialize/sparql.py | 49 +++++- .../test_sparql/test_parse.py | 23 ++- .../test_sparql/test_serialize.py | 8 +- .../test_generic/test_sparql.py | 79 ++++++++- .../test_rdflib/test_sparql.py | 162 ++++++++++++++++-- .../unit_tests/test_serialize/test_sparql.py | 6 +- 13 files changed, 612 insertions(+), 159 deletions(-) diff --git a/docs/sparql.md b/docs/sparql.md index 66d07e40..b55c05a7 100644 --- a/docs/sparql.md +++ b/docs/sparql.md @@ -10,10 +10,11 @@ With pyjelly installed, RDFLib reads and writes query results in the `jelly` for {{ code_example('rdflib/09_sparql_results.py') }} -`Result.parse` returns a `JellySparqlResult`, which is an RDFLib `Result` with two more attributes: +`Result.parse` returns a `JellySparqlResult` (from `pyjelly.integrations.rdflib.sparql`), which is an RDFLib `Result` with more attributes. RDFLib's types say that `Result.parse` returns a `Result`, so check `isinstance(result, JellySparqlResult)` to use them: - `links` – the IRIs of links that describe the results, like `head.link` in [SPARQL JSON results](https://www.w3.org/TR/sparql12-results-json/). They are written too, if the result you write has them. -- `truncated` – `True` if the stream ended without saying that the results are complete, so some may be missing (e.g., the server closed the connection). +- `truncated` – `True` if the stream ended without saying that the results are complete, so some may be missing. pyjelly also issues a `JellyTruncatedResultsWarning` then. A stream cut in the middle of a frame can't be read at all, and raises `JellyConformanceError`. +- `error` – why the results are incomplete, in a [sequence of results](#sequences-of-results), see [incomplete results](#incomplete-results). RDFLib does not support RDF 1.2 yet, so triple terms and literals with a base direction can't be read into RDFLib. Use the [generic API](#generic-api) for them. @@ -33,17 +34,21 @@ for row in Graph(store=store).query("SELECT * WHERE { ?s ?p ?o } LIMIT 10"): print(row) ``` +!!! warning + + Use the media type, `application/x-jelly-sparql`, as `returnFormat`, not `jelly`. With `jelly`, RDFLib doesn't ask for any format, so the endpoint answers in its default one. + ### Large results By default, `Result.parse` reads all rows at once. With `lazy=True`, the rows are read as the result is iterated, like RDFLib does with the results of its own queries. The stream must then stay open until you are done with the result. -When writing, the rows of an RDFLib result are written as the query makes them, so the first frames are written before the query is done. +When writing, the rows of an RDFLib result are written as the query makes them, so the first frames are written before the query is done. Each frame is flushed when it's written, so a reader at the other end of a connection gets it right away. {{ code_example('rdflib/10_sparql_results_lazy.py') }} !!! note - Iterate a lazy result with a `for` loop. `list(result)` and `len(result)` read all of its rows first, as for any RDFLib result. + Iterate a lazy result with a `for` loop. `list(result)`, `len(result)` and `bool(result)` read all of its rows first, as for any RDFLib result. ### Incomplete results @@ -61,6 +66,8 @@ except JellyIncompleteResultsError as error: partial_result = error.result ``` +If you write results that you read from Jelly-SPARQL, and they were incomplete or truncated, pyjelly writes them as incomplete too. + ### Options To change how the results are written, pass `SparqlOptions` to `serialize`: @@ -72,14 +79,14 @@ options = SparqlOptions( # Sizes of the lookups, see below lookup_preset=LookupPreset(max_names=4096, max_prefixes=256, max_datatypes=32), # How many values (rows times variables) a frame has, at most - frame_size=4096, + frame_values=4096, ) results.serialize(destination="results.jellys", format="jelly", options=options) ``` The default lookup sizes are larger than for Jelly-RDF (8192 names, 1024 prefixes and 64 datatypes), as in Jelly-JVM: a row of results tends to have more distinct terms than an RDF statement. Readers accept lookups of up to 16384 names, 4096 prefixes and 256 datatypes. -With `delimited=False`, the results are written as one frame, which some transports (e.g., a message broker) need. Then the lookups must be large enough to hold all terms of the results. +With `delimited=False`, the results are written as one frame without its size, e.g., to be one message of a message broker. Then the lookups must be large enough to hold all terms of the results. Files and HTTP responses must be delimited, which is the default. To send the frames of the results as separate messages, see [writing frame by frame](#writing-frame-by-frame). ### Sequences of results @@ -87,19 +94,21 @@ A Jelly-SPARQL stream can also hold a sequence of results, e.g., the answers to ```python from pyjelly.integrations.rdflib.sparql import ( - parse_jelly_sparql_results, - serialize_jelly_sparql_results, + parse_jelly_sparql_sequence, + serialize_jelly_sparql_sequence, ) with open("answers.jellys", "wb") as file: - serialize_jelly_sparql_results([result_1, result_2, ask_result], file) + serialize_jelly_sparql_sequence([result_1, result_2, ask_result], file) with open("answers.jellys", "rb") as file: - for result in parse_jelly_sparql_results(file): + for result in parse_jelly_sparql_sequence(file): print(result.type, len(result)) ``` -The lookups are shared by the results of the stream, so the terms they have in common are written once. +The lookups are shared by the results of the stream, so the terms they have in common are written once. Blank nodes are not: each result has its own. + +An incomplete result doesn't end a sequence, as there may be more results after it. `parse_jelly_sparql_sequence` doesn't raise an error for it, but sets its `error` attribute, so check it. ## Generic API @@ -107,7 +116,9 @@ The [generic API](generic-sink.md) reads and writes results with its own terms ( {{ code_example('generic/10_sparql_results.py') }} -`parse_jelly_sparql` returns a `SparqlResultSet`, with the `variables`, the `boolean` result of an `ASK` query (`None` for a `SELECT` query), the `links`, and whether it is `truncated`. Its rows are read as you iterate it, so it can be iterated only once. `serialize_jelly_sparql_ask` writes the result of an `ASK` query. +`parse_jelly_sparql` returns a `SparqlResultSet`, with the `variables`, the `boolean` result of an `ASK` query (`None` for a `SELECT` query, see also `is_boolean`), the `links`, and whether it is `truncated`. Its rows are read as you iterate it, so the input must stay open until then, and the rows can be iterated only once. If the writer said that the results are incomplete, iterating the rows raises `JellyIncompleteResultsError` after the last one, and its `error` attribute says why. For an `ASK` query, `parse_jelly_sparql` raises it right away. + +`serialize_jelly_sparql_ask` writes the result of an `ASK` query. `parse_jelly_sparql_sequence` and `serialize_jelly_sparql_sequence` read and write [sequences of results](#sequences-of-results): each result written is a pair of the variables and the rows, or a boolean result. ### Writing frame by frame diff --git a/pyjelly/errors.py b/pyjelly/errors.py index 1c924326..6de2a2df 100644 --- a/pyjelly/errors.py +++ b/pyjelly/errors.py @@ -23,3 +23,12 @@ def __init__(self, message: str, error: str, result: object = None) -> None: self.error = error #: The results that were read, if the reader keeps them self.result = result + + +class JellyTruncatedResultsWarning(UserWarning): + """ + Issued when a stream of SPARQL results ends without saying they are complete. + + The writer may have stopped before writing all results, e.g., because the + connection was lost. + """ diff --git a/pyjelly/integrations/generic/sparql.py b/pyjelly/integrations/generic/sparql.py index a72a6549..12e142a8 100644 --- a/pyjelly/integrations/generic/sparql.py +++ b/pyjelly/integrations/generic/sparql.py @@ -18,34 +18,40 @@ from pyjelly.serialize.sparql import FrameWriter, SparqlStream, write_solutions -def parse_jelly_sparql(inp: IO[bytes]) -> SparqlResultSet: +def parse_jelly_sparql(inp: IO[bytes] | bytes) -> SparqlResultSet: """ Read SPARQL query results from Jelly-SPARQL. - The rows are read as they are iterated, so the input must stay open until then. + The rows are read as they are iterated (once), so the input must stay open + until then. Args: - inp (IO[bytes]): the stream, delimited or not. It must have one result set - (the FLAT stream type, the default). + inp (IO[bytes] | bytes): the stream, delimited or not. It must have one + result set (the FLAT stream type, the default). Returns: SparqlResultSet: the results. Iterating it gives the rows, as tuples with one value per variable, None if the variable is unbound. + Raises: + JellyIncompleteResultsError: if the writer said that the results are + incomplete: when the rows were iterated, or now for a boolean result. + """ return read_result_set(inp, GenericStatementSinkAdapter) -def parse_jelly_sparql_results(inp: IO[bytes]) -> Iterator[SparqlResultSet]: +def parse_jelly_sparql_sequence(inp: IO[bytes] | bytes) -> Iterator[SparqlResultSet]: """ - Read a sequence of SPARQL query results from Jelly-SPARQL. + Read a sequence of SPARQL query results from one Jelly-SPARQL stream. - A stream of the PUNCTUATED type has any number of result sets, one of the FLAT - type has one. The rows are read as they are iterated. Rows not yet iterated when - the next result set is read are kept in memory. + The stream has the PUNCTUATED type, which holds any number of results (a + stream of the FLAT type, the default, has one). The rows are read as they are + iterated. Rows not yet iterated when the next result set is read are kept in + memory. Args: - inp (IO[bytes]): the stream, delimited. + inp (IO[bytes] | bytes): the stream, delimited. Yields: SparqlResultSet: the results, one per result set. @@ -88,7 +94,7 @@ def serialize_jelly_sparql( """ stream = sparql_stream(options) writer = FrameWriter(out, delimited=stream.options.delimited) - write_solutions(stream, variables, rows, writer, links) + write_solutions(stream, variables, rows, writer, links=links) def serialize_jelly_sparql_ask( @@ -112,3 +118,41 @@ def serialize_jelly_sparql_ask( FrameWriter(out, delimited=stream.options.delimited).write( stream.ask(value, links=links) ) + + +#: A result in a sequence of results: the variables and the rows of a solution +#: sequence, or a boolean result +SequenceItem = tuple[Iterable[str], Iterable[Sequence[Node | None]]] | bool + + +def serialize_jelly_sparql_sequence( + results: Iterable[SequenceItem], + out: IO[bytes], + options: SparqlOptions | None = None, +) -> None: + """ + Write a sequence of SPARQL query results in one Jelly-SPARQL stream. + + The stream has the PUNCTUATED type: each result is a result set of its own. + + Args: + results (Iterable[SequenceItem]): the results: for a SELECT query, a pair + of the variables and the rows, and for an ASK query, its boolean result. + out (IO[bytes]): where to write them. + options (SparqlOptions | None): options of the stream, which must be + punctuated. Defaults to SparqlOptions(punctuated=True). + + """ + if options is None: + options = SparqlOptions(punctuated=True) + if not options.punctuated: + msg = "a sequence of results needs SparqlOptions(punctuated=True)" + raise ValueError(msg) + stream = sparql_stream(options) + writer = FrameWriter(out, delimited=True) + for result in results: + if isinstance(result, bool): + writer.write(stream.ask(result)) + else: + variables, rows = result + write_solutions(stream, variables, rows, writer) diff --git a/pyjelly/integrations/rdflib/sparql.py b/pyjelly/integrations/rdflib/sparql.py index 3aa3edee..9eb74101 100644 --- a/pyjelly/integrations/rdflib/sparql.py +++ b/pyjelly/integrations/rdflib/sparql.py @@ -10,9 +10,9 @@ from __future__ import annotations -from collections.abc import Iterable, Iterator, Mapping +from collections.abc import Iterable, Iterator, Mapping, Sequence from pathlib import Path -from typing import IO, Any +from typing import IO, Any, cast from typing_extensions import override from rdflib.query import Result, ResultParser, ResultSerializer @@ -31,6 +31,12 @@ ) from pyjelly.serialize.sparql import FrameWriter, SparqlStream, write_solutions +#: The error that a result read from a truncated stream is passed on with +TRUNCATED_ERROR = ( + "the results were read from a stream that ended without saying that they are " + "complete, so some may be missing" +) + class JellySparqlResult(Result): """ @@ -41,6 +47,9 @@ class JellySparqlResult(Result): JSON results. truncated: whether the stream ended without saying that the results are complete, so some may be missing. + error: why the results are incomplete, if the writer said so. Only set by + parse_jelly_sparql_sequence(): parse_jelly_sparql() raises + JellyIncompleteResultsError instead. """ @@ -48,6 +57,7 @@ def __init__(self, type_: str) -> None: super().__init__(type_) self.links: list[str] = [] self.truncated = False + self.error = "" def _adapter_factory(*, validate_iris: bool) -> AdapterFactory: @@ -57,16 +67,31 @@ def make(options: ParserOptions) -> Adapter: return make -def _to_result(result_set: SparqlResultSet, *, lazy: bool) -> JellySparqlResult: +def _to_result( + result_set: SparqlResultSet, *, lazy: bool, raise_error: bool +) -> JellySparqlResult: """Return a result set as an rdflib Result.""" - if result_set.boolean is not None: + if result_set.is_boolean: result = JellySparqlResult("ASK") - result.askAnswer = result_set.boolean - else: - result = JellySparqlResult("SELECT") - result.vars = [Variable(name) for name in result_set.variables] - rows = _read_bindings(result_set, result) - if lazy and result_set.boolean is None: + # The rest of the stream, which must not have more of the result + result_set.keep() + result.links = result_set.links + try: + result.askAnswer = result_set.boolean + except JellyIncompleteResultsError as error: + if raise_error: + error.result = result + raise + # The value read, which may be wrong + result.askAnswer = result_set._boolean + result.error = error.error + result.truncated = result_set.truncated + return result + result = JellySparqlResult("SELECT") + result.vars = [Variable(name) for name in result_set.variables] + result.links = result_set.links + rows = _read_bindings(result_set, result, raise_error=raise_error) + if lazy: # rdflib makes the bindings as the result is iterated, as for its own # query results result.bindings = rows @@ -79,7 +104,7 @@ def _to_result(result_set: SparqlResultSet, *, lazy: bool) -> JellySparqlResult: def _read_bindings( - result_set: SparqlResultSet, result: JellySparqlResult + result_set: SparqlResultSet, result: JellySparqlResult, *, raise_error: bool ) -> Iterator[Mapping[Variable, Identifier]]: """Yield the bindings of the rows, and when they end, what the stream said.""" variables = result.vars or [] @@ -92,22 +117,24 @@ def _read_bindings( if value is not None } except JellyIncompleteResultsError as error: - error.result = result - raise + result.error = error.error + if raise_error: + error.result = result + raise finally: result.links = result_set.links result.truncated = result_set.truncated def parse_jelly_sparql( - inp: IO[bytes], *, validate_iris: bool = False, lazy: bool = False + inp: IO[bytes] | bytes, *, validate_iris: bool = False, lazy: bool = False ) -> JellySparqlResult: """ Read SPARQL query results from Jelly-SPARQL. Args: - inp (IO[bytes]): the stream, delimited or not. It must have one result set - (the FLAT stream type, the default). + inp (IO[bytes] | bytes): the stream, delimited or not. It must have one + result set (the FLAT stream type, the default). validate_iris (bool): whether rdflib checks the IRIs. An invalid IRI is still read, rdflib only logs a warning. Checking slows down parsing. lazy (bool): whether to read the rows as the result is iterated, rather @@ -118,40 +145,46 @@ def parse_jelly_sparql( JellySparqlResult: the results, an rdflib Result. Raises: - JellyIncompleteResultsError: if the producer said that the results are + JellyIncompleteResultsError: if the writer said that the results are incomplete, e.g., because the query failed midway. The results read are in its `result`. """ adapter_factory = _adapter_factory(validate_iris=validate_iris) - return _to_result(read_result_set(inp, adapter_factory), lazy=lazy) + try: + result_set = read_result_set(inp, adapter_factory) + except JellyIncompleteResultsError as error: + # A boolean result that is incomplete + result_set = cast("SparqlResultSet", error.result) + error.result = _to_result(result_set, lazy=False, raise_error=False) + raise + return _to_result(result_set, lazy=lazy, raise_error=True) -def parse_jelly_sparql_results( - inp: IO[bytes], *, validate_iris: bool = False +def parse_jelly_sparql_sequence( + inp: IO[bytes] | bytes, *, validate_iris: bool = False ) -> Iterator[JellySparqlResult]: """ - Read a sequence of SPARQL query results from Jelly-SPARQL. + Read a sequence of SPARQL query results from one Jelly-SPARQL stream. + + The stream has the PUNCTUATED type, which holds any number of results (a + stream of the FLAT type, the default, has one). - A stream of the PUNCTUATED type has any number of result sets, one of the FLAT - type has one. + An incomplete result does not end the sequence: it has the reason in its + `error` attribute, so check it, and the results after it are read too. Args: - inp (IO[bytes]): the stream, delimited. + inp (IO[bytes] | bytes): the stream, delimited. validate_iris (bool): whether rdflib checks the IRIs, see parse_jelly_sparql(). Yields: JellySparqlResult: the results, an rdflib Result for each result set. - Raises: - JellyIncompleteResultsError: if the producer said that a result set is - incomplete. The results after it can still be read. - """ adapter_factory = _adapter_factory(validate_iris=validate_iris) for result_set in read_result_sets(inp, adapter_factory): - yield _to_result(result_set, lazy=False) + yield _to_result(result_set, lazy=False, raise_error=False) def sparql_stream(options: SparqlOptions | None = None) -> SparqlStream: @@ -162,11 +195,25 @@ def sparql_stream(options: SparqlOptions | None = None) -> SparqlStream: return SparqlStream(encoder, options) +def _passed_on_error(result: Result) -> str: + """Return the error of a result that was read from Jelly-SPARQL, if any.""" + error = getattr(result, "error", "") + if not error and getattr(result, "truncated", False): + return TRUNCATED_ERROR + return str(error) + + def _write_result(stream: SparqlStream, result: Result, writer: FrameWriter) -> None: """Write one result set: the result of a SELECT or ASK query.""" links = getattr(result, "links", ()) if result.type == "ASK": - writer.write(stream.ask(bool(result.askAnswer), links=links)) + if result.askAnswer is None: + msg = "the result of the ASK query has no answer (askAnswer is None)" + raise JellyConformanceError(msg) + frame = stream.ask( + bool(result.askAnswer), links=links, error=_passed_on_error(result) + ) + writer.write(frame) return if result.type != "SELECT": msg = ( @@ -174,12 +221,27 @@ def _write_result(stream: SparqlStream, result: Result, writer: FrameWriter) -> f"{result.type}. Write graphs in Jelly-RDF instead." ) raise JellyConformanceError(msg) - variables = result.vars or [] - rows = ( - [binding.get(variable) for variable in variables] - for binding in _made_bindings(result) + if result.vars is None: + msg = "the result of the SELECT query has no variables (vars is None)" + raise JellyConformanceError(msg) + variables = result.vars + rows = (_row(binding, variables) for binding in _made_bindings(result)) + write_solutions( + stream, variables, rows, writer, links=links, error=_passed_on_error(result) ) - write_solutions(stream, variables, rows, writer, links) + + +def _row( + binding: Mapping[Any, Identifier], variables: Sequence[Variable] +) -> list[Identifier | None]: + """Return the values of the variables in a binding, None if unbound.""" + row = [binding.get(variable) for variable in variables] + if len(binding) + row.count(None) != len(variables): + # Not all of the binding's keys are variables: rdflib's Variable("x") is + # not equal to "x". Look the values up by name instead. + by_name = {str(key): value for key, value in binding.items()} + row = [by_name.get(str(variable)) for variable in variables] + return row def _made_bindings(result: Result) -> Iterator[Mapping[Variable, Identifier]]: @@ -196,6 +258,8 @@ def _made_bindings(result: Result) -> Iterator[Mapping[Variable, Identifier]]: return result._genbindings = None kept = result._bindings + # Those made before, e.g. by an iteration of the result that stopped early + yield from list(kept) for binding in made: kept.append(binding) yield binding @@ -209,7 +273,8 @@ def serialize_jelly_sparql( If getting the results fails midway (e.g., the query fails), the stream ends with the error, so that readers know the results are incomplete, and the - error is raised. + error is raised. Results read from Jelly-SPARQL that are incomplete or + truncated are written as incomplete. Args: result (Result): the results. @@ -221,13 +286,13 @@ def serialize_jelly_sparql( _write_result(stream, result, FrameWriter(out, delimited=stream.options.delimited)) -def serialize_jelly_sparql_results( +def serialize_jelly_sparql_sequence( results: Iterable[Result], out: IO[bytes], options: SparqlOptions | None = None ) -> None: """ Write a sequence of SPARQL query results in one Jelly-SPARQL stream. - The stream is of the PUNCTUATED type: each result is a result set of its own. + The stream has the PUNCTUATED type: each result is a result set of its own. Args: results (Iterable[Result]): the results. @@ -240,7 +305,7 @@ def serialize_jelly_sparql_results( options = SparqlOptions(punctuated=True) if not options.punctuated: msg = "a sequence of results needs SparqlOptions(punctuated=True)" - raise JellyConformanceError(msg) + raise ValueError(msg) stream = sparql_stream(options) writer = FrameWriter(out, delimited=True) for result in results: @@ -251,26 +316,25 @@ class RDFLibJellySparqlParser(ResultParser): """Reads SPARQL query results in Jelly-SPARQL, for rdflib's Result.parse().""" @override - def parse( + def parse( # type: ignore[override] self, - source: IO[bytes] | str | Path, + source: IO[bytes] | bytes | str | Path, content_type: str | None = None, *, validate_iris: bool = False, lazy: bool = False, - **unused: Any, ) -> Result: """ Read the results. Args: - source (IO[bytes] | str | Path): the stream, or the path of a file. + source (IO[bytes] | bytes | str | Path): the stream, its bytes, or the + path of a file. content_type (str | None): unused. validate_iris (bool): whether rdflib checks the IRIs, see parse_jelly_sparql(). lazy (bool): whether to read the rows as the result is iterated, see parse_jelly_sparql(). Only for streams, not paths. - **unused (Any): unused arguments of Result.parse(). """ if not isinstance(source, (str, Path)): @@ -286,13 +350,12 @@ class RDFLibJellySparqlSerializer(ResultSerializer): """Writes SPARQL query results in Jelly-SPARQL, for rdflib's Result.serialize().""" @override - def serialize( + def serialize( # type: ignore[override] self, stream: IO[bytes], encoding: str = "utf-8", *, options: SparqlOptions | None = None, - **unused: Any, ) -> None: """ Write the results. @@ -301,7 +364,6 @@ def serialize( stream (IO[bytes]): where to write them. encoding (str): unused, Jelly is binary. options (SparqlOptions | None): options of the stream. - **unused (Any): unused arguments of Result.serialize(). """ serialize_jelly_sparql(self.result, stream, options) diff --git a/pyjelly/options.py b/pyjelly/options.py index 29a79b44..e75d64ff 100644 --- a/pyjelly/options.py +++ b/pyjelly/options.py @@ -257,10 +257,11 @@ class SparqlOptions: does not allow triple terms. punctuated: whether the stream is a sequence of result sets (the PUNCTUATED stream type), rather than one result set. - frame_size: how many values (rows times variables) a frame has, at most. - A frame may end earlier, when the lookups are full. - delimited: whether the frames are delimited. A non-delimited stream has - one frame. + frame_values: how many values (rows times variables) a frame has, at + most. A frame may end earlier, when the lookups are full. + delimited: whether the frames are delimited, as files and HTTP need. A + non-delimited stream is one frame, e.g. to be one message of a + message broker. Then all results must fit in the lookups. stream_name: name of the stream. """ @@ -268,7 +269,7 @@ class SparqlOptions: lookup_preset: LookupPreset = SPARQL_LOOKUP_PRESET rdf_version: jelly.RdfVersion = jelly.RDF_VERSION_UNSPECIFIED punctuated: bool = False - frame_size: int = DEFAULT_SPARQL_FRAME_VALUES + frame_values: int = DEFAULT_SPARQL_FRAME_VALUES delimited: bool = True stream_name: str = "" @@ -280,12 +281,12 @@ def __post_init__(self) -> None: f"be at least {MIN_COLUMN_NAME_LOOKUP_SIZE}" ) raise JellyConformanceError(msg) - if self.frame_size < 1: - msg = f"the frame size must be at least 1, not {self.frame_size}" - raise JellyConformanceError(msg) + if self.frame_values < 1: + msg = f"frame_values must be at least 1, not {self.frame_values}" + raise ValueError(msg) if self.punctuated and not self.delimited: msg = "a PUNCTUATED stream must be delimited" - raise JellyConformanceError(msg) + raise ValueError(msg) def rows_per_frame(self, variables: int) -> int: """ @@ -293,7 +294,7 @@ def rows_per_frame(self, variables: int) -> int: >>> SparqlOptions().rows_per_frame(4) 1024 - >>> SparqlOptions(frame_size=10).rows_per_frame(0) + >>> SparqlOptions(frame_values=10).rows_per_frame(0) 10 """ - return max(1, self.frame_size // max(1, variables)) + return max(1, self.frame_values // max(1, variables)) diff --git a/pyjelly/parse/ioutils.py b/pyjelly/parse/ioutils.py index c409f0af..f5c539c4 100644 --- a/pyjelly/parse/ioutils.py +++ b/pyjelly/parse/ioutils.py @@ -303,10 +303,15 @@ def detect_delimited( header = read_hint_header(inp, tags) inp.seek(-len(header), os.SEEK_CUR) else: - # Input may not be seekable (e.g. a network stream) -- then we need to buffer - # it and put the bytes we looked at back in front of it. + # Input may not be seekable (e.g. a network stream) -- then we put the bytes + # we looked at back in front of it. An unbuffered stream gets a buffer, but + # not a buffered one (e.g. from socket.makefile()): the outer buffer would + # wait for the inner one to fill up, so a stream that is still being + # written would not be read as it comes. # See also: https://github.com/Jelly-RDF/pyjelly/issues/298 - buffered = io.BufferedReader(inp) # type: ignore[arg-type, type-var, unused-ignore] + buffered = inp + if not isinstance(inp, io.BufferedIOBase): + buffered = io.BufferedReader(inp) # type: ignore[arg-type, type-var, unused-ignore] header = read_hint_header(buffered, tags) inp = _StreamWithHeader(header, buffered) # type: ignore[assignment] return inp, delimited_jelly_hint(header, tags) diff --git a/pyjelly/parse/sparql.py b/pyjelly/parse/sparql.py index e5d87a61..a3d9fddd 100644 --- a/pyjelly/parse/sparql.py +++ b/pyjelly/parse/sparql.py @@ -7,13 +7,21 @@ from __future__ import annotations +import io +import warnings from collections.abc import Callable, Iterator +from itertools import chain from typing import IO, Any, Final +from google.protobuf.message import DecodeError from google.protobuf.proto import parse, parse_length_prefixed from pyjelly import jelly -from pyjelly.errors import JellyConformanceError, JellyIncompleteResultsError +from pyjelly.errors import ( + JellyConformanceError, + JellyIncompleteResultsError, + JellyTruncatedResultsWarning, +) from pyjelly.options import ( MAX_DATATYPE_LOOKUP_SIZE, MAX_NAME_LOOKUP_SIZE, @@ -367,28 +375,52 @@ def read_header(self, names: list[str]) -> None: self.header_in_effect = True -def sparql_frames(inp: IO[bytes]) -> Iterator[jelly.SparqlResultsFrame]: +def sparql_frames(inp: IO[bytes] | bytes) -> Iterator[jelly.SparqlResultsFrame]: """ Return the frames of a Jelly-SPARQL stream, delimited or not. Raises: - JellyConformanceError: if the stream is empty. + JellyConformanceError: if the stream is empty, or isn't Jelly-SPARQL. """ + if isinstance(inp, (bytes, bytearray, memoryview)): + inp = io.BytesIO(inp) inp, delimited = detect_delimited(inp, SPARQL_FRAME_TAGS) if not delimited: data = inp.read() if not data: msg = "the stream is empty, but it must have at least one frame" raise JellyConformanceError(msg) - return iter((parse(jelly.SparqlResultsFrame, data),)) + try: + return iter((parse(jelly.SparqlResultsFrame, data),)) + except DecodeError as error: + raise _not_sparql(error) from None return _delimited_frames(inp) +def _not_sparql(error: Exception) -> JellyConformanceError: + msg = ( + f"the stream is not valid Jelly-SPARQL ({error}). Is it cut short, or is it " + "Jelly-RDF?" + ) + return JellyConformanceError(msg) + + def _delimited_frames(inp: IO[bytes]) -> Iterator[jelly.SparqlResultsFrame]: while True: - # protobuf annotates `input_bytes` as io.BytesIO, but any IO[bytes] works. - frame = parse_length_prefixed(jelly.SparqlResultsFrame, inp) # type: ignore[arg-type] + try: + # protobuf annotates `input_bytes` as io.BytesIO, but any IO[bytes] works. + frame = parse_length_prefixed(jelly.SparqlResultsFrame, inp) # type: ignore[arg-type] + except DecodeError as error: + raise _not_sparql(error) from None + except ValueError as error: + if getattr(inp, "closed", False): + msg = ( + "the input was closed before all results were read. Read the " + "rows before closing it." + ) + raise ValueError(msg) from error + raise if frame is None: return yield frame @@ -417,17 +449,25 @@ def next(self) -> DecodedFrame | None: return self.decoder.decode(message) +def _incomplete(error: str, result: object) -> JellyIncompleteResultsError: + msg = f"the results are incomplete: {error}" + return JellyIncompleteResultsError(msg, error, result) + + class SparqlResultSet: """ One result set of a Jelly-SPARQL stream: a solution sequence or a boolean result. - The rows of a solution sequence are read as they are iterated, so they can be - iterated only once. Each row is a tuple, with one value per variable, or None - if the variable is unbound. Rows not yet iterated when the next result set is + The rows of a solution sequence are read as they are iterated. Each row is a + tuple, with one value per variable, or None if the variable is unbound. An + iteration that stops early can be picked up again, but once all rows were read, + they can't be iterated again. Rows not yet read when the next result set is read are kept in memory. If the producer ended the result set with an error, iterating it raises - JellyIncompleteResultsError after the last row. + JellyIncompleteResultsError after the last row, and so does `boolean`. If the + stream ended without saying that the result set is complete, `truncated` is + set, and JellyTruncatedResultsWarning is issued. """ def __init__(self, reader: ResultSetReader, first: DecodedFrame) -> None: @@ -435,8 +475,7 @@ def __init__(self, reader: ResultSetReader, first: DecodedFrame) -> None: self.first: DecodedFrame | None = first #: The variables, in projection order. Empty for a boolean result. self.variables: list[str] = first.variables - #: The boolean result (of an ASK query), None for a solution sequence - self.boolean: bool | None = first.boolean + self._boolean = first.boolean #: IRIs of links that describe the result set (head.link in SPARQL JSON) self.links: list[str] = first.links or [] #: Why the result set is incomplete, if the producer said so @@ -446,23 +485,59 @@ def __init__(self, reader: ResultSetReader, first: DecodedFrame) -> None: self.truncated = False self._rows = self._read() # The rows kept when the next result set was read before they were - self._kept: list[tuple[Any, ...]] | None = None + self._kept: Iterator[tuple[Any, ...]] = iter(()) + # Whether all rows were read by an iteration + self._done = False + if first.boolean is not None: + # A boolean result is one frame: whether it is complete is known now + if first.trailer: + self.error = first.trailer + elif first.trailer is None: + self._truncated() + + @property + def boolean(self) -> bool | None: + """ + The boolean result (of an ASK query), None for a solution sequence. + + Raises: + JellyIncompleteResultsError: if the producer could not complete it. + + """ + if self.error and self._boolean is not None: + raise _incomplete(self.error, self) + return self._boolean @property def is_boolean(self) -> bool: """Whether this is a boolean result (of an ASK query).""" - return self.boolean is not None + return self._boolean is not None def __iter__(self) -> Iterator[tuple[Any, ...]]: - kept = self._kept - if kept is not None: - self._kept = None - yield from kept - else: - yield from self._rows + if self._done: + msg = ( + "the rows of the result set were already read: they can be " + "iterated only once" + ) + raise RuntimeError(msg) + # Not `yield from`, which would close them if this iteration is stopped + for row in self._kept: + yield row + for row in self._rows: + yield row + self._done = True if self.error: - msg = f"the result set is incomplete: {self.error}" - raise JellyIncompleteResultsError(msg, self.error) + raise _incomplete(self.error, self) + + def _truncated(self) -> None: + if not self.truncated: + self.truncated = True + warnings.warn( + "the stream ended without saying that the results are complete, so " + "some may be missing", + JellyTruncatedResultsWarning, + stacklevel=3, + ) def _read(self) -> Iterator[tuple[Any, ...]]: """Yield the rows, and note the trailers, until the result set ends.""" @@ -491,19 +566,17 @@ def _read(self) -> Iterator[tuple[Any, ...]]: # The next result set of a PUNCTUATED stream reader.pending = frame return - self.truncated = not ended + if not ended: + self._truncated() def keep(self) -> None: """Read the rest of the result set, and keep its rows for later.""" - rows = list(self._rows) - if self._kept is None: - self._kept = rows - else: - self._kept += rows + if not self._done: + self._kept = chain(self._kept, list(self._rows)) def read_result_sets( - inp: IO[bytes], + inp: IO[bytes] | bytes, adapter_factory: AdapterFactory, *, max_rows: int = MAX_SPARQL_FRAME_ROWS, @@ -518,7 +591,7 @@ def read_result_sets( rows, and keeps them in memory. Args: - inp (IO[bytes]): the stream, delimited or not. + inp (IO[bytes] | bytes): the stream, delimited or not. adapter_factory (AdapterFactory): makes the adapter that makes the terms, given the options of the stream. max_rows (int): the most rows a frame can have. @@ -537,7 +610,7 @@ def read_result_sets( def read_result_set( - inp: IO[bytes], + inp: IO[bytes] | bytes, adapter_factory: AdapterFactory, *, max_rows: int = MAX_SPARQL_FRAME_ROWS, @@ -551,6 +624,8 @@ def read_result_set( Raises: JellyConformanceError: if the stream is of the PUNCTUATED type. + JellyIncompleteResultsError: if it is a boolean result, which the producer + could not complete. """ result_sets = read_result_sets( @@ -559,8 +634,11 @@ def read_result_set( result_set = next(result_sets) if result_set.reader.decoder.punctuated: msg = ( - "the stream is a sequence of result sets (PUNCTUATED). Read all of them " - "with parse_jelly_sparql_results(), or read_result_sets()" + "the stream is a sequence of result sets (PUNCTUATED). Read it with " + "parse_jelly_sparql_sequence() of pyjelly.integrations.rdflib.sparql " + "or pyjelly.integrations.generic.sparql" ) raise JellyConformanceError(msg) + if result_set.is_boolean and result_set.error: + raise _incomplete(result_set.error, result_set) return result_set diff --git a/pyjelly/serialize/sparql.py b/pyjelly/serialize/sparql.py index 010117ba..803884cd 100644 --- a/pyjelly/serialize/sparql.py +++ b/pyjelly/serialize/sparql.py @@ -12,7 +12,7 @@ from typing import IO, Final from pyjelly import jelly -from pyjelly.errors import JellyConformanceError +from pyjelly.errors import JellyConformanceError, JellyIncompleteResultsError from pyjelly.options import SPARQL_VERSION_1_0, SparqlOptions from pyjelly.serialize.columns import MAX_ROWS, Column from pyjelly.serialize.encode import EncodedTerm, TermEncoder @@ -354,13 +354,19 @@ def end(self, error: str = "") -> jelly.SparqlResultsFrame: self.variables = None return frame - def ask(self, value: bool, links: Sequence[str] = ()) -> jelly.SparqlResultsFrame: # noqa: FBT001 + def ask( + self, + value: bool, # noqa: FBT001 + links: Sequence[str] = (), + error: str = "", + ) -> jelly.SparqlResultsFrame: """ Write a boolean result (of an ASK query), and return its frame. Args: value (bool): the result. links (Sequence[str]): IRIs of links describing the result. + error (str): why the result is incomplete, if it is. Returns: jelly.SparqlResultsFrame: the only frame of the result set. @@ -371,6 +377,7 @@ def ask(self, value: bool, links: Sequence[str] = ()) -> jelly.SparqlResultsFram self.write_start(frame) frame.ask_result.value = value frame.trailer.SetInParent() + frame.trailer.error = error if links: frame.metadata[LINK_KEY] = links_value(links) if not self.options.punctuated: @@ -395,8 +402,30 @@ def stream_options(self) -> jelly.SparqlResultsOptions: ) +def error_message(error: BaseException) -> str: + """ + Return what a trailer says about an error that left the results incomplete. + + >>> error_message(KeyError("x")) + "KeyError: 'x'" + >>> error_message(JellyIncompleteResultsError("...", "timeout")) + 'timeout' + """ + if isinstance(error, JellyIncompleteResultsError): + # Results passed on from a stream that was incomplete itself + return error.error + message = str(error) + name = type(error).__name__ + return f"{name}: {message}" if message else name + + class FrameWriter: - """Writes the frames of a stream to a file: delimited, or as its only frame.""" + """ + Writes the frames of a stream to a file: delimited, or as its only frame. + + The file is flushed after each frame, so that a reader at the other end of a + connection gets the rows as they are written. + """ def __init__(self, out: IO[bytes], *, delimited: bool) -> None: self.out = out @@ -406,6 +435,7 @@ def __init__(self, out: IO[bytes], *, delimited: bool) -> None: def write(self, frame: jelly.SparqlResultsFrame) -> None: if self.delimited: write_delimited(frame, self.out) + self.out.flush() return if self.written: msg = ( @@ -414,22 +444,25 @@ def write(self, frame: jelly.SparqlResultsFrame) -> None: ) raise JellyConformanceError(msg) write_single(frame, self.out) + self.out.flush() self.written = True -def write_solutions( +def write_solutions( # noqa: PLR0913 keyword-only options stream: SparqlStream, variables: Iterable[str], rows: Iterable[Sequence[object | None]], writer: FrameWriter, + *, links: Sequence[str] = (), + error: str = "", ) -> None: """ Write a solution sequence (the result of a SELECT query) as a result set. If getting the rows fails midway (e.g., the query fails), the result set ends with the error, so that readers know it is incomplete, and the error is - raised. + raised. `error` says why the rows are incomplete, if they are known to be. """ stream.start(variables, links) try: @@ -437,9 +470,9 @@ def write_solutions( frame = stream.row(values) if frame is not None: writer.write(frame) - except Exception as error: + except Exception as failure: # If the frame that says so can still be written with suppress(Exception): - writer.write(stream.end(str(error) or type(error).__name__)) + writer.write(stream.end(error_message(failure))) raise - writer.write(stream.end()) + writer.write(stream.end(error)) diff --git a/tests/conformance_tests/test_sparql/test_parse.py b/tests/conformance_tests/test_sparql/test_parse.py index 9bf3d321..5def36ce 100644 --- a/tests/conformance_tests/test_sparql/test_parse.py +++ b/tests/conformance_tests/test_sparql/test_parse.py @@ -7,13 +7,14 @@ import pytest from rdflib.query import Result +from pyjelly.errors import JellyIncompleteResultsError from pyjelly.integrations.generic.sparql import ( parse_jelly_sparql as generic_parse_jelly_sparql, ) from pyjelly.integrations.generic.sparql import ( - parse_jelly_sparql_results as generic_parse_jelly_sparql_results, + parse_jelly_sparql_sequence as generic_parse_jelly_sparql_sequence, ) -from pyjelly.integrations.rdflib.sparql import parse_jelly_sparql_results +from pyjelly.integrations.rdflib.sparql import parse_jelly_sparql_sequence from tests.conformance_tests.test_rdf._common import ConformanceCase, load_cases from tests.meta import SPARQL_FROM_JELLY_MANIFEST from tests.utils.sparql_compare import ( @@ -43,7 +44,7 @@ def _expected(case: ConformanceCase) -> list[Results]: def test_rdflib_parses_positive(case: ConformanceCase) -> None: with case.actions[0].open("rb") as file: if case.punctuated: - results = [rdflib_results(r) for r in parse_jelly_sparql_results(file)] + results = [rdflib_results(r) for r in parse_jelly_sparql_sequence(file)] else: results = [rdflib_results(Result.parse(file, format="jelly"))] assert_same_result_sets(_expected(case), results, hint=case.id) @@ -54,24 +55,34 @@ def test_generic_parses_positive(case: ConformanceCase) -> None: with case.actions[0].open("rb") as file: if case.punctuated: results = [ - generic_results(r) for r in generic_parse_jelly_sparql_results(file) + generic_results(r) for r in generic_parse_jelly_sparql_sequence(file) ] else: results = [generic_results(generic_parse_jelly_sparql(file))] assert_same_result_sets(_expected(case), results, hint=case.id) +def _read_all_rdflib(file: IO[bytes], *, punctuated: bool) -> None: + if not punctuated: + Result.parse(file, format="jelly") + return + # Results that are incomplete don't end a sequence, they say so + for result in parse_jelly_sparql_sequence(file): + if result.error: + raise JellyIncompleteResultsError(result.error, result.error) + + @pytest.mark.parametrize("case", _params(positive=False, rdflib=True)) def test_rdflib_parsing_fails_negative(case: ConformanceCase) -> None: with ( pytest.raises(Exception), # noqa: B017,PT011 any error will do case.actions[0].open("rb") as file, ): - list(parse_jelly_sparql_results(file)) + _read_all_rdflib(file, punctuated=case.punctuated) def _read_all(file: IO[bytes]) -> None: - for result_set in generic_parse_jelly_sparql_results(file): + for result_set in generic_parse_jelly_sparql_sequence(file): list(result_set) diff --git a/tests/conformance_tests/test_sparql/test_serialize.py b/tests/conformance_tests/test_sparql/test_serialize.py index 6ecd1403..717deab0 100644 --- a/tests/conformance_tests/test_sparql/test_serialize.py +++ b/tests/conformance_tests/test_sparql/test_serialize.py @@ -10,12 +10,12 @@ from pyjelly import jelly from pyjelly.integrations.generic.sparql import ( - parse_jelly_sparql_results, + parse_jelly_sparql_sequence, sparql_stream, ) from pyjelly.integrations.rdflib.sparql import ( serialize_jelly_sparql, - serialize_jelly_sparql_results, + serialize_jelly_sparql_sequence, ) from pyjelly.options import LookupPreset, SparqlOptions from pyjelly.parse.sparql import sparql_frames @@ -87,7 +87,7 @@ def _write_rdflib(case: ConformanceCase) -> None: options = _options(case) with _output(case).open("wb") as file: if options.punctuated: - serialize_jelly_sparql_results(results, file, options) + serialize_jelly_sparql_sequence(results, file, options) else: (result,) = results serialize_jelly_sparql(result, file, options) @@ -101,7 +101,7 @@ def _check(case: ConformanceCase) -> None: assert frames[-1].HasField("trailer") assert not frames[-1].trailer.error with output.open("rb") as file: - actual = [generic_results(r) for r in parse_jelly_sparql_results(file)] + actual = [generic_results(r) for r in parse_jelly_sparql_sequence(file)] expected = [read_srj(path) for path in case.inputs] assert_same_result_sets(expected, actual, hint=case.id) diff --git a/tests/integration_tests/test_generic/test_sparql.py b/tests/integration_tests/test_generic/test_sparql.py index bd247421..34315c71 100644 --- a/tests/integration_tests/test_generic/test_sparql.py +++ b/tests/integration_tests/test_generic/test_sparql.py @@ -4,11 +4,16 @@ import io from collections.abc import Iterator +from pathlib import Path import pytest from pyjelly import jelly -from pyjelly.errors import JellyConformanceError, JellyIncompleteResultsError +from pyjelly.errors import ( + JellyConformanceError, + JellyIncompleteResultsError, + JellyTruncatedResultsWarning, +) from pyjelly.integrations.generic.generic_sink import ( IRI, BlankNode, @@ -18,9 +23,10 @@ ) from pyjelly.integrations.generic.sparql import ( parse_jelly_sparql, - parse_jelly_sparql_results, + parse_jelly_sparql_sequence, serialize_jelly_sparql, serialize_jelly_sparql_ask, + serialize_jelly_sparql_sequence, sparql_stream, ) from pyjelly.options import LookupPreset, SparqlOptions @@ -66,7 +72,7 @@ def test_round_trip() -> None: @pytest.mark.parametrize("frame_size", [1, 3, 7, 1000]) def test_frames(frame_size: int) -> None: - out = _write(ROWS * 20, SparqlOptions(frame_size=frame_size)) + out = _write(ROWS * 20, SparqlOptions(frame_values=frame_size)) frames = list(sparql_frames(out)) rows_per_frame = max(1, frame_size // 3) assert all(frame.row_count <= rows_per_frame for frame in frames) @@ -94,7 +100,7 @@ def test_triple_terms_in_several_columns() -> None: ] * 50 options = SparqlOptions( lookup_preset=LookupPreset(max_names=128, max_prefixes=4, max_datatypes=4), - frame_size=9, + frame_values=9, ) assert list(parse_jelly_sparql(_write(rows, options))) == rows @@ -163,7 +169,7 @@ def test_incomplete() -> None: rows.extend(results) # The rows before the error are read assert rows == ROWS[:2] - assert results.error == "timeout" + assert results.error == "TimeoutError: timeout" def test_sequence_of_result_sets() -> None: @@ -175,7 +181,7 @@ def test_sequence_of_result_sets() -> None: writer.write(stream.ask(False)) # noqa: FBT003 write_solutions(stream, ["x"], [(BlankNode("b"),)], writer, links=[EX + "l"]) out.seek(0) - first, second, third = parse_jelly_sparql_results(out) + first, second, third = parse_jelly_sparql_sequence(out) assert list(first) == ROWS assert second.boolean is False assert third.variables == ["x"] @@ -186,13 +192,13 @@ def test_sequence_of_result_sets() -> None: def test_rows_are_kept_for_later() -> None: - stream = sparql_stream(SparqlOptions(punctuated=True, frame_size=3)) + stream = sparql_stream(SparqlOptions(punctuated=True, frame_values=3)) out = io.BytesIO() writer = FrameWriter(out, delimited=True) write_solutions(stream, ["s", "o", "t"], ROWS, writer) write_solutions(stream, ["s", "o", "t"], ROWS[:1], writer) out.seek(0) - first, second = parse_jelly_sparql_results(out) + first, second = parse_jelly_sparql_sequence(out) assert list(second) == ROWS[:1] assert list(first) == ROWS @@ -205,5 +211,60 @@ def test_flat_reader_rejects_punctuated() -> None: SparqlOptions(punctuated=True), ) out.seek(0) - with pytest.raises(JellyConformanceError, match="parse_jelly_sparql_results"): + with pytest.raises(JellyConformanceError, match="parse_jelly_sparql_sequence"): parse_jelly_sparql(out) + + +def test_incomplete_boolean_result() -> None: + data = io.BytesIO() + FrameWriter(data, delimited=True).write( + sparql_stream().ask(True, error="no answer") # noqa: FBT003 + ) + with pytest.raises(JellyIncompleteResultsError, match="no answer"): + parse_jelly_sparql(data.getvalue()) + + +def test_rows_are_iterated_once() -> None: + results = parse_jelly_sparql(_write(ROWS)) + assert list(results) == ROWS + with pytest.raises(RuntimeError, match="only once"): + list(results) + + +def test_iteration_can_be_picked_up_again() -> None: + results = parse_jelly_sparql(_write(ROWS, SparqlOptions(frame_values=3))) + for _ in results: + break + assert list(results) == ROWS[1:] + + +def test_closed_input(tmp_path: Path) -> None: + path = tmp_path / "results.jellys" + path.write_bytes(_write(ROWS, SparqlOptions(frame_values=3)).getvalue()) + with path.open("rb") as file: + results = parse_jelly_sparql(file) + with pytest.raises(ValueError, match="closed before all results were read"): + list(results) + + +def test_truncated() -> None: + data = _write(ROWS, SparqlOptions(frame_values=3)).getvalue() + frames = list(sparql_frames(data)) + cut = data[: -(frames[-1].ByteSize() + 1)] + results = parse_jelly_sparql(cut) + with pytest.warns(JellyTruncatedResultsWarning): + rows = list(results) + assert results.truncated + assert len(rows) == len(ROWS) + + +def test_sequence_writer() -> None: + out = io.BytesIO() + serialize_jelly_sparql_sequence( + [(["s", "o", "t"], ROWS), False, (["x"], [(A,)])], + out, + ) + first, second, third = parse_jelly_sparql_sequence(out.getvalue()) + assert list(first) == ROWS + assert second.boolean is False + assert list(third) == [(A,)] diff --git a/tests/integration_tests/test_rdflib/test_sparql.py b/tests/integration_tests/test_rdflib/test_sparql.py index b461a095..bb583fee 100644 --- a/tests/integration_tests/test_rdflib/test_sparql.py +++ b/tests/integration_tests/test_rdflib/test_sparql.py @@ -3,6 +3,7 @@ from __future__ import annotations import io +import socket import threading from collections.abc import Iterator from http.server import BaseHTTPRequestHandler, HTTPServer @@ -15,16 +16,22 @@ from rdflib.query import Result, ResultRow from pyjelly import jelly -from pyjelly.errors import JellyConformanceError, JellyIncompleteResultsError +from pyjelly.errors import ( + JellyConformanceError, + JellyIncompleteResultsError, + JellyTruncatedResultsWarning, +) from pyjelly.integrations.rdflib.sparql import ( JellySparqlResult, parse_jelly_sparql, - parse_jelly_sparql_results, + parse_jelly_sparql_sequence, serialize_jelly_sparql, - serialize_jelly_sparql_results, + serialize_jelly_sparql_sequence, + sparql_stream, ) from pyjelly.options import LookupPreset, SparqlOptions from pyjelly.parse.sparql import sparql_frames +from pyjelly.serialize.sparql import FrameWriter EX = "http://example.org/" @@ -117,7 +124,7 @@ def test_empty_and_zero_variable_results(graph: Graph) -> None: def test_options(graph: Graph) -> None: options = SparqlOptions( lookup_preset=LookupPreset(max_names=128, max_prefixes=0, max_datatypes=8), - frame_size=2, + frame_values=2, stream_name="people", ) result = graph.query(SELECT) @@ -166,7 +173,7 @@ def test_error_midway_ends_the_stream_with_it() -> None: out.seek(0) with pytest.raises(JellyIncompleteResultsError, match="went away") as info: Result.parse(out, format="jelly") - assert info.value.error == "the endpoint went away" + assert info.value.error == "RuntimeError: the endpoint went away" partial = info.value.result assert isinstance(partial, JellySparqlResult) assert partial.bindings == [{Variable("x"): URIRef(EX + "a")}] @@ -174,7 +181,7 @@ def test_error_midway_ends_the_stream_with_it() -> None: def test_truncated(graph: Graph) -> None: result = graph.query(SELECT) - data = result.serialize(format="jelly", options=SparqlOptions(frame_size=3)) + data = result.serialize(format="jelly", options=SparqlOptions(frame_values=3)) assert data is not None # The stream without its last frame, the one with the trailer frames = list(sparql_frames(io.BytesIO(data))) @@ -187,9 +194,9 @@ def test_truncated(graph: Graph) -> None: def test_sequence_of_results(graph: Graph) -> None: results = [graph.query(SELECT), graph.query("ASK { ?s ?p ?o }")] out = io.BytesIO() - serialize_jelly_sparql_results(results, out) + serialize_jelly_sparql_sequence(results, out) out.seek(0) - parsed = list(parse_jelly_sparql_results(out)) + parsed = list(parse_jelly_sparql_sequence(out)) assert parsed == results out.seek(0) with pytest.raises(JellyConformanceError, match="PUNCTUATED"): @@ -197,8 +204,8 @@ def test_sequence_of_results(graph: Graph) -> None: def test_sequence_needs_punctuated_options(graph: Graph) -> None: - with pytest.raises(JellyConformanceError, match="punctuated"): - serialize_jelly_sparql_results( + with pytest.raises(ValueError, match="punctuated"): + serialize_jelly_sparql_sequence( [graph.query(SELECT)], io.BytesIO(), SparqlOptions() ) @@ -297,7 +304,7 @@ def test_frames_of_many_terms() -> None: def test_stream_options_are_written_once(graph: Graph) -> None: data = graph.query(SELECT).serialize( - format="jelly", options=SparqlOptions(frame_size=3) + format="jelly", options=SparqlOptions(frame_values=3) ) assert data is not None frames = list(sparql_frames(io.BytesIO(data))) @@ -311,7 +318,7 @@ def test_stream_options_are_written_once(graph: Graph) -> None: def test_lazy(graph: Graph) -> None: result = graph.query(SELECT) - data = result.serialize(format="jelly", options=SparqlOptions(frame_size=3)) + data = result.serialize(format="jelly", options=SparqlOptions(frame_values=3)) assert data is not None inp = io.BytesIO(data) parsed = Result.parse(inp, format="jelly", lazy=True) @@ -346,3 +353,134 @@ def test_lazy_needs_a_stream(tmp_path: Path) -> None: path = tmp_path / "results.jellys" with pytest.raises(ValueError, match="stream"): Result.parse(path, format="jelly", lazy=True) # type: ignore[arg-type] + + +def test_partly_iterated_result(graph: Graph) -> None: + result = graph.query(SELECT) + for _ in result: + break + parsed = _round_trip(result) + assert len(parsed) == len(graph.query(SELECT)) + + +def test_bindings_with_names_as_keys() -> None: + result = Result("SELECT") + result.vars = [Variable("x"), Variable("y")] + result.bindings = [{"x": URIRef(EX + "a")}, {Variable("y"): Literal(1)}] # type: ignore[dict-item] + parsed = _round_trip(result) + assert parsed.bindings == [ + {Variable("x"): URIRef(EX + "a")}, + {Variable("y"): Literal(1)}, + ] + + +def test_results_without_variables_or_answer() -> None: + with pytest.raises(JellyConformanceError, match="no variables"): + serialize_jelly_sparql(Result("SELECT"), io.BytesIO()) + with pytest.raises(JellyConformanceError, match="no answer"): + serialize_jelly_sparql(Result("ASK"), io.BytesIO()) + + +def test_sequence_goes_on_after_incomplete_results(graph: Graph) -> None: + failing = Result("SELECT") + failing.vars = [Variable("x")] + failing.bindings = _failing_bindings() + out = io.BytesIO() + with pytest.raises(RuntimeError): + serialize_jelly_sparql_sequence([graph.query("ASK { ?s ?p ?o }"), failing], out) + serialize_jelly_sparql_sequence([graph.query(SELECT)], out) + out.seek(0) + first, second, third = parse_jelly_sparql_sequence(out) + assert first.askAnswer is True + assert not first.error + assert second.error == "RuntimeError: the endpoint went away" + assert len(second) == 1 + # Not equal: the blank nodes of a result set in a sequence are its own + assert len(third) == len(graph.query(SELECT)) + + +def test_lazy_links() -> None: + result = JellySparqlResult("SELECT") + result.vars = [Variable("x")] + result.bindings = [{Variable("x"): URIRef(EX + "a")}] + result.links = [EX + "about"] + data = result.serialize(format="jelly") + assert data is not None + parsed = Result.parse(io.BytesIO(data), format="jelly", lazy=True) + assert isinstance(parsed, JellySparqlResult) + assert parsed.links == [EX + "about"] + + +def test_incomplete_and_truncated_results_are_passed_on(graph: Graph) -> None: + data = graph.query(SELECT).serialize( + format="jelly", options=SparqlOptions(frame_values=3) + ) + assert data is not None + frames = list(sparql_frames(io.BytesIO(data))) + cut = data[: -(frames[-1].ByteSize() + 1)] + with pytest.warns(JellyTruncatedResultsWarning): + parsed = Result.parse(io.BytesIO(cut), format="jelly") + assert parsed.truncated + # Written again, the results say that they may be incomplete + again = parsed.serialize(format="jelly") + assert again is not None + with pytest.raises(JellyIncompleteResultsError, match="ended without"): + Result.parse(io.BytesIO(again), format="jelly") + + +def test_cut_frame() -> None: + result = Result("SELECT") + result.vars = [Variable("x")] + result.bindings = [{Variable("x"): URIRef(EX + "a")}] + data = result.serialize(format="jelly") + assert data is not None + with pytest.raises(JellyConformanceError, match="cut short"): + Result.parse(io.BytesIO(data[:-3]), format="jelly") + + +def test_jelly_rdf_is_not_jelly_sparql(graph: Graph) -> None: + data = graph.serialize(format="jelly", encoding="utf-8") + with pytest.raises(JellyConformanceError): + Result.parse(io.BytesIO(data), format="jelly") + + +def test_bytes(graph: Graph) -> None: + data = graph.query(SELECT).serialize(format="jelly") + assert data is not None + assert parse_jelly_sparql(data) == graph.query(SELECT) + + +def test_unknown_arguments(graph: Graph) -> None: + data = graph.query(SELECT).serialize(format="jelly") + assert data is not None + with pytest.raises(TypeError, match="lazzy"): + Result.parse(io.BytesIO(data), format="jelly", lazzy=True) + with pytest.raises(TypeError, match="option"): + graph.query(SELECT).serialize(format="jelly", option=SparqlOptions()) + + +def test_lazy_reading_from_a_socket() -> None: + reader_socket, writer_socket = socket.socketpair() + first_row_read = threading.Event() + waited: list[bool] = [] + + def produce() -> None: + with writer_socket, writer_socket.makefile("wb") as out: + stream = sparql_stream(SparqlOptions(frame_values=1)) + writer = FrameWriter(out, delimited=True) + stream.start(["x"]) + frame = stream.row([URIRef(EX + "a")]) + assert frame is not None + writer.write(frame) + # The reader has the first row before the rest is written + waited.append(first_row_read.wait(timeout=10)) + writer.write(stream.end()) + + producer = threading.Thread(target=produce) + producer.start() + with reader_socket, reader_socket.makefile("rb") as inp: + parsed = parse_jelly_sparql(inp, lazy=True) + for _ in parsed: + first_row_read.set() + producer.join() + assert waited == [True] diff --git a/tests/unit_tests/test_serialize/test_sparql.py b/tests/unit_tests/test_serialize/test_sparql.py index a268e2ff..fde1c2c9 100644 --- a/tests/unit_tests/test_serialize/test_sparql.py +++ b/tests/unit_tests/test_serialize/test_sparql.py @@ -125,15 +125,15 @@ def test_punctuated() -> None: def test_punctuated_must_be_delimited() -> None: - with pytest.raises(JellyConformanceError, match="delimited"): + with pytest.raises(ValueError, match="delimited"): SparqlOptions(punctuated=True, delimited=False) def test_options_checks() -> None: with pytest.raises(JellyConformanceError, match="at least 128"): SparqlOptions(lookup_preset=LookupPreset(max_names=64)) - with pytest.raises(JellyConformanceError, match="frame size"): - SparqlOptions(frame_size=0) + with pytest.raises(ValueError, match="frame_values"): + SparqlOptions(frame_values=0) def test_encoder_lookups_must_match() -> None: From b125a2362b1c60ff7038b99e8fa19806d69c8b09 Mon Sep 17 00:00:00 2001 From: Ostrzyciel Date: Mon, 26 Oct 2026 18:23:12 +0100 Subject: [PATCH 08/24] Make rdflib's Jelly-SPARQL bindings in compiled code Reading: the decoder makes rdflib's bindings (dicts of the bound variables) straight from a frame's columns, with a row factory compiled with mypyc, instead of making tuples that a Python loop turns into dicts. An eager read takes the rows a frame at a time (SparqlResultSet.collect()). Writing: SparqlStream.binding() takes a binding and looks the variables up in compiled code, instead of a Python loop making a list for each row. Instructions per cell (geometric mean of 5 datasets, CPython 3.15): rdflib writes -24%, rdflib reads -39%. The output is the same. --- pyjelly/integrations/rdflib/decode.py | 35 +++++++++ pyjelly/integrations/rdflib/sparql.py | 108 ++++++++++++++------------ pyjelly/parse/sparql.py | 93 +++++++++++++++++----- pyjelly/serialize/sparql.py | 86 ++++++++++++++++++-- 4 files changed, 246 insertions(+), 76 deletions(-) diff --git a/pyjelly/integrations/rdflib/decode.py b/pyjelly/integrations/rdflib/decode.py index 75597856..b02381eb 100644 --- a/pyjelly/integrations/rdflib/decode.py +++ b/pyjelly/integrations/rdflib/decode.py @@ -15,6 +15,7 @@ import rdflib from mypy_extensions import mypyc_attr from rdflib.graph import DATASET_DEFAULT_GRAPH_ID +from rdflib.term import Variable from pyjelly.errors import JellyConformanceError from pyjelly.integrations.rdflib.statements import Prefix, Quad, Triple @@ -168,3 +169,37 @@ def triple(self, terms: Iterable[Any]) -> Quad: @override def graph_end(self) -> None: self._graph_id = None + + +class BindingRows: + """ + Makes the rows of Jelly-SPARQL frames as rdflib bindings. + + A binding is a dict that maps each bound Variable to its value, as in rdflib's + query results. The same names get the same Variable objects (see variables()). + """ + + def __init__(self) -> None: + self.names: list[str] = [] + self.vars: list[Variable] = [] + + def variables(self, names: list[str]) -> list[Variable]: + """Return the Variables with these names.""" + if names != self.names: + self.names = names + self.vars = [Variable(name) for name in names] + return self.vars + + def __call__( + self, names: list[str], cells: list[list[Any]], rows: int + ) -> list[Any]: + bindings: list[dict[Variable, Any]] = [{} for _ in range(rows)] + variables = self.variables(names) + for j in range(len(cells)): + variable = variables[j] + column = cells[j] + for r in range(rows): + value = column[r] + if value is not None: + bindings[r][variable] = value + return bindings diff --git a/pyjelly/integrations/rdflib/sparql.py b/pyjelly/integrations/rdflib/sparql.py index 9eb74101..f2616599 100644 --- a/pyjelly/integrations/rdflib/sparql.py +++ b/pyjelly/integrations/rdflib/sparql.py @@ -10,16 +10,16 @@ from __future__ import annotations -from collections.abc import Iterable, Iterator, Mapping, Sequence +from collections.abc import Iterable, Iterator, Mapping from pathlib import Path -from typing import IO, Any, cast +from typing import IO, cast from typing_extensions import override from rdflib.query import Result, ResultParser, ResultSerializer from rdflib.term import Identifier, Variable from pyjelly.errors import JellyConformanceError, JellyIncompleteResultsError -from pyjelly.integrations.rdflib.decode import RDFLibAdapter +from pyjelly.integrations.rdflib.decode import BindingRows, RDFLibAdapter from pyjelly.integrations.rdflib.encode import RDFLibTermEncoder from pyjelly.options import SparqlOptions from pyjelly.parse.decode import Adapter, ParserOptions @@ -29,7 +29,7 @@ read_result_set, read_result_sets, ) -from pyjelly.serialize.sparql import FrameWriter, SparqlStream, write_solutions +from pyjelly.serialize.sparql import FrameWriter, SparqlStream, write_bindings #: The error that a result read from a truncated stream is passed on with TRUNCATED_ERROR = ( @@ -68,9 +68,13 @@ def make(options: ParserOptions) -> Adapter: def _to_result( - result_set: SparqlResultSet, *, lazy: bool, raise_error: bool + result_set: SparqlResultSet, + binding_rows: BindingRows, + *, + lazy: bool, + raise_error: bool, ) -> JellySparqlResult: - """Return a result set as an rdflib Result.""" + """Return a result set, whose rows are bindings, as an rdflib Result.""" if result_set.is_boolean: result = JellySparqlResult("ASK") # The rest of the stream, which must not have more of the result @@ -88,39 +92,44 @@ def _to_result( result.truncated = result_set.truncated return result result = JellySparqlResult("SELECT") - result.vars = [Variable(name) for name in result_set.variables] + # The Variables that the bindings have + result.vars = binding_rows.variables(result_set.variables) result.links = result_set.links - rows = _read_bindings(result_set, result, raise_error=raise_error) if lazy: # rdflib makes the bindings as the result is iterated, as for its own # query results - result.bindings = rows + result.bindings = _read_bindings(result_set, result, raise_error=raise_error) return result bindings: list[Mapping[Variable, Identifier]] = [] result.bindings = bindings - # If the results are incomplete, the result has those read before the error - bindings.extend(rows) + try: + result_set.collect(bindings) + except JellyIncompleteResultsError as error: + # The result has the rows read before the error + _incomplete(result, error, raise_error=raise_error) + finally: + result.links = result_set.links + result.truncated = result_set.truncated return result +def _incomplete( + result: JellySparqlResult, error: JellyIncompleteResultsError, *, raise_error: bool +) -> None: + result.error = error.error + if raise_error: + error.result = result + raise error + + def _read_bindings( result_set: SparqlResultSet, result: JellySparqlResult, *, raise_error: bool ) -> Iterator[Mapping[Variable, Identifier]]: - """Yield the bindings of the rows, and when they end, what the stream said.""" - variables = result.vars or [] + """Yield the bindings, and when they end, what the stream said.""" try: - for row in result_set: - # Unbound variables are left out, as rdflib does - yield { - variable: value - for variable, value in zip(variables, row, strict=True) - if value is not None - } + yield from result_set except JellyIncompleteResultsError as error: - result.error = error.error - if raise_error: - error.result = result - raise + _incomplete(result, error, raise_error=raise_error) finally: result.links = result_set.links result.truncated = result_set.truncated @@ -151,14 +160,17 @@ def parse_jelly_sparql( """ adapter_factory = _adapter_factory(validate_iris=validate_iris) + binding_rows = BindingRows() try: - result_set = read_result_set(inp, adapter_factory) + result_set = read_result_set(inp, adapter_factory, row_factory=binding_rows) except JellyIncompleteResultsError as error: # A boolean result that is incomplete result_set = cast("SparqlResultSet", error.result) - error.result = _to_result(result_set, lazy=False, raise_error=False) + error.result = _to_result( + result_set, binding_rows, lazy=False, raise_error=False + ) raise - return _to_result(result_set, lazy=lazy, raise_error=True) + return _to_result(result_set, binding_rows, lazy=lazy, raise_error=True) def parse_jelly_sparql_sequence( @@ -183,8 +195,9 @@ def parse_jelly_sparql_sequence( """ adapter_factory = _adapter_factory(validate_iris=validate_iris) - for result_set in read_result_sets(inp, adapter_factory): - yield _to_result(result_set, lazy=False, raise_error=False) + binding_rows = BindingRows() + for result_set in read_result_sets(inp, adapter_factory, row_factory=binding_rows): + yield _to_result(result_set, binding_rows, lazy=False, raise_error=False) def sparql_stream(options: SparqlOptions | None = None) -> SparqlStream: @@ -224,27 +237,24 @@ def _write_result(stream: SparqlStream, result: Result, writer: FrameWriter) -> if result.vars is None: msg = "the result of the SELECT query has no variables (vars is None)" raise JellyConformanceError(msg) - variables = result.vars - rows = (_row(binding, variables) for binding in _made_bindings(result)) - write_solutions( - stream, variables, rows, writer, links=links, error=_passed_on_error(result) + # Not result.bindings first: that would make all of them + made = getattr(result, "_genbindings", None) + bindings: Iterable[Mapping[Variable, Identifier]] = ( + result.bindings if made is None else _made_bindings(result, made) + ) + write_bindings( + stream, + result.vars, + bindings, + writer, + links=links, + error=_passed_on_error(result), ) -def _row( - binding: Mapping[Any, Identifier], variables: Sequence[Variable] -) -> list[Identifier | None]: - """Return the values of the variables in a binding, None if unbound.""" - row = [binding.get(variable) for variable in variables] - if len(binding) + row.count(None) != len(variables): - # Not all of the binding's keys are variables: rdflib's Variable("x") is - # not equal to "x". Look the values up by name instead. - by_name = {str(key): value for key, value in binding.items()} - row = [by_name.get(str(variable)) for variable in variables] - return row - - -def _made_bindings(result: Result) -> Iterator[Mapping[Variable, Identifier]]: +def _made_bindings( + result: Result, made: Iterator[Mapping[Variable, Identifier]] +) -> Iterator[Mapping[Variable, Identifier]]: """ Return the bindings of a result, as they are made. @@ -252,10 +262,6 @@ def _made_bindings(result: Result) -> Iterator[Mapping[Variable, Identifier]]: one fails, nothing would be written at all. They are still kept in the result, as Result.__iter__() does. """ - made = getattr(result, "_genbindings", None) - if made is None: - yield from result.bindings - return result._genbindings = None kept = result._bindings # Those made before, e.g. by an iteration of the result that stopped early diff --git a/pyjelly/parse/sparql.py b/pyjelly/parse/sparql.py index a3d9fddd..b77b4ee9 100644 --- a/pyjelly/parse/sparql.py +++ b/pyjelly/parse/sparql.py @@ -10,7 +10,6 @@ import io import warnings from collections.abc import Callable, Iterator -from itertools import chain from typing import IO, Any, Final from google.protobuf.message import DecodeError @@ -43,6 +42,17 @@ LINK_KEY: Final = "link" AdapterFactory = Callable[[ParserOptions], Adapter] +#: Makes the rows of a frame from its variables, its cells (one list per variable, +#: None if unbound) and its row count +RowFactory = Callable[[list[str], list[list[Any]], int], list[Any]] + + +def tuple_rows(variables: list[str], cells: list[list[Any]], rows: int) -> list[Any]: + """Return the rows of a frame as tuples, with one value per variable.""" + if not variables: + return [()] * rows + return list(zip(*cells, strict=True)) + # Methods called through the message classes rather than on messages: upb looks # up any attribute of a message as a field name first @@ -147,8 +157,8 @@ def __init__(self) -> None: self.variables: list[str] = [] #: The boolean result, if the frame has one self.boolean: bool | None = None - #: The rows of the frame: one value per variable, None if it is unbound - self.rows: list[tuple[Any, ...]] = [] + #: The rows of the frame, made by the decoder's row factory + self.rows: list[Any] = [] #: The links describing the result set, if the frame has them self.links: list[str] | None = None #: The trailer: None if the frame has none, else "" if the result set is @@ -163,6 +173,7 @@ class SparqlDecoder: Args: adapter_factory (AdapterFactory): makes the adapter that makes the terms, given the options of the stream. + row_factory (RowFactory): makes the rows of a frame. Tuples by default. max_rows (int): the most rows a frame can have. max_values (int): the most values (rows times variables) a frame can have. Each frame is decoded at once, so this limits the memory it can take. @@ -173,10 +184,12 @@ def __init__( self, adapter_factory: AdapterFactory, *, + row_factory: RowFactory = tuple_rows, max_rows: int = MAX_SPARQL_FRAME_ROWS, max_values: int = MAX_SPARQL_FRAME_VALUES, ) -> None: self.adapter_factory = adapter_factory + self.row_factory = row_factory self.max_rows = min(max_rows, MAX_ROWS) self.max_values = max_values #: The stream options, once read @@ -319,13 +332,13 @@ def read_solutions( if column_count: msg = "a result set with no variables cannot have columns" raise JellyConformanceError(msg) - out.rows = [()] * rows + out.rows = self.row_factory(variables, [], rows) elif column_count == len(variables): cells = [ columns.cells(column, rows, "?" + name) for column, name in zip(frame.columns, variables, strict=True) ] - out.rows = list(zip(*cells, strict=True)) + out.rows = self.row_factory(variables, cells, rows) elif column_count or rows: msg = ( f"the frame has {column_count} columns, but the result set has " @@ -483,9 +496,9 @@ def __init__(self, reader: ResultSetReader, first: DecodedFrame) -> None: #: Whether the stream ended without a trailer after the result set: the #: result set may be cut short. Known once all rows were read. self.truncated = False - self._rows = self._read() - # The rows kept when the next result set was read before they were - self._kept: Iterator[tuple[Any, ...]] = iter(()) + # The rows of each frame, and those of the current frame not yet read + self._frames = self._read() + self._rows: Iterator[Any] = iter(()) # Whether all rows were read by an iteration self._done = False if first.boolean is not None: @@ -513,18 +526,42 @@ def is_boolean(self) -> bool: """Whether this is a boolean result (of an ASK query).""" return self._boolean is not None - def __iter__(self) -> Iterator[tuple[Any, ...]]: + def __iter__(self) -> Iterator[Any]: + self._check_not_done() + while True: + # An iterator of a list: an iteration stopped early leaves it where it + # was, and the next one goes on from there + yield from self._rows + rows = next(self._frames, None) + if rows is None: + break + self._rows = iter(rows) + self._end() + + def collect(self, rows: list[Any]) -> None: + """ + Read the rest of the rows into a list, faster than iterating them. + + Raises: + JellyIncompleteResultsError: if the result set is incomplete, after the + rows are read. + + """ + self._check_not_done() + rows.extend(self._rows) + for frame_rows in self._frames: + rows.extend(frame_rows) + self._end() + + def _check_not_done(self) -> None: if self._done: msg = ( "the rows of the result set were already read: they can be " "iterated only once" ) raise RuntimeError(msg) - # Not `yield from`, which would close them if this iteration is stopped - for row in self._kept: - yield row - for row in self._rows: - yield row + + def _end(self) -> None: self._done = True if self.error: raise _incomplete(self.error, self) @@ -539,8 +576,8 @@ def _truncated(self) -> None: stacklevel=3, ) - def _read(self) -> Iterator[tuple[Any, ...]]: - """Yield the rows, and note the trailers, until the result set ends.""" + def _read(self) -> Iterator[list[Any]]: + """Yield the rows of each frame, and note the trailers, until it ends.""" reader = self.reader frame = self.first self.first = None @@ -558,7 +595,8 @@ def _read(self) -> Iterator[tuple[Any, ...]]: ended = False if frame.links is not None and not self.links: self.links = frame.links - yield from frame.rows + if frame.rows: + yield frame.rows if ended and reader.decoder.punctuated: return frame = reader.next() @@ -572,13 +610,17 @@ def _read(self) -> Iterator[tuple[Any, ...]]: def keep(self) -> None: """Read the rest of the result set, and keep its rows for later.""" if not self._done: - self._kept = chain(self._kept, list(self._rows)) + kept = list(self._rows) + for rows in self._frames: + kept += rows + self._rows = iter(kept) def read_result_sets( inp: IO[bytes] | bytes, adapter_factory: AdapterFactory, *, + row_factory: RowFactory = tuple_rows, max_rows: int = MAX_SPARQL_FRAME_ROWS, max_values: int = MAX_SPARQL_FRAME_VALUES, ) -> Iterator[SparqlResultSet]: @@ -594,6 +636,7 @@ def read_result_sets( inp (IO[bytes] | bytes): the stream, delimited or not. adapter_factory (AdapterFactory): makes the adapter that makes the terms, given the options of the stream. + row_factory (RowFactory): makes the rows. Tuples by default. max_rows (int): the most rows a frame can have. max_values (int): the most values (rows times variables) a frame can have. @@ -601,7 +644,12 @@ def read_result_sets( SparqlResultSet: the result sets, in order. """ - decoder = SparqlDecoder(adapter_factory, max_rows=max_rows, max_values=max_values) + decoder = SparqlDecoder( + adapter_factory, + row_factory=row_factory, + max_rows=max_rows, + max_values=max_values, + ) reader = ResultSetReader(sparql_frames(inp), decoder) while (first := reader.next()) is not None: result_set = SparqlResultSet(reader, first) @@ -613,6 +661,7 @@ def read_result_set( inp: IO[bytes] | bytes, adapter_factory: AdapterFactory, *, + row_factory: RowFactory = tuple_rows, max_rows: int = MAX_SPARQL_FRAME_ROWS, max_values: int = MAX_SPARQL_FRAME_VALUES, ) -> SparqlResultSet: @@ -629,7 +678,11 @@ def read_result_set( """ result_sets = read_result_sets( - inp, adapter_factory, max_rows=max_rows, max_values=max_values + inp, + adapter_factory, + row_factory=row_factory, + max_rows=max_rows, + max_values=max_values, ) result_set = next(result_sets) if result_set.reader.decoder.punctuated: diff --git a/pyjelly/serialize/sparql.py b/pyjelly/serialize/sparql.py index 803884cd..489a3da6 100644 --- a/pyjelly/serialize/sparql.py +++ b/pyjelly/serialize/sparql.py @@ -7,9 +7,9 @@ from __future__ import annotations -from collections.abc import Iterable, Iterator, Sequence +from collections.abc import Iterable, Iterator, Mapping, Sequence from contextlib import suppress -from typing import IO, Final +from typing import IO, Any, Final from pyjelly import jelly from pyjelly.errors import JellyConformanceError, JellyIncompleteResultsError @@ -114,6 +114,8 @@ def __init__( #: The variables of the result set being written, None between result sets self.variables: list[str] | None = None self.columns: list[VariableColumn] = [] + # The values of a row given as a binding, see binding() + self.values: list[object | None] = [] self.row_count = 0 self.rows_per_frame = 1 self.links: list[str] = [] @@ -142,6 +144,7 @@ def start(self, variables: Iterable[str], links: Sequence[str] = ()) -> None: names = variable_names(variables) self.variables = names self.columns = [VariableColumn() for _ in names] + self.values = [None] * len(names) self.rows_per_frame = self.options.rows_per_frame(len(names)) self.header_pending = True self.links = list(links) @@ -211,6 +214,40 @@ def row(self, values: Sequence[object | None]) -> jelly.SparqlResultsFrame | Non return self.end_frame() return frame + def binding( + self, binding: Mapping[Any, object | None], variables: Sequence[object] + ) -> jelly.SparqlResultsFrame | None: + """ + Add a row (solution) given as a binding, which maps variables to values. + + Args: + binding (Mapping[Any, object | None]): the values of the bound + variables. The keys are the variables, or their names. + variables (Sequence[object]): the variables of the result set, in its + order. + + Returns: + jelly.SparqlResultsFrame | None: a frame, if one is full. + + """ + values = self.values + if len(variables) != len(values): + msg = f"expected {len(values)} variables, got {len(variables)}" + raise JellyConformanceError(msg) + bound = 0 + for i in range(len(values)): + value = binding.get(variables[i]) + values[i] = value + if value is not None: + bound += 1 + if bound != len(binding): + # Not all keys are variables: e.g., rdflib's Variable("x") is not equal + # to "x". The values are looked up by name instead. + by_name = {str(key): value for key, value in binding.items()} + for i in range(len(values)): + values[i] = by_name.get(str(variables[i])) + return self.row(values) + def rows( self, rows: Iterable[Sequence[object | None]] ) -> Iterator[jelly.SparqlResultsFrame]: @@ -471,8 +508,47 @@ def write_solutions( # noqa: PLR0913 keyword-only options if frame is not None: writer.write(frame) except Exception as failure: - # If the frame that says so can still be written - with suppress(Exception): - writer.write(stream.end(error_message(failure))) + _end_with(stream, writer, failure) raise writer.write(stream.end(error)) + + +def write_bindings( # noqa: PLR0913 keyword-only options + stream: SparqlStream, + variables: Sequence[object], + bindings: Iterable[Mapping[Any, object | None]], + writer: FrameWriter, + *, + links: Sequence[str] = (), + error: str = "", +) -> None: + """ + Write a solution sequence whose rows are bindings, see write_solutions(). + + Args: + stream (SparqlStream): the stream. + variables (Sequence[object]): the variables, each a str (e.g., rdflib's + Variable), whose names are written. + bindings (Iterable[Mapping[Any, object | None]]): the rows, see + SparqlStream.binding(). + writer (FrameWriter): where the frames go. + links (Sequence[str]): IRIs of links that describe the results. + error (str): why the rows are incomplete, if they are known to be. + + """ + stream.start([str(variable) for variable in variables], links) + try: + for binding in bindings: + frame = stream.binding(binding, variables) + if frame is not None: + writer.write(frame) + except Exception as failure: + _end_with(stream, writer, failure) + raise + writer.write(stream.end(error)) + + +def _end_with(stream: SparqlStream, writer: FrameWriter, failure: Exception) -> None: + """End the result set with an error, if the frame that says so can be written.""" + with suppress(Exception): + writer.write(stream.end(error_message(failure))) From aa1e16f125f464651ba189215f54c5e3e8b2d0c1 Mon Sep 17 00:00:00 2001 From: Ostrzyciel Date: Mon, 26 Oct 2026 18:44:23 +0100 Subject: [PATCH 09/24] Compare rdflib literals by their parts when writing Two literals with the same lexical form, language tag and datatype are the same term, so rdflib's __eq__ (Python code) is only called when these differ. The variables of a dict binding are looked up with dict's own get. Instructions per cell of rdflib writes, geometric mean of 5 datasets: -4.8%. Jelly-RDF writes use the same comparison. The output is the same. --- pyjelly/integrations/rdflib/encode.py | 21 +++++++++++- pyjelly/serialize/sparql.py | 49 +++++++++++++++++---------- 2 files changed, 51 insertions(+), 19 deletions(-) diff --git a/pyjelly/integrations/rdflib/encode.py b/pyjelly/integrations/rdflib/encode.py index 7ac4d5f5..5676ba26 100644 --- a/pyjelly/integrations/rdflib/encode.py +++ b/pyjelly/integrations/rdflib/encode.py @@ -16,6 +16,17 @@ from pyjelly.serialize.encode import EncodedTerm, TermEncoder +def _same_str(a: object, b: object) -> bool: + """Return whether two optional strings (e.g., URIRefs) are the same strings.""" + if a is b: + return True + if a is None or b is None: + return False + a_str: str = cast("str", a) + b_str: str = cast("str", b) + return a_str == b_str + + @mypyc_attr(allow_interpreted_subclasses=True) class RDFLibTermEncoder(TermEncoder): def same_term(self, a: object, b: object) -> bool: @@ -27,7 +38,8 @@ def same_term(self, a: object, b: object) -> bool: - Two URIRefs or two BNodes are equal if their strings are. - Two Literals are never equal if their lexical forms differ. If they are - the same, the language tag and datatype matter too: rdflib decides. + the same, and so are the language tags and datatypes, they are equal. + Otherwise rdflib decides (e.g., language tags differ in case only). - Terms of different classes among these three are never equal. """ if a is b: @@ -44,6 +56,13 @@ def same_term(self, a: object, b: object) -> bool: b_lex: str = cast("str", b) if a_lex != b_lex: return False + # The slots behind the language and datatype properties + a_node: Any = a + b_node: Any = b + if _same_str(a_node._language, b_node._language) and _same_str( + a_node._datatype, b_node._datatype + ): + return True elif (cls is URIRef or cls is BNode or cls is Literal) and ( other_cls is URIRef or other_cls is BNode or other_cls is Literal ): diff --git a/pyjelly/serialize/sparql.py b/pyjelly/serialize/sparql.py index 489a3da6..09deab48 100644 --- a/pyjelly/serialize/sparql.py +++ b/pyjelly/serialize/sparql.py @@ -188,20 +188,8 @@ def row(self, values: Sequence[object | None]) -> jelly.SparqlResultsFrame | Non """ columns = self.columns - if self.variables is None or self.failed: - if self.failed: - raise _failed() - msg = "start a result set with start() before adding rows" - raise JellyConformanceError(msg) count = len(values) - if count != len(columns): - msg = f"expected {len(columns)} values in the row, got {count}" - raise JellyConformanceError(msg) - frame = None - if self.row_count and ( - self.row_count >= MAX_ROWS or not self.encoder.has_room() - ): - frame = self.full_frame() + frame = self._start_row(count) try: for i in range(count): self.add(columns[i], values[i]) @@ -214,6 +202,22 @@ def row(self, values: Sequence[object | None]) -> jelly.SparqlResultsFrame | Non return self.end_frame() return frame + def _start_row(self, count: int) -> jelly.SparqlResultsFrame | None: + """Check that a row of `count` values can be added, and end a full frame.""" + if self.variables is None or self.failed: + if self.failed: + raise _failed() + msg = "start a result set with start() before adding rows" + raise JellyConformanceError(msg) + if count != len(self.columns): + msg = f"expected {len(self.columns)} values in the row, got {count}" + raise JellyConformanceError(msg) + if self.row_count and ( + self.row_count >= MAX_ROWS or not self.encoder.has_room() + ): + return self.full_frame() + return None + def binding( self, binding: Mapping[Any, object | None], variables: Sequence[object] ) -> jelly.SparqlResultsFrame | None: @@ -235,11 +239,20 @@ def binding( msg = f"expected {len(values)} variables, got {len(variables)}" raise JellyConformanceError(msg) bound = 0 - for i in range(len(values)): - value = binding.get(variables[i]) - values[i] = value - if value is not None: - bound += 1 + if type(binding) is dict: + # The same, with dict methods that are not looked up for each call + mapping: dict[Any, object | None] = binding + for i in range(len(values)): + value = mapping.get(variables[i]) + values[i] = value + if value is not None: + bound += 1 + else: + for i in range(len(values)): + value = binding.get(variables[i]) + values[i] = value + if value is not None: + bound += 1 if bound != len(binding): # Not all keys are variables: e.g., rdflib's Variable("x") is not equal # to "x". The values are looked up by name instead. From f39ef5cef1ac6b9cb41a973efd17d90be7169df1 Mon Sep 17 00:00:00 2001 From: Ostrzyciel Date: Mon, 26 Oct 2026 18:48:46 +0100 Subject: [PATCH 10/24] Decode short runs of Jelly-SPARQL cells one by one A few cells are cheaper to append than to slice or repeat as a new list, which the layouts of columns with runs and unbound cells did for every token. A non-delimited stream no longer looks up its options for each row. Instructions per cell of reads, geometric mean of 5 datasets: generic -2.5%, rdflib -1.7% (up to -6% with many runs). The output is the same. --- pyjelly/parse/columns.py | 27 +++++++++++++++++++++++++-- pyjelly/serialize/sparql.py | 9 +++++++-- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/pyjelly/parse/columns.py b/pyjelly/parse/columns.py index 0315632c..bb337fff 100644 --- a/pyjelly/parse/columns.py +++ b/pyjelly/parse/columns.py @@ -96,6 +96,29 @@ def _check_kinds(kinds: bytes, value_count: int, position: str) -> None: raise JellyConformanceError(msg) +# More cells than this are added at once, fewer one by one: cheaper than making a +# list of a few cells to add +_FEW_CELLS: Final = 16 + + +def _add_values(cells: list[Any], values: list[Any], start: int, count: int) -> None: + """Add `count` values, from values[start], one per cell.""" + if count > _FEW_CELLS: + cells += values[start : start + count] + else: + for i in range(start, start + count): + cells.append(values[i]) + + +def _add_run(cells: list[Any], term: Any, count: int) -> None: + """Add `count` cells of one term.""" + if count > _FEW_CELLS: + cells += [term] * count + else: + for _ in range(count): + cells.append(term) + + def _too_many_cells(position: str) -> JellyConformanceError: msg = f"the {position} column has more cells than the frame has rows" return JellyConformanceError(msg) @@ -133,7 +156,7 @@ def _layout_cells( f"corrupt layout of the {position} column: it skips past its values" ) raise JellyConformanceError(msg) - cells += values[index : index + skip] + _add_values(cells, values, index, skip) index += skip if token & _UNBOUND: term = None @@ -151,7 +174,7 @@ def _layout_cells( # Checked before the cells are made, as a run can claim any length if len(cells) + count > rows: raise _too_many_cells(position) - cells += [term] * count + _add_run(cells, term, count) if index < len(values): cells += values[index:] return cells diff --git a/pyjelly/serialize/sparql.py b/pyjelly/serialize/sparql.py index 09deab48..dd36617f 100644 --- a/pyjelly/serialize/sparql.py +++ b/pyjelly/serialize/sparql.py @@ -145,7 +145,12 @@ def start(self, variables: Iterable[str], links: Sequence[str] = ()) -> None: self.variables = names self.columns = [VariableColumn() for _ in names] self.values = [None] * len(names) - self.rows_per_frame = self.options.rows_per_frame(len(names)) + # A non-delimited stream is one frame, which no row count ends + self.rows_per_frame = ( + self.options.rows_per_frame(len(names)) + if self.options.delimited + else MAX_ROWS + 1 + ) self.header_pending = True self.links = list(links) self.new_frame() @@ -198,7 +203,7 @@ def row(self, values: Sequence[object | None]) -> jelly.SparqlResultsFrame | Non self.failed = True raise self.row_count += 1 - if self.row_count >= self.rows_per_frame and self.options.delimited: + if self.row_count >= self.rows_per_frame: return self.end_frame() return frame From a52b00c77f21055f1e4961371732a5a573e6fd1e Mon Sep 17 00:00:00 2001 From: Ostrzyciel Date: Mon, 26 Oct 2026 18:52:47 +0100 Subject: [PATCH 11/24] Index the rows and variables of Jelly-SPARQL as tuples or lists mypyc indexes a value typed as Sequence through the general item lookup, with the index as an int object, which took 12% of a generic write. Rows that are tuples or lists, and lists of variables, are now indexed as such. Instructions per cell of writes, geometric mean of 5 datasets: generic -5.5%, rdflib -5.2%. The output is the same. --- pyjelly/serialize/sparql.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/pyjelly/serialize/sparql.py b/pyjelly/serialize/sparql.py index dd36617f..67646eda 100644 --- a/pyjelly/serialize/sparql.py +++ b/pyjelly/serialize/sparql.py @@ -196,8 +196,19 @@ def row(self, values: Sequence[object | None]) -> jelly.SparqlResultsFrame | Non count = len(values) frame = self._start_row(count) try: - for i in range(count): - self.add(columns[i], values[i]) + # Indexing a tuple or list typed as such is much cheaper than indexing + # any sequence, which goes through a general lookup with an int object + if type(values) is tuple: + row_tuple: tuple[object | None, ...] = values + for i in range(count): + self.add(columns[i], row_tuple[i]) + elif type(values) is list: + row_list: list[object | None] = values + for i in range(count): + self.add(columns[i], row_list[i]) + else: + for i in range(count): + self.add(columns[i], values[i]) except BaseException: # The frame is half-written: it can't be completed self.failed = True @@ -243,18 +254,20 @@ def binding( if len(variables) != len(values): msg = f"expected {len(values)} variables, got {len(variables)}" raise JellyConformanceError(msg) + # A list typed as such is cheaper to index (see row()) + keys: list[object] = variables if type(variables) is list else list(variables) bound = 0 if type(binding) is dict: # The same, with dict methods that are not looked up for each call mapping: dict[Any, object | None] = binding for i in range(len(values)): - value = mapping.get(variables[i]) + value = mapping.get(keys[i]) values[i] = value if value is not None: bound += 1 else: for i in range(len(values)): - value = binding.get(variables[i]) + value = binding.get(keys[i]) values[i] = value if value is not None: bound += 1 From 69aff02a88bc86d1f5bd48e6d779ee187a429b9d Mon Sep 17 00:00:00 2001 From: Ostrzyciel Date: Mon, 26 Oct 2026 18:58:52 +0100 Subject: [PATCH 12/24] Iterate lists rather than index them when decoding columns In code compiled with mypyc, iterating a list is cheaper than indexing it, which checks the bounds each time. Done for the name ids of IRIs (also in Jelly-RDF) and for rdflib's bindings. Instructions per cell of reads, geometric mean of 5 datasets: generic -1.6%, rdflib -2.8%. The output is the same. --- pyjelly/integrations/rdflib/decode.py | 5 ++--- pyjelly/parse/columns.py | 5 +++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pyjelly/integrations/rdflib/decode.py b/pyjelly/integrations/rdflib/decode.py index b02381eb..828479d6 100644 --- a/pyjelly/integrations/rdflib/decode.py +++ b/pyjelly/integrations/rdflib/decode.py @@ -197,9 +197,8 @@ def __call__( variables = self.variables(names) for j in range(len(cells)): variable = variables[j] - column = cells[j] - for r in range(rows): - value = column[r] + # Iterating a list is cheaper than indexing it + for r, value in enumerate(cells[j]): if value is not None: bindings[r][variable] = value return bindings diff --git a/pyjelly/parse/columns.py b/pyjelly/parse/columns.py index bb337fff..a78899e2 100644 --- a/pyjelly/parse/columns.py +++ b/pyjelly/parse/columns.py @@ -642,10 +642,11 @@ def iris(self, column: jelly.RdfColumn) -> list[Any]: name_index = 0 terms: list[Any] = [] try: - for i in range(count): + # Iterating a list is cheaper than indexing it + for i, name_id in enumerate(name_ids): if prefix_count > 1 and prefix_ids[i]: prefix_index = prefix_ids[i] - name_index = name_ids[i] or name_index + 1 + name_index = name_id or name_index + 1 name = names[name_index - 1] prefix = prefixes[prefix_index - 1] if prefix_index else "" if name is None or prefix is None: From 89f8eb65b771f9286262f369461c86917d6ee587 Mon Sep 17 00:00:00 2001 From: Ostrzyciel Date: Mon, 26 Oct 2026 19:03:02 +0100 Subject: [PATCH 13/24] Compare terms of the generic API by their fields when writing Consecutive values of a column are compared to find runs. IRIs, blank nodes and literals of the generic API are now compared by their strings, without a call to their __eq__, and terms of different classes are never equal. Jelly-RDF writes use the same comparison. Instructions per cell of generic writes, geometric mean of 5 datasets: -3.1%. The output is the same. --- pyjelly/integrations/generic/serialize.py | 27 +++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/pyjelly/integrations/generic/serialize.py b/pyjelly/integrations/generic/serialize.py index 7a6feb38..c18b74fd 100644 --- a/pyjelly/integrations/generic/serialize.py +++ b/pyjelly/integrations/generic/serialize.py @@ -30,6 +30,33 @@ class GenericSinkTermEncoder(TermEncoder): + def same_term(self, a: object, b: object) -> bool: + """ + Return whether two terms are equal, for repeated terms. + + IRIs, blank nodes and literals are compared by their strings, without a + call to their __eq__. + """ + if a is b: + return True + cls = type(a) + if cls is not type(b): + return False + if cls is IRI: + return cast(IRI, a)._iri == cast(IRI, b)._iri + if cls is BlankNode: + return cast(BlankNode, a)._id == cast(BlankNode, b)._id + if cls is Literal: + x = cast(Literal, a) + y = cast(Literal, b) + return ( + x._lex == y._lex + and x._langtag == y._langtag + and x._datatype == y._datatype + and x._direction == y._direction + ) + return a == b + def encode_spo(self, term: object, out: EncodedTerm) -> None: """ Encode term based on its GenericSink object. From 056e1c1bf1e58795c7cb9bea609c2067898e77d7 Mon Sep 17 00:00:00 2001 From: Ostrzyciel Date: Mon, 26 Oct 2026 19:07:44 +0100 Subject: [PATCH 14/24] Fix the types of a Jelly-SPARQL test helper The decoder now also takes a row factory, which **kwargs: int doesn't fit. --- tests/unit_tests/test_parse/test_sparql.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/unit_tests/test_parse/test_sparql.py b/tests/unit_tests/test_parse/test_sparql.py index 20a77ba3..03dbf081 100644 --- a/tests/unit_tests/test_parse/test_sparql.py +++ b/tests/unit_tests/test_parse/test_sparql.py @@ -8,14 +8,19 @@ from pyjelly.errors import JellyConformanceError from pyjelly.integrations.generic.decode import GenericStatementSinkAdapter from pyjelly.integrations.generic.generic_sink import IRI +from pyjelly.options import MAX_SPARQL_FRAME_ROWS, MAX_SPARQL_FRAME_VALUES from pyjelly.parse.sparql import SparqlDecoder, read_result_set from pyjelly.serialize.ioutils import write_delimited, write_single OPTIONS = jelly.SparqlResultsOptions(max_name_table_size=128, version=1) -def _decoder(**kwargs: int) -> SparqlDecoder: - return SparqlDecoder(GenericStatementSinkAdapter, **kwargs) +def _decoder( + *, max_rows: int = MAX_SPARQL_FRAME_ROWS, max_values: int = MAX_SPARQL_FRAME_VALUES +) -> SparqlDecoder: + return SparqlDecoder( + GenericStatementSinkAdapter, max_rows=max_rows, max_values=max_values + ) def _frame(**fields: object) -> jelly.SparqlResultsFrame: From 358b4543a7888c3e76cd3743fb036f1eff1d0216 Mon Sep 17 00:00:00 2001 From: Ostrzyciel Date: Mon, 26 Oct 2026 19:10:57 +0100 Subject: [PATCH 15/24] End a full Jelly-SPARQL frame after the row that fills it A frame whose lookups had no room for another row was ended before the next row was encoded. If that row failed, the full frame was never returned, so its rows were lost, and if it was the first frame, the stream had no options. Now the frame ends right after the row that fills it. The frames are the same. --- pyjelly/serialize/sparql.py | 47 +++++++++---------- .../test_generic/test_sparql.py | 23 +++++++++ 2 files changed, 45 insertions(+), 25 deletions(-) diff --git a/pyjelly/serialize/sparql.py b/pyjelly/serialize/sparql.py index 67646eda..958f2744 100644 --- a/pyjelly/serialize/sparql.py +++ b/pyjelly/serialize/sparql.py @@ -118,6 +118,8 @@ def __init__( self.values: list[object | None] = [] self.row_count = 0 self.rows_per_frame = 1 + # Whether the frame of a non-delimited stream has no room for another row + self.full = False self.links: list[str] = [] # Whether the stream options and the header are yet to be written self.options_pending = True @@ -145,11 +147,10 @@ def start(self, variables: Iterable[str], links: Sequence[str] = ()) -> None: self.variables = names self.columns = [VariableColumn() for _ in names] self.values = [None] * len(names) - # A non-delimited stream is one frame, which no row count ends self.rows_per_frame = ( - self.options.rows_per_frame(len(names)) + min(self.options.rows_per_frame(len(names)), MAX_ROWS) if self.options.delimited - else MAX_ROWS + 1 + else MAX_ROWS ) self.header_pending = True self.links = list(links) @@ -179,6 +180,7 @@ def new_frame(self) -> None: self.encoder.new_frame(iris, len(columns)) self.triple_owner = None self.row_count = 0 + self.full = False def row(self, values: Sequence[object | None]) -> jelly.SparqlResultsFrame | None: """ @@ -194,7 +196,7 @@ def row(self, values: Sequence[object | None]) -> jelly.SparqlResultsFrame | Non """ columns = self.columns count = len(values) - frame = self._start_row(count) + self._check_row(count) try: # Indexing a tuple or list typed as such is much cheaper than indexing # any sequence, which goes through a general lookup with an int object @@ -214,12 +216,16 @@ def row(self, values: Sequence[object | None]) -> jelly.SparqlResultsFrame | Non self.failed = True raise self.row_count += 1 - if self.row_count >= self.rows_per_frame: - return self.end_frame() - return frame + # The frame ends after the row that leaves no room for another one, not + # before the next row: if that row fails, the frame is not lost + if self.row_count >= self.rows_per_frame or not self.encoder.has_room(): + if self.options.delimited: + return self.end_frame() + self.full = True + return None - def _start_row(self, count: int) -> jelly.SparqlResultsFrame | None: - """Check that a row of `count` values can be added, and end a full frame.""" + def _check_row(self, count: int) -> None: + """Check that a row of `count` values can be added.""" if self.variables is None or self.failed: if self.failed: raise _failed() @@ -228,11 +234,13 @@ def _start_row(self, count: int) -> jelly.SparqlResultsFrame | None: if count != len(self.columns): msg = f"expected {len(self.columns)} values in the row, got {count}" raise JellyConformanceError(msg) - if self.row_count and ( - self.row_count >= MAX_ROWS or not self.encoder.has_room() - ): - return self.full_frame() - return None + if self.full: + msg = ( + "the results do not fit in one frame: the lookups can't hold all of " + "their terms. A non-delimited stream can only have one frame. Write " + "a delimited stream, or use larger lookups." + ) + raise JellyConformanceError(msg) def binding( self, binding: Mapping[Any, object | None], variables: Sequence[object] @@ -294,17 +302,6 @@ def rows( if frame is not None: yield frame - def full_frame(self) -> jelly.SparqlResultsFrame: - """End the frame, which has no room for another row.""" - if not self.options.delimited: - msg = ( - "the results do not fit in one frame: the lookups can't hold all of " - "their terms. A non-delimited stream can only have one frame. Write " - "a delimited stream, or use larger lookups." - ) - raise JellyConformanceError(msg) - return self.end_frame() - def add(self, column: VariableColumn, term: object | None) -> None: """Add a value to its column, None for an unbound one.""" if term is None: diff --git a/tests/integration_tests/test_generic/test_sparql.py b/tests/integration_tests/test_generic/test_sparql.py index 34315c71..6caf316d 100644 --- a/tests/integration_tests/test_generic/test_sparql.py +++ b/tests/integration_tests/test_generic/test_sparql.py @@ -268,3 +268,26 @@ def test_sequence_writer() -> None: assert list(first) == ROWS assert second.boolean is False assert list(third) == [(A,)] + + +def test_row_failing_after_a_full_frame() -> None: + # 128 names, so the 128th row with a new IRI fills the frame: the frame must + # still be written when the row after it fails + options = SparqlOptions( + lookup_preset=LookupPreset(max_names=128, max_prefixes=0, max_datatypes=0) + ) + good: list[tuple[Node | None, ...]] = [(IRI(f"{EX}{i}"),) for i in range(128)] + + def rows() -> Iterator[tuple[object, ...]]: + yield from good + yield (42,) + + out = io.BytesIO() + with pytest.raises(NotImplementedError): + serialize_jelly_sparql(["x"], rows(), out, options) # type: ignore[arg-type] + out.seek(0) + results = parse_jelly_sparql(out) + read: list[tuple[Node | None, ...]] = [] + with pytest.raises(JellyIncompleteResultsError, match="NotImplementedError"): + read.extend(results) + assert read == good From f6784c85cea7ae88a6774795d69e071a7ffe83f6 Mon Sep 17 00:00:00 2001 From: Ostrzyciel Date: Mon, 26 Oct 2026 20:09:45 +0100 Subject: [PATCH 16/24] Reword the comments and docstrings of Jelly-SPARQL Plain, direct wording instead of roundabout phrases like "Return how many cells write writes". --- pyjelly/errors.py | 14 +- pyjelly/integrations/generic/serialize.py | 6 +- pyjelly/integrations/generic/sparql.py | 50 ++--- pyjelly/integrations/rdflib/decode.py | 8 +- pyjelly/integrations/rdflib/encode.py | 10 +- pyjelly/integrations/rdflib/sparql.py | 102 +++++----- pyjelly/options.py | 45 +++-- pyjelly/parse/columns.py | 26 +-- pyjelly/parse/ioutils.py | 17 +- pyjelly/parse/sparql.py | 179 +++++++++--------- pyjelly/serialize/encode.py | 2 +- pyjelly/serialize/ioutils.py | 8 +- pyjelly/serialize/sparql.py | 169 ++++++++--------- .../test_sparql/test_parse.py | 2 +- .../test_sparql/test_serialize.py | 2 +- .../test_generic/test_sparql.py | 10 +- .../test_rdflib/test_sparql.py | 16 +- tests/unit_tests/test_parse/test_sparql.py | 2 +- .../unit_tests/test_serialize/test_sparql.py | 2 +- tests/utils/sparql_compare.py | 10 +- 20 files changed, 338 insertions(+), 342 deletions(-) diff --git a/pyjelly/errors.py b/pyjelly/errors.py index 6de2a2df..b3c94923 100644 --- a/pyjelly/errors.py +++ b/pyjelly/errors.py @@ -12,23 +12,23 @@ class JellyNotImplementedError(NotImplementedError): class JellyIncompleteResultsError(Exception): """ - Raised when the producer of SPARQL results says that they are incomplete. + Raised when the producer of SPARQL results reports that they are incomplete. - For example, when the query failed midway. The rows read before are valid. + For example, the query failed midway. The rows read before the error are valid. """ def __init__(self, message: str, error: str, result: object = None) -> None: super().__init__(message) - #: Why the producer could not complete the results, in its words + #: Error message from the producer self.error = error - #: The results that were read, if the reader keeps them + #: Results read before the error, if the reader keeps them self.result = result class JellyTruncatedResultsWarning(UserWarning): """ - Issued when a stream of SPARQL results ends without saying they are complete. + Issued when a stream of SPARQL results ends without a trailer. - The writer may have stopped before writing all results, e.g., because the - connection was lost. + Some results may be missing, e.g., because the writer stopped or the connection + was lost. """ diff --git a/pyjelly/integrations/generic/serialize.py b/pyjelly/integrations/generic/serialize.py index c18b74fd..cef90cb5 100644 --- a/pyjelly/integrations/generic/serialize.py +++ b/pyjelly/integrations/generic/serialize.py @@ -32,10 +32,10 @@ class GenericSinkTermEncoder(TermEncoder): def same_term(self, a: object, b: object) -> bool: """ - Return whether two terms are equal, for repeated terms. + Check whether two terms are equal, to find repeated terms. - IRIs, blank nodes and literals are compared by their strings, without a - call to their __eq__. + IRIs, blank nodes and literals are compared field by field, which is + cheaper than calling their __eq__. """ if a is b: return True diff --git a/pyjelly/integrations/generic/sparql.py b/pyjelly/integrations/generic/sparql.py index 12e142a8..7230988e 100644 --- a/pyjelly/integrations/generic/sparql.py +++ b/pyjelly/integrations/generic/sparql.py @@ -1,8 +1,8 @@ """ -Jelly-SPARQL with the generic API: SPARQL query results, without rdflib. +Jelly-SPARQL support for the generic API (SPARQL query results without rdflib). -The values of the results are the terms of the generic API (IRI, Literal, -BlankNode, and Triple for triple terms), and None for unbound variables. +Values are generic API terms (IRI, Literal, BlankNode, and Triple for triple +terms). Unbound variables are None. """ from __future__ import annotations @@ -22,7 +22,7 @@ def parse_jelly_sparql(inp: IO[bytes] | bytes) -> SparqlResultSet: """ Read SPARQL query results from Jelly-SPARQL. - The rows are read as they are iterated (once), so the input must stay open + Rows are read during iteration (only once), so the input must stay open until then. Args: @@ -34,8 +34,9 @@ def parse_jelly_sparql(inp: IO[bytes] | bytes) -> SparqlResultSet: one value per variable, None if the variable is unbound. Raises: - JellyIncompleteResultsError: if the writer said that the results are - incomplete: when the rows were iterated, or now for a boolean result. + JellyIncompleteResultsError: if the writer reported that the results are + incomplete. Raised at the end of the iteration, or right away for a + boolean result. """ return read_result_set(inp, GenericStatementSinkAdapter) @@ -45,10 +46,9 @@ def parse_jelly_sparql_sequence(inp: IO[bytes] | bytes) -> Iterator[SparqlResult """ Read a sequence of SPARQL query results from one Jelly-SPARQL stream. - The stream has the PUNCTUATED type, which holds any number of results (a - stream of the FLAT type, the default, has one). The rows are read as they are - iterated. Rows not yet iterated when the next result set is read are kept in - memory. + This reads PUNCTUATED streams, which hold any number of results. A FLAT + stream (the default) holds one. Rows are read during iteration. If the next + result set is read first, the unread rows are kept in memory. Args: inp (IO[bytes] | bytes): the stream, delimited. @@ -61,7 +61,7 @@ def parse_jelly_sparql_sequence(inp: IO[bytes] | bytes) -> Iterator[SparqlResult def sparql_stream(options: SparqlOptions | None = None) -> SparqlStream: - """Return a stream that writes Jelly-SPARQL with terms of the generic API.""" + """Create a Jelly-SPARQL writer for generic API terms.""" if options is None: options = SparqlOptions() encoder = GenericSinkTermEncoder(lookup_preset=options.lookup_preset) @@ -79,17 +79,17 @@ def serialize_jelly_sparql( """ Write a solution sequence (the result of a SELECT query) in Jelly-SPARQL. - If getting the rows fails midway (e.g., the query fails), the stream ends with - the error, so that readers know the results are incomplete, and the error is - raised. + If the rows fail midway (e.g., the query fails), the stream is ended with the + error so readers know the results are incomplete, and the exception is + re-raised. Args: - variables (Iterable[str]): names of the variables, in projection order. + variables (Iterable[str]): variable names, in projection order. rows (Iterable[Sequence[Node | None]]): the rows, with one value per variable, None if the variable is unbound. - out (IO[bytes]): where to write the results. + out (IO[bytes]): output stream. options (SparqlOptions | None): options of the stream. - links (Sequence[str]): IRIs of links that describe the results. + links (Sequence[str]): link IRIs for the results. """ stream = sparql_stream(options) @@ -109,9 +109,9 @@ def serialize_jelly_sparql_ask( Args: value (bool): the result. - out (IO[bytes]): where to write it. + out (IO[bytes]): output stream. options (SparqlOptions | None): options of the stream. - links (Sequence[str]): IRIs of links that describe the result. + links (Sequence[str]): link IRIs for the result. """ stream = sparql_stream(options) @@ -120,8 +120,8 @@ def serialize_jelly_sparql_ask( ) -#: A result in a sequence of results: the variables and the rows of a solution -#: sequence, or a boolean result +#: One result in a sequence: (variables, rows) for a SELECT query, or a bool for an +#: ASK query SequenceItem = tuple[Iterable[str], Iterable[Sequence[Node | None]]] | bool @@ -133,12 +133,12 @@ def serialize_jelly_sparql_sequence( """ Write a sequence of SPARQL query results in one Jelly-SPARQL stream. - The stream has the PUNCTUATED type: each result is a result set of its own. + The stream is PUNCTUATED, with one result set per result. Args: - results (Iterable[SequenceItem]): the results: for a SELECT query, a pair - of the variables and the rows, and for an ASK query, its boolean result. - out (IO[bytes]): where to write them. + results (Iterable[SequenceItem]): the results, each a (variables, rows) + pair for a SELECT query, or a bool for an ASK query. + out (IO[bytes]): output stream. options (SparqlOptions | None): options of the stream, which must be punctuated. Defaults to SparqlOptions(punctuated=True). diff --git a/pyjelly/integrations/rdflib/decode.py b/pyjelly/integrations/rdflib/decode.py index 828479d6..b7d45fc0 100644 --- a/pyjelly/integrations/rdflib/decode.py +++ b/pyjelly/integrations/rdflib/decode.py @@ -173,10 +173,10 @@ def graph_end(self) -> None: class BindingRows: """ - Makes the rows of Jelly-SPARQL frames as rdflib bindings. + Row factory that turns the cells of Jelly-SPARQL frames into rdflib bindings. - A binding is a dict that maps each bound Variable to its value, as in rdflib's - query results. The same names get the same Variable objects (see variables()). + A binding is a dict from each bound Variable to its value, as in rdflib's query + results. Equal names share one Variable object (see variables()). """ def __init__(self) -> None: @@ -184,7 +184,7 @@ def __init__(self) -> None: self.vars: list[Variable] = [] def variables(self, names: list[str]) -> list[Variable]: - """Return the Variables with these names.""" + """Get the Variable objects for these names.""" if names != self.names: self.names = names self.vars = [Variable(name) for name in names] diff --git a/pyjelly/integrations/rdflib/encode.py b/pyjelly/integrations/rdflib/encode.py index 5676ba26..7e0b3034 100644 --- a/pyjelly/integrations/rdflib/encode.py +++ b/pyjelly/integrations/rdflib/encode.py @@ -17,7 +17,7 @@ def _same_str(a: object, b: object) -> bool: - """Return whether two optional strings (e.g., URIRefs) are the same strings.""" + """Compare two optional strings (e.g., URIRefs) as plain strings.""" if a is b: return True if a is None or b is None: @@ -37,9 +37,9 @@ def same_term(self, a: object, b: object) -> bool: by comparing the terms as strings: - Two URIRefs or two BNodes are equal if their strings are. - - Two Literals are never equal if their lexical forms differ. If they are - the same, and so are the language tags and datatypes, they are equal. - Otherwise rdflib decides (e.g., language tags differ in case only). + - Two Literals with different lexical forms are never equal. With the same + lexical form, language tag and datatype, they are equal. Otherwise rdflib + decides (e.g., for language tags that differ only in case). - Terms of different classes among these three are never equal. """ if a is b: @@ -56,7 +56,7 @@ def same_term(self, a: object, b: object) -> bool: b_lex: str = cast("str", b) if a_lex != b_lex: return False - # The slots behind the language and datatype properties + # Slots behind the language and datatype properties a_node: Any = a b_node: Any = b if _same_str(a_node._language, b_node._language) and _same_str( diff --git a/pyjelly/integrations/rdflib/sparql.py b/pyjelly/integrations/rdflib/sparql.py index f2616599..6819672c 100644 --- a/pyjelly/integrations/rdflib/sparql.py +++ b/pyjelly/integrations/rdflib/sparql.py @@ -1,7 +1,7 @@ """ -Jelly-SPARQL for rdflib: SPARQL query results (rdflib.query.Result) in Jelly. +Jelly-SPARQL support for rdflib's SPARQL query results (rdflib.query.Result). -With pyjelly installed, rdflib reads and writes them in the "jelly" format: +With pyjelly installed, rdflib reads and writes results in the "jelly" format: result = graph.query("SELECT ?s ?o WHERE { ?s ?p ?o }") result.serialize("results.jellys", format="jelly") @@ -31,7 +31,7 @@ ) from pyjelly.serialize.sparql import FrameWriter, SparqlStream, write_bindings -#: The error that a result read from a truncated stream is passed on with +#: Trailer error written for results that were read from a truncated stream TRUNCATED_ERROR = ( "the results were read from a stream that ended without saying that they are " "complete, so some may be missing" @@ -40,15 +40,14 @@ class JellySparqlResult(Result): """ - SPARQL query results read from Jelly-SPARQL: an rdflib Result, with more. + An rdflib Result read from Jelly-SPARQL, with a few extra attributes. Attributes: - links: IRIs of links that describe the results, like head.link of SPARQL - JSON results. - truncated: whether the stream ended without saying that the results are - complete, so some may be missing. - error: why the results are incomplete, if the writer said so. Only set by - parse_jelly_sparql_sequence(): parse_jelly_sparql() raises + links: link IRIs for the results, as in head.link of SPARQL JSON results. + truncated: whether the stream ended without a trailer, so some results may + be missing. + error: error message from the writer, if the results are incomplete. Only + set by parse_jelly_sparql_sequence(). parse_jelly_sparql() raises JellyIncompleteResultsError instead. """ @@ -74,10 +73,10 @@ def _to_result( lazy: bool, raise_error: bool, ) -> JellySparqlResult: - """Return a result set, whose rows are bindings, as an rdflib Result.""" + """Convert a result set with bindings as rows to an rdflib Result.""" if result_set.is_boolean: result = JellySparqlResult("ASK") - # The rest of the stream, which must not have more of the result + # Read to the end of the stream: nothing may follow a boolean result result_set.keep() result.links = result_set.links try: @@ -86,18 +85,18 @@ def _to_result( if raise_error: error.result = result raise - # The value read, which may be wrong + # Keep the value, even though the result is incomplete result.askAnswer = result_set._boolean result.error = error.error result.truncated = result_set.truncated return result result = JellySparqlResult("SELECT") - # The Variables that the bindings have + # The same Variable objects as the keys of the bindings result.vars = binding_rows.variables(result_set.variables) result.links = result_set.links if lazy: - # rdflib makes the bindings as the result is iterated, as for its own - # query results + # rdflib builds the bindings during iteration, as for its own query + # results result.bindings = _read_bindings(result_set, result, raise_error=raise_error) return result bindings: list[Mapping[Variable, Identifier]] = [] @@ -125,7 +124,7 @@ def _incomplete( def _read_bindings( result_set: SparqlResultSet, result: JellySparqlResult, *, raise_error: bool ) -> Iterator[Mapping[Variable, Identifier]]: - """Yield the bindings, and when they end, what the stream said.""" + """Yield the bindings, then record the links, truncation and error.""" try: yield from result_set except JellyIncompleteResultsError as error: @@ -146,17 +145,17 @@ def parse_jelly_sparql( result set (the FLAT stream type, the default). validate_iris (bool): whether rdflib checks the IRIs. An invalid IRI is still read, rdflib only logs a warning. Checking slows down parsing. - lazy (bool): whether to read the rows as the result is iterated, rather - than all of them now. The stream must then stay open until the result - was iterated, and the error of incomplete results is raised then. + lazy (bool): read the rows during iteration instead of all at once. The + stream must stay open until the result has been iterated, and the + error of incomplete results is raised at the end of the iteration. Returns: JellySparqlResult: the results, an rdflib Result. Raises: JellyIncompleteResultsError: if the writer said that the results are - incomplete, e.g., because the query failed midway. The results read - are in its `result`. + incomplete, e.g., because the query failed midway. Its `result` holds + the results read before the error. """ adapter_factory = _adapter_factory(validate_iris=validate_iris) @@ -164,7 +163,7 @@ def parse_jelly_sparql( try: result_set = read_result_set(inp, adapter_factory, row_factory=binding_rows) except JellyIncompleteResultsError as error: - # A boolean result that is incomplete + # An incomplete boolean result result_set = cast("SparqlResultSet", error.result) error.result = _to_result( result_set, binding_rows, lazy=False, raise_error=False @@ -179,11 +178,11 @@ def parse_jelly_sparql_sequence( """ Read a sequence of SPARQL query results from one Jelly-SPARQL stream. - The stream has the PUNCTUATED type, which holds any number of results (a - stream of the FLAT type, the default, has one). + This reads PUNCTUATED streams, which hold any number of results. A FLAT + stream (the default) holds one. - An incomplete result does not end the sequence: it has the reason in its - `error` attribute, so check it, and the results after it are read too. + An incomplete result does not stop the sequence. Instead, its `error` + attribute is set, so check it. The results after it are still read. Args: inp (IO[bytes] | bytes): the stream, delimited. @@ -201,7 +200,7 @@ def parse_jelly_sparql_sequence( def sparql_stream(options: SparqlOptions | None = None) -> SparqlStream: - """Return a stream that writes Jelly-SPARQL with terms of rdflib.""" + """Create a Jelly-SPARQL writer for rdflib terms.""" if options is None: options = SparqlOptions() encoder = RDFLibTermEncoder(lookup_preset=options.lookup_preset) @@ -209,7 +208,7 @@ def sparql_stream(options: SparqlOptions | None = None) -> SparqlStream: def _passed_on_error(result: Result) -> str: - """Return the error of a result that was read from Jelly-SPARQL, if any.""" + """Get the error to pass on for a result read from Jelly-SPARQL, if any.""" error = getattr(result, "error", "") if not error and getattr(result, "truncated", False): return TRUNCATED_ERROR @@ -217,7 +216,7 @@ def _passed_on_error(result: Result) -> str: def _write_result(stream: SparqlStream, result: Result, writer: FrameWriter) -> None: - """Write one result set: the result of a SELECT or ASK query.""" + """Write the result of a SELECT or ASK query as one result set.""" links = getattr(result, "links", ()) if result.type == "ASK": if result.askAnswer is None: @@ -237,7 +236,8 @@ def _write_result(stream: SparqlStream, result: Result, writer: FrameWriter) -> if result.vars is None: msg = "the result of the SELECT query has no variables (vars is None)" raise JellyConformanceError(msg) - # Not result.bindings first: that would make all of them + # Check for the generator first, as reading result.bindings would build all + # bindings at once made = getattr(result, "_genbindings", None) bindings: Iterable[Mapping[Variable, Identifier]] = ( result.bindings if made is None else _made_bindings(result, made) @@ -256,15 +256,15 @@ def _made_bindings( result: Result, made: Iterator[Mapping[Variable, Identifier]] ) -> Iterator[Mapping[Variable, Identifier]]: """ - Return the bindings of a result, as they are made. + Yield the bindings of a result as its generator produces them. - Result.bindings would make all of them before any is written, and if making - one fails, nothing would be written at all. They are still kept in the result, - as Result.__iter__() does. + Result.bindings would build all of them before anything is written, and if one + failed, nothing would be written at all. As in Result.__iter__(), the bindings + are also kept in the result. """ result._genbindings = None kept = result._bindings - # Those made before, e.g. by an iteration of the result that stopped early + # Bindings already built, e.g. by an earlier loop over the result that stopped yield from list(kept) for binding in made: kept.append(binding) @@ -277,14 +277,14 @@ def serialize_jelly_sparql( """ Write SPARQL query results (of a SELECT or ASK query) in Jelly-SPARQL. - If getting the results fails midway (e.g., the query fails), the stream ends - with the error, so that readers know the results are incomplete, and the - error is raised. Results read from Jelly-SPARQL that are incomplete or - truncated are written as incomplete. + If the results fail midway (e.g., the query fails), the stream is ended with + the error so readers know the results are incomplete, and the exception is + re-raised. Results read from Jelly-SPARQL that were incomplete or truncated + are written as incomplete. Args: result (Result): the results. - out (IO[bytes]): where to write them. + out (IO[bytes]): output stream. options (SparqlOptions | None): options of the stream. """ @@ -298,11 +298,11 @@ def serialize_jelly_sparql_sequence( """ Write a sequence of SPARQL query results in one Jelly-SPARQL stream. - The stream has the PUNCTUATED type: each result is a result set of its own. + The stream is PUNCTUATED, with one result set per result. Args: results (Iterable[Result]): the results. - out (IO[bytes]): where to write them. + out (IO[bytes]): output stream. options (SparqlOptions | None): options of the stream, which must be punctuated. Defaults to SparqlOptions(punctuated=True). @@ -319,7 +319,7 @@ def serialize_jelly_sparql_sequence( class RDFLibJellySparqlParser(ResultParser): - """Reads SPARQL query results in Jelly-SPARQL, for rdflib's Result.parse().""" + """rdflib parser plugin for Jelly-SPARQL results, used by Result.parse().""" @override def parse( # type: ignore[override] @@ -334,13 +334,13 @@ def parse( # type: ignore[override] Read the results. Args: - source (IO[bytes] | bytes | str | Path): the stream, its bytes, or the - path of a file. + source (IO[bytes] | bytes | str | Path): a stream, bytes, or a file + path. content_type (str | None): unused. validate_iris (bool): whether rdflib checks the IRIs, see parse_jelly_sparql(). - lazy (bool): whether to read the rows as the result is iterated, see - parse_jelly_sparql(). Only for streams, not paths. + lazy (bool): read the rows during iteration, see parse_jelly_sparql(). + Only for streams, not paths. """ if not isinstance(source, (str, Path)): @@ -353,7 +353,7 @@ def parse( # type: ignore[override] class RDFLibJellySparqlSerializer(ResultSerializer): - """Writes SPARQL query results in Jelly-SPARQL, for rdflib's Result.serialize().""" + """rdflib serializer plugin for Jelly-SPARQL, used by Result.serialize().""" @override def serialize( # type: ignore[override] @@ -367,8 +367,8 @@ def serialize( # type: ignore[override] Write the results. Args: - stream (IO[bytes]): where to write them. - encoding (str): unused, Jelly is binary. + stream (IO[bytes]): output stream. + encoding (str): unused, as Jelly is binary. options (SparqlOptions | None): options of the stream. """ diff --git a/pyjelly/options.py b/pyjelly/options.py index e75d64ff..5bd7cac0 100644 --- a/pyjelly/options.py +++ b/pyjelly/options.py @@ -51,21 +51,21 @@ MIMETYPES = ("application/x-jelly-rdf",) -#: The media type of Jelly-SPARQL +#: Media type of Jelly-SPARQL SPARQL_MIMETYPE: Final = "application/x-jelly-sparql" -#: The file extension of Jelly-SPARQL +#: File extension of Jelly-SPARQL SPARQL_EXTENSION: Final = ".jellys" -# Jelly-SPARQL has version tags of its own: 1 is Jelly-SPARQL 1.0 +# Jelly-SPARQL has its own version tags, separate from Jelly-RDF. 1 is 1.0. SPARQL_VERSION_1_0: Final[int] = 1 SPARQL_MAX_VERSION: Final[int] = SPARQL_VERSION_1_0 -#: How many values (rows times variables) a writer puts in a frame of results +#: Default maximum values (rows times variables) per frame when writing DEFAULT_SPARQL_FRAME_VALUES: Final[int] = 4096 -#: How many rows a reader accepts in one frame of results +#: Maximum rows per frame accepted when reading MAX_SPARQL_FRAME_ROWS: Final[int] = 1 << 20 -#: How many values a reader accepts in one frame of results. It makes room for all -#: of them at once. +#: Maximum values per frame accepted when reading. A frame is decoded all at once, +#: so this caps its memory use. MAX_SPARQL_FRAME_VALUES: Final[int] = 1 << 24 @@ -110,8 +110,9 @@ def small(cls) -> Self: return cls(max_names=128, max_prefixes=32, max_datatypes=32) -#: The lookup sizes of Jelly-SPARQL streams. As in Jelly-JVM, larger than those of -#: Jelly-RDF: a row of results tends to have more distinct terms than a statement. +#: Default lookup sizes of Jelly-SPARQL streams. As in Jelly-JVM, they are larger +#: than for Jelly-RDF, because a row of results usually has more distinct terms +#: than a statement. SPARQL_LOOKUP_PRESET: Final = LookupPreset( max_names=8192, max_prefixes=1024, max_datatypes=64 ) @@ -247,21 +248,19 @@ def validate_type_compatibility( @dataclass(frozen=True) class SparqlOptions: """ - Options of a Jelly-SPARQL stream that is written. + Options for writing a Jelly-SPARQL stream. Attributes: - lookup_preset: the sizes of the lookups. The name lookup must have at - least 128 entries. - rdf_version: the version of RDF whose terms may occur in the stream. - RDF 1.1 does not allow base directions or triple terms, RDF 1.2 Basic - does not allow triple terms. - punctuated: whether the stream is a sequence of result sets (the - PUNCTUATED stream type), rather than one result set. - frame_values: how many values (rows times variables) a frame has, at - most. A frame may end earlier, when the lookups are full. - delimited: whether the frames are delimited, as files and HTTP need. A - non-delimited stream is one frame, e.g. to be one message of a - message broker. Then all results must fit in the lookups. + lookup_preset: lookup sizes. The name lookup needs at least 128 entries. + rdf_version: RDF version of the terms in the stream. RDF 1.1 allows no + base directions or triple terms, RDF 1.2 Basic no triple terms. + punctuated: write a sequence of result sets (the PUNCTUATED stream type) + instead of one result set. + frame_values: maximum values (rows times variables) per frame. A frame + can end earlier if the lookups fill up. + delimited: write delimited frames, which files and HTTP need. A + non-delimited stream is a single frame, e.g. one message for a message + broker. All terms of the results must then fit in the lookups. stream_name: name of the stream. """ @@ -290,7 +289,7 @@ def __post_init__(self) -> None: def rows_per_frame(self, variables: int) -> int: """ - Return how many rows a frame of a result set has, at most. + Compute the maximum rows per frame for this number of variables. >>> SparqlOptions().rows_per_frame(4) 1024 diff --git a/pyjelly/parse/columns.py b/pyjelly/parse/columns.py index a78899e2..03c5df58 100644 --- a/pyjelly/parse/columns.py +++ b/pyjelly/parse/columns.py @@ -96,13 +96,13 @@ def _check_kinds(kinds: bytes, value_count: int, position: str) -> None: raise JellyConformanceError(msg) -# More cells than this are added at once, fewer one by one: cheaper than making a -# list of a few cells to add +# Above this count, cells are added in bulk. Below it, appending them one at a +# time is cheaper than building a temporary list. _FEW_CELLS: Final = 16 def _add_values(cells: list[Any], values: list[Any], start: int, count: int) -> None: - """Add `count` values, from values[start], one per cell.""" + """Append values[start:start + count] to the cells.""" if count > _FEW_CELLS: cells += values[start : start + count] else: @@ -111,7 +111,7 @@ def _add_values(cells: list[Any], values: list[Any], start: int, count: int) -> def _add_run(cells: list[Any], term: Any, count: int) -> None: - """Add `count` cells of one term.""" + """Append `count` cells holding the same term.""" if count > _FEW_CELLS: cells += [term] * count else: @@ -135,7 +135,7 @@ def _kinds_mismatch(position: str, kind: int) -> JellyConformanceError: def _layout_cells( values: list[Any], layouts: list[int], rows: int, position: str ) -> list[Any]: - """Return the cells that the layouts of a column give, in order.""" + """Expand a column's layout tokens into its cells, in order.""" cells: list[Any] = [] index = 0 k = 0 @@ -171,7 +171,7 @@ def _layout_cells( "its values" ) raise JellyConformanceError(msg) - # Checked before the cells are made, as a run can claim any length + # Check before building the cells, since a run can claim any length if len(cells) + count > rows: raise _too_many_cells(position) _add_run(cells, term, count) @@ -314,8 +314,8 @@ def __init__(self, decoder: Decoder) -> None: self.options_read = False # Statements of the current RDF Message so far, in this and earlier frames self.rows_in_message = 0 - #: Put before the labels of blank nodes, to tell apart the blank nodes of - #: result sets that use the same labels (see pyjelly.parse.sparql) + #: Prefix for blank node labels, so that equal labels in different result + #: sets give different blank nodes (see pyjelly.parse.sparql) self.bnode_scope = "" def iter_frame(self, frame: jelly.RdfStreamFrame) -> Iterator[Any]: @@ -435,7 +435,7 @@ def ingest_entries(self, batch: jelly.RdfColumnBatch) -> None: self.apply_entries(batch.names, batch.prefixes, batch.datatypes) def apply_entries(self, names: Any, prefixes: Any, datatypes: Any) -> None: - """Apply the packed lookup entries of a frame.""" + """Apply a frame's packed lookup entries.""" decoder = self.decoder _apply_entries(names, decoder.names) _apply_entries(prefixes, decoder.prefixes) @@ -544,18 +544,18 @@ def runs( def cells(self, column: jelly.RdfColumn, rows: int, position: str) -> list[Any]: """ - Decode a column of Jelly-SPARQL into its cells, in order. + Decode a Jelly-SPARQL column into its cells, in order. - The column can have terms of any type, and unbound cells, which are None. + The column may hold terms of any type. Unbound cells are None. """ values = self.run_values(column, _ANY, position) layouts: list[int] = column.layouts[:] - # Without layouts, each value takes one cell + # Without layout tokens, each value fills one cell cells = _layout_cells(values, layouts, rows, position) if layouts else values if len(cells) > rows: raise _too_many_cells(position) if len(cells) < rows: - # The rest are unbound + # Pad with unbound cells cells += [None] * (rows - len(cells)) return cells diff --git a/pyjelly/parse/ioutils.py b/pyjelly/parse/ioutils.py index f5c539c4..c03e153d 100644 --- a/pyjelly/parse/ioutils.py +++ b/pyjelly/parse/ioutils.py @@ -72,7 +72,7 @@ def single_byte_tags(descriptor: Descriptor) -> frozenset[int]: SPARQL_FRAME_TAGS: Final = single_byte_tags( cast("Descriptor", jelly.SparqlResultsFrame.DESCRIPTOR) ) -"""Tag bytes a non-delimited frame of Jelly-SPARQL can start with.""" +"""Tag bytes that a non-delimited Jelly-SPARQL frame can start with.""" def hint_bytes_needed(first_byte: int, tags: frozenset[int] = FRAME_TAGS) -> int: @@ -288,26 +288,25 @@ def detect_delimited( inp: IO[bytes], tags: frozenset[int] = FRAME_TAGS ) -> tuple[IO[bytes], bool]: """ - Detect whether a stream is delimited, from its first bytes. + Detect from the first bytes whether a stream is delimited. Args: inp (IO[bytes]): the stream tags (frozenset[int]): tag bytes a frame can start with Returns: - tuple[IO[bytes], bool]: the stream to read from its start, and whether it - is delimited + tuple[IO[bytes], bool]: a stream positioned at the start again, and + whether it is delimited """ if inp.seekable(): header = read_hint_header(inp, tags) inp.seek(-len(header), os.SEEK_CUR) else: - # Input may not be seekable (e.g. a network stream) -- then we put the bytes - # we looked at back in front of it. An unbuffered stream gets a buffer, but - # not a buffered one (e.g. from socket.makefile()): the outer buffer would - # wait for the inner one to fill up, so a stream that is still being - # written would not be read as it comes. + # Input may not be seekable (e.g. a network stream). Then the bytes we read + # are put back in front of it. Only an unbuffered stream gets a buffer: a + # second buffer around a buffered stream (e.g. from socket.makefile()) + # waits for the inner one to fill up, which blocks streaming. # See also: https://github.com/Jelly-RDF/pyjelly/issues/298 buffered = inp if not isinstance(inp, io.BufferedIOBase): diff --git a/pyjelly/parse/sparql.py b/pyjelly/parse/sparql.py index b77b4ee9..9a7831f8 100644 --- a/pyjelly/parse/sparql.py +++ b/pyjelly/parse/sparql.py @@ -1,8 +1,8 @@ """ -Reading of Jelly-SPARQL: SPARQL query results in frames, one column per variable. +Jelly-SPARQL reader: SPARQL query results in frames, with one column per variable. -SparqlDecoder decodes the frames one by one, and read_result_sets() puts their rows -together into result sets. +SparqlDecoder decodes frames one at a time. read_result_sets() groups the rows of +the frames into result sets. """ from __future__ import annotations @@ -38,24 +38,24 @@ from pyjelly.parse.decode import Adapter, Decoder, ParserOptions from pyjelly.parse.ioutils import SPARQL_FRAME_TAGS, detect_delimited -#: The metadata key of the links of a result set (head.link in SPARQL JSON results) +#: Metadata key for the links of a result set (head.link in SPARQL JSON results) LINK_KEY: Final = "link" AdapterFactory = Callable[[ParserOptions], Adapter] -#: Makes the rows of a frame from its variables, its cells (one list per variable, -#: None if unbound) and its row count +#: Builds the rows of a frame from the variables, the cells (one list per variable, +#: None for unbound) and the row count RowFactory = Callable[[list[str], list[list[Any]], int], list[Any]] def tuple_rows(variables: list[str], cells: list[list[Any]], rows: int) -> list[Any]: - """Return the rows of a frame as tuples, with one value per variable.""" + """Build the rows of a frame as tuples, with one value per variable.""" if not variables: return [()] * rows return list(zip(*cells, strict=True)) -# Methods called through the message classes rather than on messages: upb looks -# up any attribute of a message as a field name first +# Methods are called through the message classes, not the messages: upb tries +# every attribute name of a message as a field name first _FRAME_HAS_FIELD: Final = jelly.SparqlResultsFrame.HasField _STREAM_TYPES: Final = ( @@ -67,7 +67,7 @@ def tuple_rows(variables: list[str], cells: list[list[Any]], rows: int) -> list[ def check_sparql_options(options: jelly.SparqlResultsOptions) -> None: """ - Check that the parser can read a Jelly-SPARQL stream with these options. + Check that this parser supports a Jelly-SPARQL stream with these options. Raises: JellyConformanceError: if it can't. @@ -109,10 +109,10 @@ def _check_size(lookup: str, size: int, smallest: int, largest: int) -> None: def sparql_parser_options(options: jelly.SparqlResultsOptions) -> ParserOptions: """ - Return the options that the adapter of a Jelly-SPARQL stream is made with. + Build the adapter options for a Jelly-SPARQL stream. - The terms are decoded as in a Jelly-RDF 1.2 stream with the same lookups and - RDF version. The stream types are unspecified. + Terms are decoded like in a Jelly-RDF 1.2 stream with the same lookup sizes and + RDF version. The stream types are left unspecified. """ return ParserOptions( stream_types=StreamTypes(), @@ -131,7 +131,7 @@ def sparql_parser_options(options: jelly.SparqlResultsOptions) -> ParserOptions: def decode_links(value: bytes) -> list[str] | None: r""" - Return the IRIs of the link metadata key, or None if it is not valid UTF-8. + Split the "link" metadata value into IRIs. Return None if it isn't UTF-8. >>> decode_links(b"http://example.org/a\nhttp://example.org/b") ['http://example.org/a', 'http://example.org/b'] @@ -148,35 +148,35 @@ def decode_links(value: bytes) -> list[str] | None: class DecodedFrame: - """What one frame of a Jelly-SPARQL stream holds.""" + """Contents of one decoded Jelly-SPARQL frame.""" def __init__(self) -> None: - #: Whether the frame starts a result set + #: Whether this is the first frame of a result set self.starts = False - #: The variables of the result set, if the frame starts a solution sequence + #: Variables of the result set, if this frame starts a solution sequence self.variables: list[str] = [] - #: The boolean result, if the frame has one + #: Boolean result, if the frame has one self.boolean: bool | None = None - #: The rows of the frame, made by the decoder's row factory + #: Rows of the frame, built by the decoder's row factory self.rows: list[Any] = [] - #: The links describing the result set, if the frame has them + #: Link IRIs for the result set, if the frame has them self.links: list[str] | None = None - #: The trailer: None if the frame has none, else "" if the result set is - #: complete, or why it is not + #: Trailer: None if the frame has none, "" if the result set is complete, + #: otherwise the error message self.trailer: str | None = None class SparqlDecoder: """ - Decodes the frames of a Jelly-SPARQL stream, in order. + Decoder of the frames of a Jelly-SPARQL stream, in stream order. Args: - adapter_factory (AdapterFactory): makes the adapter that makes the terms, - given the options of the stream. - row_factory (RowFactory): makes the rows of a frame. Tuples by default. - max_rows (int): the most rows a frame can have. - max_values (int): the most values (rows times variables) a frame can have. - Each frame is decoded at once, so this limits the memory it can take. + adapter_factory (AdapterFactory): builds the term adapter from the stream + options. + row_factory (RowFactory): builds the rows of a frame. Tuples by default. + max_rows (int): maximum rows per frame. + max_values (int): maximum values (rows times variables) per frame. A frame + is decoded all at once, so this caps its memory use. """ @@ -192,22 +192,22 @@ def __init__( self.row_factory = row_factory self.max_rows = min(max_rows, MAX_ROWS) self.max_values = max_values - #: The stream options, once read + #: Stream options, once read self.options: jelly.SparqlResultsOptions | None = None self.decoder: Decoder | None = None self.columns: ColumnDecoder | None = None #: Whether the stream is a sequence of result sets self.punctuated = False - # The variables of the result set, as the first header declares them, and - # whether a header is in effect. Repeated stream options end its effect. + # Variables declared by the first header of the result set, and whether a + # header is in effect. Repeated stream options cancel the header. self.variables: list[str] | None = None self.header_in_effect = False - # Whether the next frame starts a result set, whether the result set is a - # boolean result, and whether a trailer was read since the stream options + # Whether the next frame starts a result set, whether the current result + # set is a boolean result, and whether a trailer came after the options self.result_set_start = True self.boolean_read = False self.trailer_read = False - # How many result sets were started + # Number of result sets started so far self.result_sets = 0 def decode(self, frame: jelly.SparqlResultsFrame) -> DecodedFrame: @@ -247,7 +247,7 @@ def decode(self, frame: jelly.SparqlResultsFrame) -> DecodedFrame: return out def read_options(self, options: jelly.SparqlResultsOptions) -> None: - """Read the stream options, which empty the lookups if they are repeated.""" + """Read the stream options. Repeated options also empty the lookups.""" check_sparql_options(options) punctuated = options.stream_type == jelly.SPARQL_STREAM_TYPE_PUNCTUATED parser_options = sparql_parser_options(options) @@ -277,13 +277,13 @@ def read_options(self, options: jelly.SparqlResultsOptions) -> None: self.trailer_read = False def start_result_set(self) -> None: - """Forget the result set before, in a PUNCTUATED stream.""" + """Reset the per-result-set state, for the next result set.""" self.variables = None self.header_in_effect = False self.boolean_read = False self.trailer_read = False if self.result_sets: - # Blank node labels are scoped to a result set + # Each result set has its own blank node labels columns = self.columns assert columns is not None columns.bnode_scope = f"r{self.result_sets}-" @@ -298,7 +298,7 @@ def read_boolean(self, frame: jelly.SparqlResultsFrame, out: DecodedFrame) -> No raise JellyConformanceError(msg) self.boolean_read = True out.boolean = frame.ask_result.value - # The next result set of a PUNCTUATED stream may use the entries + # The next result set of a PUNCTUATED stream may refer to these entries columns = self.columns assert columns is not None columns.apply_entries(frame.names, frame.prefixes, frame.datatypes) @@ -325,7 +325,7 @@ def read_solutions( columns = self.columns assert columns is not None - # All lookup entries come first, so that the columns can refer to them + # Apply all lookup entries before decoding the columns that use them columns.apply_entries(frame.names, frame.prefixes, frame.datatypes) column_count = len(frame.columns) if not variables: @@ -349,7 +349,7 @@ def read_solutions( def read_variables( self, frame: jelly.SparqlResultsFrame, *, has_options: bool ) -> list[str]: - """Read the header of the frame, if any, and return the variables.""" + """Read the frame's header, if any, and return the current variables.""" if frame.variables: if not has_options and not self.result_set_start: msg = ( @@ -359,8 +359,8 @@ def read_variables( raise JellyConformanceError(msg) self.read_header(list(frame.variables)) elif not self.header_in_effect: - # No header next to the options, or at the start of a result set, - # declares zero variables + # A missing header next to the options, or at the start of a result + # set, means zero variables if self.variables: msg = ( "the frame repeats the stream options, so it must repeat the " @@ -390,7 +390,7 @@ def read_header(self, names: list[str]) -> None: def sparql_frames(inp: IO[bytes] | bytes) -> Iterator[jelly.SparqlResultsFrame]: """ - Return the frames of a Jelly-SPARQL stream, delimited or not. + Iterate the frames of a Jelly-SPARQL stream, delimited or not. Raises: JellyConformanceError: if the stream is empty, or isn't Jelly-SPARQL. @@ -440,14 +440,14 @@ def _delimited_frames(inp: IO[bytes]) -> Iterator[jelly.SparqlResultsFrame]: class ResultSetReader: - """Decodes the frames of a stream, one at a time, for its result sets.""" + """Reads and decodes frames one at a time, for the result sets.""" def __init__( self, frames: Iterator[jelly.SparqlResultsFrame], decoder: SparqlDecoder ) -> None: self.frames = frames self.decoder = decoder - # A frame that was read, but not yet taken + # A frame read ahead that belongs to the next result set self.pending: DecodedFrame | None = None def next(self) -> DecodedFrame | None: @@ -471,38 +471,38 @@ class SparqlResultSet: """ One result set of a Jelly-SPARQL stream: a solution sequence or a boolean result. - The rows of a solution sequence are read as they are iterated. Each row is a - tuple, with one value per variable, or None if the variable is unbound. An - iteration that stops early can be picked up again, but once all rows were read, - they can't be iterated again. Rows not yet read when the next result set is - read are kept in memory. + Rows are read from the stream lazily, during iteration. Each row is a tuple + with one value per variable, None if the variable is unbound. A loop that + stops early can be resumed later. Once all rows have been read, iterating + again raises an error. If the next result set is read first, the unread rows + are kept in memory. - If the producer ended the result set with an error, iterating it raises + If the producer ended the result set with an error, iterating raises JellyIncompleteResultsError after the last row, and so does `boolean`. If the - stream ended without saying that the result set is complete, `truncated` is - set, and JellyTruncatedResultsWarning is issued. + stream ends without a trailer, `truncated` is set and a + JellyTruncatedResultsWarning is issued. """ def __init__(self, reader: ResultSetReader, first: DecodedFrame) -> None: self.reader = reader self.first: DecodedFrame | None = first - #: The variables, in projection order. Empty for a boolean result. + #: Variables in projection order. Empty for a boolean result. self.variables: list[str] = first.variables self._boolean = first.boolean - #: IRIs of links that describe the result set (head.link in SPARQL JSON) + #: Link IRIs for the result set (head.link in SPARQL JSON) self.links: list[str] = first.links or [] - #: Why the result set is incomplete, if the producer said so + #: Error message from the producer, if the result set is incomplete self.error = "" - #: Whether the stream ended without a trailer after the result set: the - #: result set may be cut short. Known once all rows were read. + #: Whether the stream ended without a trailer, so rows may be missing. + #: Only known after all rows have been read. self.truncated = False - # The rows of each frame, and those of the current frame not yet read + # Row lists of the coming frames, and the unread rows of the current one self._frames = self._read() self._rows: Iterator[Any] = iter(()) - # Whether all rows were read by an iteration + # Whether an iteration has read all rows self._done = False if first.boolean is not None: - # A boolean result is one frame: whether it is complete is known now + # A boolean result is a single frame, so its trailer is already known if first.trailer: self.error = first.trailer elif first.trailer is None: @@ -511,10 +511,10 @@ def __init__(self, reader: ResultSetReader, first: DecodedFrame) -> None: @property def boolean(self) -> bool | None: """ - The boolean result (of an ASK query), None for a solution sequence. + Boolean result of an ASK query, or None for a solution sequence. Raises: - JellyIncompleteResultsError: if the producer could not complete it. + JellyIncompleteResultsError: if the producer reported an error. """ if self.error and self._boolean is not None: @@ -529,8 +529,8 @@ def is_boolean(self) -> bool: def __iter__(self) -> Iterator[Any]: self._check_not_done() while True: - # An iterator of a list: an iteration stopped early leaves it where it - # was, and the next one goes on from there + # A list iterator keeps its position when a loop stops early, so the + # next loop resumes from there yield from self._rows rows = next(self._frames, None) if rows is None: @@ -540,11 +540,11 @@ def __iter__(self) -> Iterator[Any]: def collect(self, rows: list[Any]) -> None: """ - Read the rest of the rows into a list, faster than iterating them. + Append all remaining rows to a list. Faster than iterating. Raises: - JellyIncompleteResultsError: if the result set is incomplete, after the - rows are read. + JellyIncompleteResultsError: if the result set is incomplete. The rows + are appended first. """ self._check_not_done() @@ -577,7 +577,7 @@ def _truncated(self) -> None: ) def _read(self) -> Iterator[list[Any]]: - """Yield the rows of each frame, and note the trailers, until it ends.""" + """Yield each frame's rows until the result set ends, recording trailers.""" reader = self.reader frame = self.first self.first = None @@ -586,12 +586,12 @@ def _read(self) -> Iterator[list[Any]]: if frame.trailer is not None: ended = True if frame.trailer: - # An error stays even if the result set goes on, in a stream - # made by concatenating streams + # Keep the first error, even if more rows follow in a + # concatenated stream self.error = self.error or frame.trailer else: - # In a FLAT stream, rows can follow a trailer if the stream options - # are repeated (concatenated streams): it no longer ends the stream + # In a FLAT stream, repeated options after a trailer continue the + # result set (concatenated streams), so the trailer no longer counts ended = False if frame.links is not None and not self.links: self.links = frame.links @@ -601,14 +601,14 @@ def _read(self) -> Iterator[list[Any]]: return frame = reader.next() if frame is not None and frame.starts: - # The next result set of a PUNCTUATED stream + # This frame starts the next result set of a PUNCTUATED stream reader.pending = frame return if not ended: self._truncated() def keep(self) -> None: - """Read the rest of the result set, and keep its rows for later.""" + """Read the remaining rows into memory, for iteration later.""" if not self._done: kept = list(self._rows) for rows in self._frames: @@ -627,18 +627,17 @@ def read_result_sets( """ Read the result sets of a Jelly-SPARQL stream. - A stream of the FLAT type (the default) has one result set, one of the - PUNCTUATED type has any number of them. Each result set is read when the one - before was read whole: moving on to the next result set reads the rest of the - rows, and keeps them in memory. + A FLAT stream (the default) has one result set, a PUNCTUATED stream any + number. Moving on to the next result set reads the rest of the current one + and keeps its unread rows in memory. Args: inp (IO[bytes] | bytes): the stream, delimited or not. - adapter_factory (AdapterFactory): makes the adapter that makes the terms, - given the options of the stream. - row_factory (RowFactory): makes the rows. Tuples by default. - max_rows (int): the most rows a frame can have. - max_values (int): the most values (rows times variables) a frame can have. + adapter_factory (AdapterFactory): builds the term adapter from the stream + options. + row_factory (RowFactory): builds the rows. Tuples by default. + max_rows (int): maximum rows per frame. + max_values (int): maximum values (rows times variables) per frame. Yields: SparqlResultSet: the result sets, in order. @@ -666,14 +665,14 @@ def read_result_set( max_values: int = MAX_SPARQL_FRAME_VALUES, ) -> SparqlResultSet: """ - Read the result set of a Jelly-SPARQL stream of the FLAT type. + Read the single result set of a FLAT Jelly-SPARQL stream. - The first frame is read right away, the rows as they are iterated. To read a - stream of the PUNCTUATED type, use read_result_sets(). + The first frame is read now, the rows during iteration. For PUNCTUATED + streams, use read_result_sets(). Raises: - JellyConformanceError: if the stream is of the PUNCTUATED type. - JellyIncompleteResultsError: if it is a boolean result, which the producer + JellyConformanceError: if the stream is PUNCTUATED. + JellyIncompleteResultsError: for a boolean result that the producer could not complete. """ diff --git a/pyjelly/serialize/encode.py b/pyjelly/serialize/encode.py index f0dea44a..262968b7 100644 --- a/pyjelly/serialize/encode.py +++ b/pyjelly/serialize/encode.py @@ -167,7 +167,7 @@ def new_frame(self, iris: int, literals: int = 1) -> None: Args: iris (int): the most IRIs that one statement can have. - literals (int): the most literals that one statement can have. In a + literals (int): maximum number of literals in one statement or row. In a statement, only the object can be a literal. """ diff --git a/pyjelly/serialize/ioutils.py b/pyjelly/serialize/ioutils.py index 7af210e8..829bb98c 100644 --- a/pyjelly/serialize/ioutils.py +++ b/pyjelly/serialize/ioutils.py @@ -8,7 +8,7 @@ def varint(value: int) -> bytes: r""" - Return a non-negative integer as a protobuf varint. + Encode a non-negative integer as a protobuf varint. >>> varint(300) b'\xac\x02' @@ -22,9 +22,9 @@ def varint(value: int) -> bytes: def write_delimited(frame: Message, output_stream: IO[bytes]) -> None: - """Write a frame, after its size (the delimited variant of Jelly).""" - # Not protobuf's serialize_length_prefixed(), which needs write() to return the - # number of bytes written. Not every stream does, e.g., rdflib's for results. + """Write a frame prefixed with its size (the delimited variant of Jelly).""" + # protobuf's serialize_length_prefixed() needs write() to return the number of + # bytes written, which not every stream does (e.g., rdflib's result streams) data = frame.SerializeToString() output_stream.write(varint(len(data))) output_stream.write(data) diff --git a/pyjelly/serialize/sparql.py b/pyjelly/serialize/sparql.py index 958f2744..180cde50 100644 --- a/pyjelly/serialize/sparql.py +++ b/pyjelly/serialize/sparql.py @@ -1,8 +1,8 @@ """ -Writing of Jelly-SPARQL: SPARQL query results in frames, one column per variable. +Jelly-SPARQL writer: SPARQL query results in frames, with one column per variable. -The columns are those of Jelly-RDF 1.2 (see pyjelly.serialize.columns), in which -any cell may be unbound. +The columns use the Jelly-RDF 1.2 column encoding (pyjelly.serialize.columns), +except that any cell may be unbound. """ from __future__ import annotations @@ -18,16 +18,16 @@ from pyjelly.serialize.encode import EncodedTerm, TermEncoder from pyjelly.serialize.ioutils import write_delimited, write_single -#: The metadata key of the links of a result set (head.link in SPARQL JSON results) +#: Metadata key for the links of a result set (head.link in SPARQL JSON results) LINK_KEY: Final = "link" -# A column of results can have terms of any type +# A result column accepts every term type _ANY: Final = 0b1111 def links_value(links: Sequence[str]) -> bytes: r""" - Return the value of the link metadata key: the IRIs, separated by LF. + Join link IRIs with LF, for the "link" metadata key. >>> links_value(["http://example.org/a", "http://example.org/b"]) b'http://example.org/a\nhttp://example.org/b' @@ -41,7 +41,7 @@ def links_value(links: Sequence[str]) -> bytes: def variable_names(variables: Iterable[str]) -> list[str]: """ - Return the names of variables as written: without a leading "?" or "$". + Strip a leading "?" or "$" from each variable name. >>> variable_names(["?s", "$p", "o"]) ['s', 'p', 'o'] @@ -67,32 +67,32 @@ def _failed() -> JellyConformanceError: class VariableColumn(Column): - """The cells of one variable in the frame being built.""" + """Cells of one variable in the frame being built.""" def __init__(self) -> None: super().__init__(_ANY, "variable") - # The IRI sequence of the column's triple terms: the indices of the last IRI, - # while another column's triple terms are encoded (see TermEncoder) + # Prefix and name indices of the last IRI in this column's triple terms. + # The encoder holds them while it encodes this column (see TermEncoder). self.triple_prefix_index = -1 self.triple_name_index = 0 - # The most IRIs a triple term of the column had, so far in the result set + # Largest number of IRIs in one triple term of this column so far self.max_triple_iris = 1 class SparqlStream: """ - Writes SPARQL query results as the frames of a Jelly-SPARQL stream. + Writer of SPARQL query results as Jelly-SPARQL frames. - A result set is a solution sequence (the result of a SELECT query) or a boolean - result (ASK). Start a solution sequence with start(), add its rows with row(), - and end it with end(). row() returns a frame when one is full, end() returns the - last frame of the result set. A boolean result is one frame, made by ask(). + A result set is either a solution sequence (SELECT) or a boolean result (ASK). + For a solution sequence, call start(), then row() for each row, then end(). + row() returns a frame when one is full, and end() returns the last frame. ask() + returns the single frame of a boolean result. - A stream has one result set, or with SparqlOptions(punctuated=True), any number - of them. + A stream holds one result set, or any number of them with + SparqlOptions(punctuated=True). Args: - encoder (TermEncoder): encoder of the terms, with the lookup sizes of the + encoder (TermEncoder): term encoder, with the same lookup sizes as the options. options (SparqlOptions | None): options of the stream. @@ -111,24 +111,24 @@ def __init__( encoder.column_layout = True encoder.rdf_version = options.rdf_version self.encoded = EncodedTerm() - #: The variables of the result set being written, None between result sets + #: Variables of the current result set, None between result sets self.variables: list[str] | None = None self.columns: list[VariableColumn] = [] - # The values of a row given as a binding, see binding() + # Reused buffer for the values of a binding, see binding() self.values: list[object | None] = [] self.row_count = 0 self.rows_per_frame = 1 - # Whether the frame of a non-delimited stream has no room for another row + # Set when the single frame of a non-delimited stream is full self.full = False self.links: list[str] = [] - # Whether the stream options and the header are yet to be written + # Whether the stream options and the header still have to be written self.options_pending = True self.header_pending = False - # Whether nothing more can be written: the result set of a FLAT stream ended, - # or a row failed to encode + # `ended`: the single result set of a FLAT stream is done. `failed`: a row + # failed to encode. Either way, nothing more can be written. self.ended = False self.failed = False - # The column whose IRI sequence of triple terms the encoder has, if any + # Column whose triple-term IRI state the encoder currently holds self.triple_owner: VariableColumn | None = None def start(self, variables: Iterable[str], links: Sequence[str] = ()) -> None: @@ -136,10 +136,10 @@ def start(self, variables: Iterable[str], links: Sequence[str] = ()) -> None: Start a solution sequence. Args: - variables (Iterable[str]): names of the variables, in projection order. A - leading "?" or "$" is left out. - links (Sequence[str]): IRIs of links describing the result set, like - head.link of SPARQL JSON results. + variables (Iterable[str]): variable names, in projection order. A + leading "?" or "$" is removed. + links (Sequence[str]): link IRIs for the result set, as in head.link of + SPARQL JSON results. """ self._check_can_start() @@ -173,7 +173,7 @@ def new_frame(self) -> None: columns = self.columns iris = len(columns) for column in columns: - # A cell of a triple term can need that many IRIs + # A triple term can take more than one IRI per cell iris += column.max_triple_iris - 1 column.triple_prefix_index = -1 column.triple_name_index = 0 @@ -187,19 +187,19 @@ def row(self, values: Sequence[object | None]) -> jelly.SparqlResultsFrame | Non Add a row (solution) to the result set. Args: - values (Sequence[object | None]): the values of the variables, None for - an unbound variable. + values (Sequence[object | None]): one value per variable, None if the + variable is unbound. Returns: - jelly.SparqlResultsFrame | None: a frame, if one is full. + jelly.SparqlResultsFrame | None: the frame, if this row filled it. """ columns = self.columns count = len(values) self._check_row(count) try: - # Indexing a tuple or list typed as such is much cheaper than indexing - # any sequence, which goes through a general lookup with an int object + # mypyc indexes a typed tuple or list directly. Any other sequence goes + # through the generic item lookup with a boxed int, which is slow. if type(values) is tuple: row_tuple: tuple[object | None, ...] = values for i in range(count): @@ -212,12 +212,12 @@ def row(self, values: Sequence[object | None]) -> jelly.SparqlResultsFrame | Non for i in range(count): self.add(columns[i], values[i]) except BaseException: - # The frame is half-written: it can't be completed + # The frame is half-written and can't be finished self.failed = True raise self.row_count += 1 - # The frame ends after the row that leaves no room for another one, not - # before the next row: if that row fails, the frame is not lost + # End the frame as soon as it's full, rather than at the start of the next + # row, so that a failing next row can't lose it if self.row_count >= self.rows_per_frame or not self.encoder.has_room(): if self.options.delimited: return self.end_frame() @@ -246,27 +246,26 @@ def binding( self, binding: Mapping[Any, object | None], variables: Sequence[object] ) -> jelly.SparqlResultsFrame | None: """ - Add a row (solution) given as a binding, which maps variables to values. + Add a row (solution) given as a mapping from variables to values. Args: - binding (Mapping[Any, object | None]): the values of the bound - variables. The keys are the variables, or their names. - variables (Sequence[object]): the variables of the result set, in its - order. + binding (Mapping[Any, object | None]): values of the bound variables, + keyed by variable or by variable name. + variables (Sequence[object]): variables of the result set, in order. Returns: - jelly.SparqlResultsFrame | None: a frame, if one is full. + jelly.SparqlResultsFrame | None: the frame, if this row filled it. """ values = self.values if len(variables) != len(values): msg = f"expected {len(values)} variables, got {len(variables)}" raise JellyConformanceError(msg) - # A list typed as such is cheaper to index (see row()) + # A typed list is cheaper to index, see row() keys: list[object] = variables if type(variables) is list else list(variables) bound = 0 if type(binding) is dict: - # The same, with dict methods that are not looked up for each call + # Same loop, but with direct dict calls instead of method lookups mapping: dict[Any, object | None] = binding for i in range(len(values)): value = mapping.get(keys[i]) @@ -280,8 +279,8 @@ def binding( if value is not None: bound += 1 if bound != len(binding): - # Not all keys are variables: e.g., rdflib's Variable("x") is not equal - # to "x". The values are looked up by name instead. + # Some keys are not the variable objects themselves (rdflib's + # Variable("x") does not equal "x"), so look the values up by name by_name = {str(key): value for key, value in binding.items()} for i in range(len(values)): values[i] = by_name.get(str(variables[i])) @@ -291,10 +290,10 @@ def rows( self, rows: Iterable[Sequence[object | None]] ) -> Iterator[jelly.SparqlResultsFrame]: """ - Add rows to the result set, see row(). + Add many rows, see row(). Yields: - jelly.SparqlResultsFrame: full frames. + jelly.SparqlResultsFrame: each frame as it fills up. """ for values in rows: @@ -303,7 +302,7 @@ def rows( yield frame def add(self, column: VariableColumn, term: object | None) -> None: - """Add a value to its column, None for an unbound one.""" + """Add a value to a column. None is an unbound cell.""" if term is None: if not column.run_unbound: column.end_run() @@ -329,7 +328,7 @@ def add(self, column: VariableColumn, term: object | None) -> None: self._triple_term_added(column) def _switch_triple_iris(self, column: VariableColumn) -> None: - """Give the encoder the IRI sequence of a column's triple terms.""" + """Swap the encoder's triple-term IRI state over to another column.""" owner = self.triple_owner assert owner is not None encoder = self.encoder @@ -345,23 +344,23 @@ def _triple_term_added(self, column: VariableColumn) -> None: self.triple_owner = column iris = self.encoder.triple_iris if iris > column.max_triple_iris: - # From now on, each row may need this many IRIs for the column + # Every following row may need this many IRIs for the column self.encoder.reserve_iris(iris - column.max_triple_iris) column.max_triple_iris = iris def end_frame(self) -> jelly.SparqlResultsFrame: - """End the current frame of the result set, and return it.""" + """End the current frame and return it.""" if self.variables is None or self.failed: if self.failed: raise _failed() - msg = "there is no result set to end a frame of" + msg = "there is no result set in progress" raise JellyConformanceError(msg) frame = jelly.SparqlResultsFrame() self.write_frame(frame) return frame def write_frame(self, frame: jelly.SparqlResultsFrame) -> None: - """Write the frame's options, header, lookup entries and rows.""" + """Fill in the frame: options, header, lookup entries and rows.""" self.write_start(frame) encoder = self.encoder row_count = self.row_count @@ -369,14 +368,14 @@ def write_frame(self, frame: jelly.SparqlResultsFrame) -> None: encoder.names.packed.write(frame.names) encoder.prefixes.packed.write(frame.prefixes) encoder.datatypes.packed.write(frame.datatypes) - # A frame with no rows needs no columns + # Columns are left out of a frame with no rows if row_count: for column in self.columns: column.write(frame.columns.add()) self.new_frame() def write_start(self, frame: jelly.SparqlResultsFrame) -> None: - """Write the stream options and the header, if the frame needs them.""" + """Add the stream options and the header, if they are still due.""" if self.options_pending: frame.options.CopyFrom(self.stream_options()) self.options_pending = False @@ -389,22 +388,22 @@ def write_start(self, frame: jelly.SparqlResultsFrame) -> None: def end(self, error: str = "") -> jelly.SparqlResultsFrame: """ - End the result set, and return its last frame. + End the result set and return its last frame, which has the trailer. Args: - error (str): why the result set is incomplete, if it is. Readers then - know that some results are missing. + error (str): reason the result set is incomplete, if it is. It tells + readers that some results are missing. Returns: - jelly.SparqlResultsFrame: the last frame, with the trailer. + jelly.SparqlResultsFrame: the last frame. """ frame = jelly.SparqlResultsFrame() if self.failed: if not error or self.variables is None: raise _failed() - # The rows of the frame are dropped, with the lookup entries they added, - # so nothing can follow + # Drop the frame's rows and their lookup entries. The lookups are then + # out of sync with the reader, so the stream must end here. self.write_start(frame) self.ended = True elif self.variables is None: @@ -426,15 +425,15 @@ def ask( error: str = "", ) -> jelly.SparqlResultsFrame: """ - Write a boolean result (of an ASK query), and return its frame. + Return the frame of a boolean result (of an ASK query). Args: value (bool): the result. - links (Sequence[str]): IRIs of links describing the result. - error (str): why the result is incomplete, if it is. + links (Sequence[str]): link IRIs for the result. + error (str): reason the result is incomplete, if it is. Returns: - jelly.SparqlResultsFrame: the only frame of the result set. + jelly.SparqlResultsFrame: the result's only frame. """ self._check_can_start() @@ -469,7 +468,7 @@ def stream_options(self) -> jelly.SparqlResultsOptions: def error_message(error: BaseException) -> str: """ - Return what a trailer says about an error that left the results incomplete. + Format an exception as the error message of a trailer. >>> error_message(KeyError("x")) "KeyError: 'x'" @@ -477,7 +476,7 @@ def error_message(error: BaseException) -> str: 'timeout' """ if isinstance(error, JellyIncompleteResultsError): - # Results passed on from a stream that was incomplete itself + # Passing on the error of a stream that was itself incomplete return error.error message = str(error) name = type(error).__name__ @@ -486,10 +485,10 @@ def error_message(error: BaseException) -> str: class FrameWriter: """ - Writes the frames of a stream to a file: delimited, or as its only frame. + Writer of frames to a file, either delimited or as a single frame. - The file is flushed after each frame, so that a reader at the other end of a - connection gets the rows as they are written. + Each frame is flushed, so a reader on the other end of a connection gets the + rows right away. """ def __init__(self, out: IO[bytes], *, delimited: bool) -> None: @@ -523,11 +522,11 @@ def write_solutions( # noqa: PLR0913 keyword-only options error: str = "", ) -> None: """ - Write a solution sequence (the result of a SELECT query) as a result set. + Write a solution sequence (the result of a SELECT query) as one result set. - If getting the rows fails midway (e.g., the query fails), the result set ends - with the error, so that readers know it is incomplete, and the error is - raised. `error` says why the rows are incomplete, if they are known to be. + If iterating the rows raises (e.g., the query fails midway), the result set is + ended with that error so readers know it is incomplete, and the exception is + re-raised. Pass `error` if the rows are already known to be incomplete. """ stream.start(variables, links) try: @@ -551,17 +550,17 @@ def write_bindings( # noqa: PLR0913 keyword-only options error: str = "", ) -> None: """ - Write a solution sequence whose rows are bindings, see write_solutions(). + Like write_solutions(), with rows given as bindings. Args: stream (SparqlStream): the stream. - variables (Sequence[object]): the variables, each a str (e.g., rdflib's - Variable), whose names are written. + variables (Sequence[object]): the variables, as str objects (e.g., rdflib's + Variable). Their names are written. bindings (Iterable[Mapping[Any, object | None]]): the rows, see SparqlStream.binding(). - writer (FrameWriter): where the frames go. - links (Sequence[str]): IRIs of links that describe the results. - error (str): why the rows are incomplete, if they are known to be. + writer (FrameWriter): writer of the frames. + links (Sequence[str]): link IRIs for the results. + error (str): reason the rows are incomplete, if known. """ stream.start([str(variable) for variable in variables], links) @@ -577,6 +576,6 @@ def write_bindings( # noqa: PLR0913 keyword-only options def _end_with(stream: SparqlStream, writer: FrameWriter, failure: Exception) -> None: - """End the result set with an error, if the frame that says so can be written.""" + """End the result set with an error, if writing the frame still works.""" with suppress(Exception): writer.write(stream.end(error_message(failure))) diff --git a/tests/conformance_tests/test_sparql/test_parse.py b/tests/conformance_tests/test_sparql/test_parse.py index 5def36ce..3b957b3e 100644 --- a/tests/conformance_tests/test_sparql/test_parse.py +++ b/tests/conformance_tests/test_sparql/test_parse.py @@ -66,7 +66,7 @@ def _read_all_rdflib(file: IO[bytes], *, punctuated: bool) -> None: if not punctuated: Result.parse(file, format="jelly") return - # Results that are incomplete don't end a sequence, they say so + # An incomplete result doesn't stop a sequence, it sets `error` instead for result in parse_jelly_sparql_sequence(file): if result.error: raise JellyIncompleteResultsError(result.error, result.error) diff --git a/tests/conformance_tests/test_sparql/test_serialize.py b/tests/conformance_tests/test_sparql/test_serialize.py index 717deab0..3d2f2622 100644 --- a/tests/conformance_tests/test_sparql/test_serialize.py +++ b/tests/conformance_tests/test_sparql/test_serialize.py @@ -51,7 +51,7 @@ def _stream_options(case: ConformanceCase) -> jelly.SparqlResultsOptions: def _options(case: ConformanceCase) -> SparqlOptions: - """Return the options of a test, as written in its stream options file.""" + """Read a test's options from its stream options file.""" options = _stream_options(case) return SparqlOptions( lookup_preset=LookupPreset( diff --git a/tests/integration_tests/test_generic/test_sparql.py b/tests/integration_tests/test_generic/test_sparql.py index 6caf316d..2eb102fb 100644 --- a/tests/integration_tests/test_generic/test_sparql.py +++ b/tests/integration_tests/test_generic/test_sparql.py @@ -90,7 +90,7 @@ def test_rdf_version() -> None: def test_triple_terms_in_several_columns() -> None: - # Each column has an IRI sequence of its own in its triple terms + # Each column keeps its own IRI state for its triple terms t1 = Triple(IRI(EX + "s1"), P, IRI(EX + "o1")) t2 = Triple(IRI(EX + "s2"), IRI(EX + "q"), Triple(A, P, IRI(EX + "o2"))) rows: list[tuple[Node | None, ...]] = [ @@ -167,7 +167,7 @@ def test_incomplete() -> None: rows: list[tuple[Node | None, ...]] = [] with pytest.raises(JellyIncompleteResultsError, match="timeout"): rows.extend(results) - # The rows before the error are read + # The rows before the error were read assert rows == ROWS[:2] assert results.error == "TimeoutError: timeout" @@ -187,7 +187,7 @@ def test_sequence_of_result_sets() -> None: assert third.variables == ["x"] assert third.links == [EX + "l"] (row,) = list(third) - # Blank node labels are scoped to a result set + # Each result set has its own blank node labels assert row[0] != ROWS[2][0] @@ -271,8 +271,8 @@ def test_sequence_writer() -> None: def test_row_failing_after_a_full_frame() -> None: - # 128 names, so the 128th row with a new IRI fills the frame: the frame must - # still be written when the row after it fails + # With 128 names, the 128th row with a new IRI fills the frame. The frame must + # still be written when the next row fails. options = SparqlOptions( lookup_preset=LookupPreset(max_names=128, max_prefixes=0, max_datatypes=0) ) diff --git a/tests/integration_tests/test_rdflib/test_sparql.py b/tests/integration_tests/test_rdflib/test_sparql.py index bb583fee..88271e9c 100644 --- a/tests/integration_tests/test_rdflib/test_sparql.py +++ b/tests/integration_tests/test_rdflib/test_sparql.py @@ -1,4 +1,4 @@ -"""Jelly-SPARQL with rdflib: query results written and read through the plugins.""" +"""Jelly-SPARQL with rdflib: writing and reading query results via the plugins.""" from __future__ import annotations @@ -102,7 +102,7 @@ def test_files(graph: Graph, tmp_path: Path) -> None: result = graph.query(SELECT) path = tmp_path / "results.jellys" result.serialize(str(path), format="jelly") - # Paths work too, but rdflib's types say only streams do + # Paths work too, although rdflib's type hints only allow streams assert Result.parse(str(path), format="jelly") == result # type: ignore[arg-type] assert Result.parse(path, format="jelly") == result # type: ignore[arg-type] with path.open("rb") as file: @@ -183,7 +183,7 @@ def test_truncated(graph: Graph) -> None: result = graph.query(SELECT) data = result.serialize(format="jelly", options=SparqlOptions(frame_values=3)) assert data is not None - # The stream without its last frame, the one with the trailer + # Cut off the last frame, which has the trailer frames = list(sparql_frames(io.BytesIO(data))) cut = data[: -(frames[-1].ByteSize() + 1)] parsed = Result.parse(io.BytesIO(cut), format="jelly") @@ -281,7 +281,7 @@ def test_validate_iris_is_passed_on(caplog: pytest.LogCaptureFixture) -> None: def test_frames_of_many_terms() -> None: - # More distinct IRIs than the name lookup has: frames end early + # More distinct IRIs than the name lookup holds, so frames end early options = SparqlOptions( lookup_preset=LookupPreset(max_names=128, max_prefixes=8, max_datatypes=8) ) @@ -343,7 +343,7 @@ def test_lazy_incomplete() -> None: parsed = parse_jelly_sparql(out, lazy=True) rows: list[object] = [] with pytest.raises(JellyIncompleteResultsError): # noqa: PT012 - # Not rows.extend(parsed): it would call len(parsed), which reads all rows + # Not rows.extend(parsed): that calls len(parsed), which reads all rows for row in parsed: rows.append(row) # noqa: PERF402 assert len(rows) == 1 @@ -395,7 +395,7 @@ def test_sequence_goes_on_after_incomplete_results(graph: Graph) -> None: assert not first.error assert second.error == "RuntimeError: the endpoint went away" assert len(second) == 1 - # Not equal: the blank nodes of a result set in a sequence are its own + # Not equal, because each result set in a sequence has its own blank nodes assert len(third) == len(graph.query(SELECT)) @@ -421,7 +421,7 @@ def test_incomplete_and_truncated_results_are_passed_on(graph: Graph) -> None: with pytest.warns(JellyTruncatedResultsWarning): parsed = Result.parse(io.BytesIO(cut), format="jelly") assert parsed.truncated - # Written again, the results say that they may be incomplete + # When written again, the results are marked as possibly incomplete again = parsed.serialize(format="jelly") assert again is not None with pytest.raises(JellyIncompleteResultsError, match="ended without"): @@ -472,7 +472,7 @@ def produce() -> None: frame = stream.row([URIRef(EX + "a")]) assert frame is not None writer.write(frame) - # The reader has the first row before the rest is written + # The reader gets the first row before the rest is written waited.append(first_row_read.wait(timeout=10)) writer.write(stream.end()) diff --git a/tests/unit_tests/test_parse/test_sparql.py b/tests/unit_tests/test_parse/test_sparql.py index 03dbf081..0cfa86de 100644 --- a/tests/unit_tests/test_parse/test_sparql.py +++ b/tests/unit_tests/test_parse/test_sparql.py @@ -34,7 +34,7 @@ def test_row_limit() -> None: def test_value_limit() -> None: - # Checked before the cells are made + # Checked before any cells are built decoder = _decoder(max_values=100) frame = _frame(variables=["x", "y"], row_count=51) with pytest.raises(JellyConformanceError, match="100 values allowed"): diff --git a/tests/unit_tests/test_serialize/test_sparql.py b/tests/unit_tests/test_serialize/test_sparql.py index fde1c2c9..47577112 100644 --- a/tests/unit_tests/test_serialize/test_sparql.py +++ b/tests/unit_tests/test_serialize/test_sparql.py @@ -100,7 +100,7 @@ def test_failed_row() -> None: with pytest.raises(JellyConformanceError, match="failed to encode"): stream.end() frame = stream.end("the row failed") - # The rows of the frame are dropped + # The frame's rows are dropped assert frame.row_count == 0 assert frame.HasField("options") assert list(frame.variables) == ["x"] diff --git a/tests/utils/sparql_compare.py b/tests/utils/sparql_compare.py index 3cd0298b..ef72853f 100644 --- a/tests/utils/sparql_compare.py +++ b/tests/utils/sparql_compare.py @@ -1,5 +1,5 @@ """ -Reading SPARQL JSON results and comparing results, for Jelly-SPARQL tests. +Helpers for Jelly-SPARQL tests: reading SPARQL JSON results and comparing results. Two results are equivalent if both are boolean results with the same value, or if they have the same variables and the same rows in the same order, with a @@ -37,7 +37,7 @@ @dataclass class Results: - """Results, with their terms as in rdf_compare.""" + """Query results, with terms in the form used by rdf_compare.""" variables: list[str] = field(default_factory=list) rows: list[Row] = field(default_factory=list) @@ -83,7 +83,7 @@ def read_srj(path: Path) -> Results: def generic_node(term: Term) -> Node: - """Return the term of the generic API for a term.""" + """Convert a term to a generic API term.""" kind = term[0] if kind == "iri": return IRI(term[1]) @@ -98,7 +98,7 @@ def generic_node(term: Term) -> Node: def generic_rows(results: Results) -> list[tuple[Node | None, ...]]: - """Return the rows with terms of the generic API.""" + """Convert the rows to generic API terms.""" return [ tuple(None if term is None else generic_node(term) for term in row) for row in results.rows @@ -106,7 +106,7 @@ def generic_rows(results: Results) -> list[tuple[Node | None, ...]]: def generic_results(result_set: SparqlResultSet) -> Results: - """Read a result set of the generic API whole.""" + """Read all rows of a generic API result set.""" rows = [ tuple(None if value is None else generic_term(value) for value in row) for row in result_set From 40deae7aeb0d6baf3714743a630b38bfd04dcaa9 Mon Sep 17 00:00:00 2001 From: Ostrzyciel Date: Mon, 26 Oct 2026 20:37:01 +0100 Subject: [PATCH 17/24] Add a read-only rdflib-tsv method to the Jelly-SPARQL benchmarks rdflib reads SPARQL TSV results but can't write them, so the TSV it reads is written by a helper, and writing is skipped. Also rewords the comments of the benchmark code. --- benchmarks/README.md | 3 ++ benchmarks/__main__.py | 2 +- benchmarks/measure.py | 5 ++- benchmarks/report.py | 6 ++-- benchmarks/sparql_data.py | 66 +++++++++++++++++----------------- benchmarks/sparql_methods.py | 69 ++++++++++++++++++++++++++++++------ benchmarks/suites.py | 4 +-- 7 files changed, 104 insertions(+), 51 deletions(-) diff --git a/benchmarks/README.md b/benchmarks/README.md index 8447d195..ae86c603 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -135,6 +135,7 @@ Everything else (processes, iterations, sweeps, cores, the round-trip check) is | `rdflib-srj` | rdflib's SPARQL JSON results: `Result.serialize(format="json")` | `Result.parse(format="json")` | | `rdflib-srx` | rdflib's SPARQL XML results | `Result.parse(format="xml")` | | `rdflib-csv` | rdflib's SPARQL CSV results | `Result.parse(format="csv")` | +| `rdflib-tsv` | - (rdflib can't write TSV results) | `Result.parse(format="tsv")` | | `rdflib-jelly` | Jelly-SPARQL through rdflib's plugin API: `Result.serialize(format="jelly")` | `Result.parse(format="jelly")` | | `generic-jelly` | Jelly-SPARQL with pyjelly's generic API, from a list of rows | `parse_jelly_sparql` of the generic API | @@ -142,6 +143,8 @@ The rdflib methods write a `Result` whose bindings are a list of dicts, made bef CSV has no term types: every value that is not an IRI or a blank node is read back as a simple literal. The round-trip check then prints a warning, as in Jelly-JVM. +rdflib reads SPARQL TSV results but can't write them, so `rdflib-tsv` is only measured reading. The TSV it reads is written by a helper in `sparql_methods.py`. + ### Throughput Throughput counts cells, that is, rows times variables, as in Jelly-JVM. A result set without variables counts one cell per row. diff --git a/benchmarks/__main__.py b/benchmarks/__main__.py index c12cfc8e..fb6db343 100644 --- a/benchmarks/__main__.py +++ b/benchmarks/__main__.py @@ -88,7 +88,7 @@ def _add_timing(parser: argparse.ArgumentParser) -> None: def _fill_selection(args: argparse.Namespace, parser: argparse.ArgumentParser) -> None: - """Choose the datasets and methods of the suite, unless given.""" + """Fill in the suite's default datasets and methods, if none are given.""" suite = SUITES[args.suite] args.datasets = args.datasets or list(suite.default_datasets) args.methods = args.methods or list(suite.methods) diff --git a/benchmarks/measure.py b/benchmarks/measure.py index 11cf1ddd..6ddf0081 100644 --- a/benchmarks/measure.py +++ b/benchmarks/measure.py @@ -62,7 +62,7 @@ class CaseResult: #: The CPU cores the case ran on cpus: list[int] | None = None sweep: int = 0 - #: "rdf" or "sparql", see suites.py + #: Suite of the case: "rdf" or "sparql", see suites.py suite: str = "rdf" @@ -115,6 +115,9 @@ def run_case( # noqa: PLR0913 if not method.supports(data): result.skipped = "the format can't hold this kind of statements" return result + if task == "write" and not getattr(method, "can_write", True): + result.skipped = "the library can't write the format" + return result # Setup, not measured: make the terms (and the graph, for writers that need # one), check that the method round-trips the data, and make the bytes to read diff --git a/benchmarks/report.py b/benchmarks/report.py index 7cd08547..c3f13672 100644 --- a/benchmarks/report.py +++ b/benchmarks/report.py @@ -118,7 +118,7 @@ def size_tables( datasets: Sequence[str], methods: Sequence[str], rows: int, suite: str = "rdf" ) -> str: """ - Return tables of serialized sizes in bytes per statement (or cell). + Build tables of serialized sizes in bytes per statement (or cell). Always divided by the number of statements given to the writer, also when the writer dropped duplicates. @@ -129,11 +129,11 @@ def size_tables( sizes: dict[str, dict[tuple[str, str], float]] = defaultdict(dict) for dataset in datasets: data = benchmarks.load(dataset, rows) - # As many as the writer is given: statements, or cells + # Number of statements (or cells) given to the writer count = data.cells if suite == "sparql" else rows for name in methods: method = benchmarks.methods[name] - if not method.supports(data): + if not method.supports(data) or not getattr(method, "can_write", True): continue written = method.write_bytes(data) for compression, compress in compressors.items(): diff --git a/benchmarks/sparql_data.py b/benchmarks/sparql_data.py index aeb39342..6360ec9f 100644 --- a/benchmarks/sparql_data.py +++ b/benchmarks/sparql_data.py @@ -22,7 +22,7 @@ from benchmarks import data as rdf_data from pyjelly.integrations.generic import generic_sink as generic -# --- The generator, ported from Jelly-JVM's SparqlDataGen --- +# --- Result set generator, ported from Jelly-JVM's SparqlDataGen --- IRI = "iri" BNODE = "bnode" @@ -32,8 +32,8 @@ DIR_LANG_LITERAL = "dirlang" TRIPLE_TERM = "triple" -# How the unbound cells of a column are spread: each cell on its own, every -# second cell, or blocks of `block_size` cells +# Placement of the unbound cells in a column: random cells, every second cell, or +# blocks of `block_size` cells RANDOM = "random" ALTERNATING = "alternating" BLOCKS = "blocks" @@ -45,28 +45,28 @@ @dataclass(frozen=True) class ColumnSpec: - """One variable of a generated result set. The defaults: dense IRIs.""" + """Settings of one variable of a generated result set. Defaults: IRIs, all bound.""" kind: str = IRI - #: How many distinct values the column draws from + #: Number of distinct values in the column distinct_values: int = 1024 - #: How many IRI namespaces, datatypes, or language tags + #: Number of IRI namespaces, datatypes, or language tags namespaces: int = 1 #: Length of the local name of IRIs, or of the lexical form of literals value_length: int = 12 - #: How many of the cells are bound + #: Fraction of the cells that are bound bound_fraction: float = 1.0 sparsity: str = RANDOM block_size: int = 1 - #: Mean length of a run of one value in consecutive rows + #: Mean number of consecutive rows with the same value run_length: float = 1.0 - #: Whether runs have random lengths, or exactly `run_length` + #: Whether run lengths are random, rather than exactly `run_length` jitter_runs: bool = True - #: Whether values are drawn in order (as with ORDER BY) or at random + #: Whether values are taken in order (as with ORDER BY), rather than at random sorted: bool = False - #: How many of the bound cells are of another term type + #: Fraction of the bound cells that have another term type mix_fraction: float = 0.0 - #: The first row with a value of the other term type + #: First row that may have a value of the other term type mix_start_row: int = 0 @@ -94,7 +94,7 @@ def _iri(column: ColumnSpec, j: int, name: str) -> generic.IRI: def _pool(column: ColumnSpec, kind: str) -> list[Any]: - """Return the distinct values a column draws from, as generic terms.""" + """Build the distinct values of a column, as generic terms.""" n = max(1, column.distinct_values) length = max(1, column.value_length) if kind == IRI: @@ -124,7 +124,7 @@ def _literal_pool(column: ColumnSpec, kind: str) -> list[Any]: for j in range(n) ] assert kind == DIR_LANG_LITERAL - # Each "namespace" is one combination of a language tag and a direction + # Here each "namespace" is one pair of a language tag and a direction return [ generic.Literal( _pad(f"value {j} ", length), @@ -137,7 +137,7 @@ def _literal_pool(column: ColumnSpec, kind: str) -> list[Any]: def _triple_term(column: ColumnSpec, j: int) -> generic.Triple: - """Return a triple term with fresh IRIs, some nested.""" + """Build a triple term with fresh IRIs, sometimes nested.""" o: Any if j % 3 == 0: o = _iri(column, j, f"o{j}") @@ -178,7 +178,7 @@ def run_length() -> int: return 1 if not column.jitter_runs: return max(1, round(column.run_length)) - # Geometric, with the mean run_length + # Geometric distribution with mean run_length p = 1 / column.run_length n = 1 while rnd.random() > p and n < 4096: # noqa: PLR2004 @@ -200,7 +200,7 @@ def run_length() -> int: def generate(spec: ResultSetSpec) -> list[tuple[Any, ...]]: - """Return the rows of a result set, with generic terms and None if unbound.""" + """Generate the rows of a result set: generic terms, None if unbound.""" if not spec.columns: return [()] * spec.rows columns = [ @@ -215,14 +215,14 @@ def generate(spec: ResultSetSpec) -> list[tuple[Any, ...]]: PRESETS: dict[str, ResultSetSpec] = { spec.name: spec for spec in ( - # IRIs: the lookups, and the prefix and name inference + # IRIs: lookups, and prefix and name inference ResultSetSpec("iri-sorted", (ColumnSpec(distinct_values=_N, sorted=True),), _N), ResultSetSpec("iri-shuffled", (ColumnSpec(distinct_values=_N),), _N), ResultSetSpec("iri-small-pool", (ColumnSpec(distinct_values=2048),), _N), ResultSetSpec( "iri-many-ns", (ColumnSpec(distinct_values=2048, namespaces=32),), _N ), - # Repeated values: the runs of the layouts + # Repeated values: runs in the layouts ResultSetSpec( "runs-2", (ColumnSpec(distinct_values=2048, run_length=2, sorted=True),), @@ -321,7 +321,7 @@ def generate(spec: ResultSetSpec) -> list[tuple[Any, ...]]: ), _N, ), - # The shape of the result set + # Shape of the result set ResultSetSpec("wide-5", (ColumnSpec(distinct_values=2048),) * 5, _N), ResultSetSpec("wide-20", (ColumnSpec(distinct_values=2048),) * 20, _N // 2), ResultSetSpec("single-row", (ColumnSpec(distinct_values=8),) * 3, 1), @@ -345,8 +345,8 @@ def generate(spec: ResultSetSpec) -> list[tuple[Any, ...]]: ) } -#: The datasets of a run, unless chosen: all RiverBench datasets, and the presets -#: that look most like the results of real queries +#: Datasets used when none are given: all RiverBench datasets, and the presets +#: closest to the results of real queries DEFAULT_DATASETS = ( *rdf_data.DATASETS, "realistic-mixed", @@ -360,7 +360,7 @@ def generate(spec: ResultSetSpec) -> list[tuple[Any, ...]]: def _resized(spec: ResultSetSpec, rows: int) -> ResultSetSpec: - """Change the row count of a preset, but not its shape.""" + """Resize a preset to `rows` rows, keeping its shape.""" factor = rows / spec.rows return replace( spec, @@ -372,7 +372,7 @@ def _resized(spec: ResultSetSpec, rows: int) -> ResultSetSpec: ) -# --- The results, in the terms of each library --- +# --- Results in the terms of each library --- def _rdflib_term(term: Any) -> Any: @@ -386,7 +386,7 @@ def _rdflib_term(term: Any) -> Any: if term._direction: msg = "rdflib has no literals with a base direction" raise TypeError(msg) - # As pyjelly reads them: the lexical form stays as it is + # Keep the lexical form unchanged, as pyjelly's reader does return Literal( term._lex, lang=term._langtag, datatype=term._datatype, normalize=False ) @@ -397,19 +397,19 @@ def _rdflib_term(term: Any) -> Any: @dataclass class SparqlBenchData: """ - A result set: its variables and rows, with generic terms and rdflib terms. + A result set: its variables, and its rows as generic terms and rdflib terms. - `cells` counts what the throughput is measured in: rows times variables, and - rows for a result set without variables. + Throughput is measured in `cells`: rows times variables, or just rows for a + result set without variables. """ - #: As in the data of RDF, which the benchmark code also reads + #: Always False. The shared benchmark code reads this attribute of RDF data. quads: ClassVar[bool] = False name: str variables: list[str] generic: list[tuple[Any, ...]] = field(repr=False) - #: The rows with rdflib terms, if not made from the generic ones + #: Rows as rdflib terms, if not converted from the generic ones rdflib_rows: list[tuple[Any, ...]] | None = field(default=None, repr=False) @property @@ -418,7 +418,7 @@ def cells(self) -> int: @cached_property def rdf_12(self) -> bool: - """Whether the results have terms of RDF 1.2, which rdflib has not.""" + """Whether the results have RDF 1.2 terms, which rdflib doesn't support.""" return any( isinstance(term, generic.Triple) or (isinstance(term, generic.Literal) and term._direction) @@ -428,7 +428,7 @@ def rdf_12(self) -> bool: @cached_property def rdflib(self) -> Result: - """The results as an rdflib Result, with the bindings in a list.""" + """The results as an rdflib Result, with a list of bindings.""" result = Result("SELECT") variables = [Variable(name) for name in self.variables] result.vars = variables @@ -447,7 +447,7 @@ def rdflib(self) -> Result: def load(dataset: str, rows: int = rdf_data.DEFAULT_ROWS) -> SparqlBenchData: - """Return the first `rows` rows of a dataset.""" + """Load the first `rows` rows of a dataset.""" if dataset in PRESETS: spec = _resized(PRESETS[dataset], rows) return SparqlBenchData(dataset, spec.variables, generate(spec)) diff --git a/benchmarks/sparql_methods.py b/benchmarks/sparql_methods.py index 40acc67e..c6d50160 100644 --- a/benchmarks/sparql_methods.py +++ b/benchmarks/sparql_methods.py @@ -19,24 +19,28 @@ from benchmarks.sparql_data import SparqlBenchData from pyjelly.integrations.generic import generic_sink from pyjelly.integrations.generic import sparql as generic_sparql +from pyjelly.options import STRING_DATATYPE_IRI class SparqlMethod(ABC): name: str + #: Whether the library can write the format. If not, `write` is only a helper + #: that produces the input for reading, and writing isn't measured. + can_write: bool = True def supports(self, data: SparqlBenchData) -> bool: # noqa: ARG002 return True @abstractmethod def rows(self, data: SparqlBenchData) -> list[tuple[Any, ...]]: - """Return the rows in this method's term model, None if unbound.""" + """Get the rows as this method's terms, None if unbound.""" def prepare(self, data: SparqlBenchData) -> None: - """Make everything the writer starts from, so that it isn't measured.""" + """Build the writer's input in advance, outside the measurement.""" self.rows(data) def write_count(self, data: SparqlBenchData) -> int: - """Return how many cells `write` writes.""" + """Count the cells that `write` writes.""" return data.cells @abstractmethod @@ -46,14 +50,14 @@ def write(self, data: SparqlBenchData, out: IO[bytes]) -> None: @abstractmethod def read(self, data: bytes, *, quads: bool = False) -> int: """ - Read the rows, do nothing with them, and return how many cells there were. + Read the rows without keeping them, and return the number of cells read. - `quads` is not used: it is there to read like the methods of RDF. + `quads` is unused. It's there to match the RDF methods. """ @abstractmethod def read_all(self, data: bytes) -> list[tuple[Any, ...]]: - """Read the rows and return them, for checking.""" + """Read and return the rows, for the round-trip check.""" def write_bytes(self, data: SparqlBenchData) -> bytes: out = io.BytesIO() @@ -66,7 +70,7 @@ def _cells(rows: int, variables: int) -> int: class RdflibResults(SparqlMethod): - """A result format of rdflib, written and read with Result.serialize and parse.""" + """An rdflib result format, written with Result.serialize, read with parse.""" def __init__(self, name: str, rdflib_format: str) -> None: self.name = name @@ -101,8 +105,50 @@ def read_all(self, data: bytes) -> list[tuple[Any, ...]]: return [tuple(b.get(v) for v in variables) for b in result.bindings] +def _tsv_term(term: Any) -> str: + """Format a term for SPARQL TSV results, in N-Triples syntax.""" + if term is None: + return "" + if isinstance(term, URIRef): + return f"<{term}>" + if isinstance(term, BNode): + return f"_:{term}" + lex = ( + str(term) + .replace("\\", "\\\\") + .replace('"', '\\"') + .replace("\n", "\\n") + .replace("\r", "\\r") + .replace("\t", "\\t") + ) + if term.language: + return f'"{lex}"@{term.language}' + if term.datatype is not None and str(term.datatype) != STRING_DATATYPE_IRI: + return f'"{lex}"^^<{term.datatype}>' + return f'"{lex}"' + + +class RdflibTsv(RdflibResults): + """ + SPARQL TSV results, read by rdflib. + + rdflib has no TSV writer, so `write` is a helper written for these benchmarks. + """ + + can_write = False + + def __init__(self) -> None: + super().__init__("rdflib-tsv", "tsv") + + @override + def write(self, data: SparqlBenchData, out: IO[bytes]) -> None: + lines = ["\t".join(f"?{name}" for name in data.variables)] + lines += ["\t".join(map(_tsv_term, row)) for row in self.rows(data)] + out.write(("\n".join(lines) + "\n").encode()) + + class GenericJelly(SparqlMethod): - """Jelly-SPARQL with pyjelly's generic API, with its own terms.""" + """Jelly-SPARQL with pyjelly's generic API and its own terms.""" name = "generic-jelly" @@ -133,6 +179,7 @@ def read_all(self, data: bytes) -> list[tuple[Any, ...]]: RdflibResults("rdflib-srj", "json"), RdflibResults("rdflib-srx", "xml"), RdflibResults("rdflib-csv", "csv"), + RdflibTsv(), # Jelly through rdflib's plugin API: Result.serialize and Result.parse RdflibResults("rdflib-jelly", "jelly"), GenericJelly(), @@ -141,7 +188,7 @@ def read_all(self, data: bytes) -> list[tuple[Any, ...]]: def _normalize(term: Any) -> Any: - """Make terms comparable. Blank node labels are not compared.""" + """Normalize a term for comparison. Blank node labels are ignored.""" if term is None: return None if isinstance(term, (BNode, generic_sink.BlankNode)): @@ -157,8 +204,8 @@ def sparql_round_trip( """ Write the result set with the method, read it back, and compare the rows. - Returns the written bytes, and a list of problems found (empty if none). - Raises the reader's exception if the method can't read its own output. + Returns the written bytes and a list of problems (empty if none). Raises the + reader's exception if the method can't read its own output. """ written = method.write_bytes(data) expected = [tuple(map(_normalize, row)) for row in method.rows(data)] diff --git a/benchmarks/suites.py b/benchmarks/suites.py index f0e5bf2f..3beab54e 100644 --- a/benchmarks/suites.py +++ b/benchmarks/suites.py @@ -1,4 +1,4 @@ -"""The suites of benchmarks: Jelly-RDF and Jelly-SPARQL, each with other formats.""" +"""Benchmark suites: Jelly-RDF and Jelly-SPARQL, each compared with other formats.""" from __future__ import annotations @@ -14,7 +14,7 @@ @dataclass(frozen=True) class Suite: name: str - #: What the throughput counts + #: Unit of the throughput: statements or cells unit: str datasets: tuple[str, ...] default_datasets: tuple[str, ...] From 66f66fc529ae6cf366dce90be0351fe745cdbde8 Mon Sep 17 00:00:00 2001 From: Ostrzyciel Date: Mon, 26 Oct 2026 20:43:12 +0100 Subject: [PATCH 18/24] Accept "jellys" as a format name of Jelly-SPARQL in rdflib Result.parse() and Result.serialize() take format="jellys" too, after the file extension. "jelly" stays the documented name. --- pyproject.toml | 2 ++ tests/integration_tests/test_rdflib/test_sparql.py | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 732f59b5..9c99b2df 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -128,10 +128,12 @@ jelly = "pyjelly.integrations.rdflib.serialize:RDFLibJellySerializer" [project.entry-points."rdf.plugins.resultparser"] jelly = "pyjelly.integrations.rdflib.sparql:RDFLibJellySparqlParser" +jellys = "pyjelly.integrations.rdflib.sparql:RDFLibJellySparqlParser" "application/x-jelly-sparql" = "pyjelly.integrations.rdflib.sparql:RDFLibJellySparqlParser" [project.entry-points."rdf.plugins.resultserializer"] jelly = "pyjelly.integrations.rdflib.sparql:RDFLibJellySparqlSerializer" +jellys = "pyjelly.integrations.rdflib.sparql:RDFLibJellySparqlSerializer" "application/x-jelly-sparql" = "pyjelly.integrations.rdflib.sparql:RDFLibJellySparqlSerializer" [dependency-groups] diff --git a/tests/integration_tests/test_rdflib/test_sparql.py b/tests/integration_tests/test_rdflib/test_sparql.py index 88271e9c..9fc0090e 100644 --- a/tests/integration_tests/test_rdflib/test_sparql.py +++ b/tests/integration_tests/test_rdflib/test_sparql.py @@ -88,6 +88,14 @@ def test_ask_round_trip(graph: Graph, query: str) -> None: assert bool(parsed) is bool(result) +@pytest.mark.parametrize("name", ["application/x-jelly-sparql", "jellys"]) +def test_other_format_names(graph: Graph, name: str) -> None: + result = graph.query(SELECT) + data = result.serialize(format=name) + assert data is not None + assert Result.parse(io.BytesIO(data), format=name) == result + + def test_media_type(graph: Graph) -> None: result = graph.query(SELECT) data = result.serialize(format="application/x-jelly-sparql") From dbc5b64328f41c96db4bc5db6f8cd2c1fbdc24d4 Mon Sep 17 00:00:00 2001 From: Ostrzyciel Date: Mon, 26 Oct 2026 21:00:53 +0100 Subject: [PATCH 19/24] Report incomplete Jelly-SPARQL results when junk follows the error trailer When Fuseki cancels a query (e.g., on a timeout), Jelly-JVM ends the stream with an error trailer, and then Fuseki appends a text message to the response. The reader went on after the trailer, as concatenated streams may follow, and raised a "stream is not valid" error instead. If the trailer already said that the results are incomplete, the reader now stops there and raises JellyIncompleteResultsError, with the rows read before. --- pyjelly/parse/sparql.py | 14 ++++++++++++- .../test_generic/test_sparql.py | 21 +++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/pyjelly/parse/sparql.py b/pyjelly/parse/sparql.py index 9a7831f8..2bb98826 100644 --- a/pyjelly/parse/sparql.py +++ b/pyjelly/parse/sparql.py @@ -599,7 +599,7 @@ def _read(self) -> Iterator[list[Any]]: yield frame.rows if ended and reader.decoder.punctuated: return - frame = reader.next() + frame = self._next_frame(ended=ended) if frame is not None and frame.starts: # This frame starts the next result set of a PUNCTUATED stream reader.pending = frame @@ -607,6 +607,18 @@ def _read(self) -> Iterator[list[Any]]: if not ended: self._truncated() + def _next_frame(self, *, ended: bool) -> DecodedFrame | None: + """Read the next frame, or None at the end of the stream.""" + try: + return self.reader.next() + except JellyConformanceError: + if ended and self.error: + # The trailer already said that the results are incomplete, so what + # follows doesn't matter. E.g., Fuseki appends a text message to the + # response of a query it cancels. + return None + raise + def keep(self) -> None: """Read the remaining rows into memory, for iteration later.""" if not self._done: diff --git a/tests/integration_tests/test_generic/test_sparql.py b/tests/integration_tests/test_generic/test_sparql.py index 2eb102fb..6c759aed 100644 --- a/tests/integration_tests/test_generic/test_sparql.py +++ b/tests/integration_tests/test_generic/test_sparql.py @@ -291,3 +291,24 @@ def rows() -> Iterator[tuple[object, ...]]: with pytest.raises(JellyIncompleteResultsError, match="NotImplementedError"): read.extend(results) assert read == good + + +def test_garbage_after_an_error_trailer() -> None: + # As Fuseki does when it cancels a query: after the error trailer, it appends + # a text message to the response + out = io.BytesIO() + with pytest.raises(TimeoutError): + serialize_jelly_sparql(["s", "o", "t"], _failing_rows(), out) + data = out.getvalue() + b"\n## Query cancelled due to timeout ##\n" + results = parse_jelly_sparql(data) + rows: list[tuple[Node | None, ...]] = [] + with pytest.raises(JellyIncompleteResultsError, match="TimeoutError"): + rows.extend(results) + assert rows == ROWS[:2] + + +def test_garbage_after_a_complete_trailer() -> None: + data = _write(ROWS).getvalue() + b"\n## not Jelly ##\n" + results = parse_jelly_sparql(data) + with pytest.raises(JellyConformanceError): + list(results) From 6d0770b6b466ceab52e7fab32269f30c0be77422 Mon Sep 17 00:00:00 2001 From: Ostrzyciel Date: Mon, 26 Oct 2026 21:02:18 +0100 Subject: [PATCH 20/24] self-review --- benchmarks/README.md | 17 ++++-------- benchmarks/sparql_data.py | 10 +------ benchmarks/sparql_methods.py | 4 +-- docs/sparql.md | 38 +++++++++++++------------- pyjelly/integrations/generic/sparql.py | 7 +---- pyjelly/integrations/rdflib/sparql.py | 4 +-- 6 files changed, 31 insertions(+), 49 deletions(-) diff --git a/benchmarks/README.md b/benchmarks/README.md index ae86c603..ddf7b0f3 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -1,6 +1,6 @@ # Benchmarks -Throughput and size benchmarks of Jelly-RDF in pyjelly, compared with the RDF formats of rdflib, and of Jelly-SPARQL, compared with rdflib's formats of SPARQL query results (see [Jelly-SPARQL](#jelly-sparql)). +Throughput and size benchmarks of Jelly-RDF and Jelly-SPARQL in pyjelly, compared with the formats of rdflib. The method follows the [Jelly-JVM benchmarks](https://github.com/Jelly-RDF/jelly-jvm/tree/main/jmh) as closely as Python allows, so that the results can be read the same way. @@ -124,9 +124,9 @@ Everything else (processes, iterations, sweeps, cores, the round-trip check) is ### Data -- The 13 RiverBench datasets, read as the results of `SELECT * WHERE { ?s ?p ?o }` for triples and `SELECT * WHERE { GRAPH ?g { ?s ?p ?o } }` for quads, with `?g` unbound in the default graph. The first 100,000 rows, as for RDF. -- Synthetic result sets, from the presets of Jelly-JVM's generator (`SparqlDataGen`), ported in `sparql_data.py`. Each brings out one property of the format: IRIs sorted or shuffled, repeated values, unbound cells, kinds of literals, columns that mix term types, RDF 1.2 terms, and the shape of the result set (`wide-5`, `wide-20`, `zero-vars`). `realistic-mixed` looks like the results of a typical query: an entity, a property, a label, an optional value, and a value of mixed types. As in Jelly-JVM, a preset has `--rows` rows, 100,000 by default. -- By default, `run` and `size` use all RiverBench datasets and the presets `realistic-mixed`, `wide-20`, `sparse-random-50`, `lit-lang` and `iri-shuffled`. Pass any presets with `--datasets`; their names are in `PRESETS` in `sparql_data.py`. +- The 13 RiverBench datasets, read as the results of `SELECT * WHERE { ?s ?p ?o }` for triples and `SELECT * WHERE { GRAPH ?g { ?s ?p ?o } }` for quads, with `?g` unbound in the default graph. The first 100,000 rows are used. +- Synthetic result sets, from the presets of Jelly-JVM's generator, ported in `sparql_data.py`. +- By default, `run` and `size` use all RiverBench datasets and `realistic-mixed`, `wide-20`, `sparse-random-50`, `lit-lang`, `iri-shuffled`. Pass any presets with `--datasets`. Their names are in `PRESETS` in `sparql_data.py`. ### Methods @@ -139,13 +139,8 @@ Everything else (processes, iterations, sweeps, cores, the round-trip check) is | `rdflib-jelly` | Jelly-SPARQL through rdflib's plugin API: `Result.serialize(format="jelly")` | `Result.parse(format="jelly")` | | `generic-jelly` | Jelly-SPARQL with pyjelly's generic API, from a list of rows | `parse_jelly_sparql` of the generic API | -The rdflib methods write a `Result` whose bindings are a list of dicts, made before measuring. Reading makes a whole `Result`, with its bindings. Jelly uses pyjelly's default options: lookups of 8192 names, 1024 prefixes and 64 datatypes, and frames of 4096 values. rdflib can't hold RDF 1.2 terms, so its methods skip the presets that have them. +The rdflib methods write a `Result` whose bindings are a list of dicts, made before measuring. Reading makes a whole `Result`, with its bindings. Jelly uses pyjelly's default options. -CSV has no term types: every value that is not an IRI or a blank node is read back as a simple literal. The round-trip check then prints a warning, as in Jelly-JVM. +Note that CSV is a lossy result set format, so it should not be compared directly with others. rdflib reads SPARQL TSV results but can't write them, so `rdflib-tsv` is only measured reading. The TSV it reads is written by a helper in `sparql_methods.py`. - -### Throughput - -Throughput counts cells, that is, rows times variables, as in Jelly-JVM. A result set without variables counts one cell per row. - diff --git a/benchmarks/sparql_data.py b/benchmarks/sparql_data.py index 6360ec9f..12fe63f5 100644 --- a/benchmarks/sparql_data.py +++ b/benchmarks/sparql_data.py @@ -1,12 +1,4 @@ -""" -SPARQL query results for the Jelly-SPARQL benchmarks, as in Jelly-JVM. - -A dataset is either a RiverBench dataset, read as the results of -`SELECT * WHERE { ?s ?p ?o }` (triples) or `SELECT * WHERE { GRAPH ?g { ?s ?p ?o } }` -(quads, with ?g unbound in the default graph), or a synthetic result set made from -one of the presets of Jelly-JVM's generator (SparqlDataGen). Each preset brings -out one property of the format. -""" +"""Benchmark datasets for Jelly-SPARQL.""" from __future__ import annotations diff --git a/benchmarks/sparql_methods.py b/benchmarks/sparql_methods.py index c6d50160..63e8a24d 100644 --- a/benchmarks/sparql_methods.py +++ b/benchmarks/sparql_methods.py @@ -1,8 +1,8 @@ """ -Ways of writing and reading SPARQL query results, each through one library's API. +SPARQL query result ser/des implementations for benchmarking. Each method writes the rows of a result set to a stream, and reads them back from -bytes in memory. The throughput counts cells: rows times variables. +bytes in memory. """ from __future__ import annotations diff --git a/docs/sparql.md b/docs/sparql.md index b55c05a7..d55ead53 100644 --- a/docs/sparql.md +++ b/docs/sparql.md @@ -10,13 +10,13 @@ With pyjelly installed, RDFLib reads and writes query results in the `jelly` for {{ code_example('rdflib/09_sparql_results.py') }} -`Result.parse` returns a `JellySparqlResult` (from `pyjelly.integrations.rdflib.sparql`), which is an RDFLib `Result` with more attributes. RDFLib's types say that `Result.parse` returns a `Result`, so check `isinstance(result, JellySparqlResult)` to use them: +`Result.parse` returns a `JellySparqlResult` (from `pyjelly.integrations.rdflib.sparql`), which is an RDFLib `Result` with more attributes: -- `links` – the IRIs of links that describe the results, like `head.link` in [SPARQL JSON results](https://www.w3.org/TR/sparql12-results-json/). They are written too, if the result you write has them. -- `truncated` – `True` if the stream ended without saying that the results are complete, so some may be missing. pyjelly also issues a `JellyTruncatedResultsWarning` then. A stream cut in the middle of a frame can't be read at all, and raises `JellyConformanceError`. -- `error` – why the results are incomplete, in a [sequence of results](#sequences-of-results), see [incomplete results](#incomplete-results). +- `truncated` – `True` if the stream ended without indicating that the results are complete. pyjelly also issues a `JellyTruncatedResultsWarning` then. A stream cut in the middle of a frame can't be read at all, and raises `JellyConformanceError`. +- `error` – why the results are incomplete, in a [sequence of results](#sequences-of-results). See: [incomplete results](#incomplete-results). +- `links` – the IRIs of links that describe the results, like `head.link` in [SPARQL JSON results](https://www.w3.org/TR/sparql12-results-json/). -RDFLib does not support RDF 1.2 yet, so triple terms and literals with a base direction can't be read into RDFLib. Use the [generic API](#generic-api) for them. +RDFLib does not support RDF 1.2 yet, so triple terms and literals with a base direction can be read only with the [generic API](#generic-api). ### Querying a SPARQL endpoint @@ -36,13 +36,13 @@ for row in Graph(store=store).query("SELECT * WHERE { ?s ?p ?o } LIMIT 10"): !!! warning - Use the media type, `application/x-jelly-sparql`, as `returnFormat`, not `jelly`. With `jelly`, RDFLib doesn't ask for any format, so the endpoint answers in its default one. + Use the media type, `application/x-jelly-sparql`, as `returnFormat`, not `jelly`. ### Large results By default, `Result.parse` reads all rows at once. With `lazy=True`, the rows are read as the result is iterated, like RDFLib does with the results of its own queries. The stream must then stay open until you are done with the result. -When writing, the rows of an RDFLib result are written as the query makes them, so the first frames are written before the query is done. Each frame is flushed when it's written, so a reader at the other end of a connection gets it right away. +When writing, the rows of an RDFLib result are written as the query makes them, so the first frames are written before the query is done. Each frame is flushed when it's written. {{ code_example('rdflib/10_sparql_results_lazy.py') }} @@ -52,9 +52,9 @@ When writing, the rows of an RDFLib result are written as the query makes them, ### Incomplete results -A query can fail after some of its results were already written: a timeout, or a broken backend. Jelly-SPARQL then ends the stream with an error, so that readers know that the results are incomplete. pyjelly does this for you: if getting a row fails while results are being written, the stream ends with the error, and the error is raised. +A query can fail after some of its results were already written, for example in case of a timeout, or a broken backend. Jelly-SPARQL then ends the stream with an error, so that readers know that the results are incomplete. pyjelly does this for you: if getting a row fails while results are being written, the stream ends with the error, and the error is raised. -When reading such results, pyjelly raises `JellyIncompleteResultsError` after the rows that were read. Its `error` attribute is the message of the writer, and its `result` attribute holds the results read before the error: +When reading such results, pyjelly raises `JellyIncompleteResultsError` after the rows that were read. Its `error` attribute is the message of the writer, and its `result` attribute contains the results read before the error: ```python from pyjelly.errors import JellyIncompleteResultsError @@ -70,7 +70,7 @@ If you write results that you read from Jelly-SPARQL, and they were incomplete o ### Options -To change how the results are written, pass `SparqlOptions` to `serialize`: +To change how results are written, pass `SparqlOptions` to `serialize`: ```python from pyjelly.options import LookupPreset, SparqlOptions @@ -78,19 +78,19 @@ from pyjelly.options import LookupPreset, SparqlOptions options = SparqlOptions( # Sizes of the lookups, see below lookup_preset=LookupPreset(max_names=4096, max_prefixes=256, max_datatypes=32), - # How many values (rows times variables) a frame has, at most + # How many values (rows times variables) a frame may have, at most frame_values=4096, ) results.serialize(destination="results.jellys", format="jelly", options=options) ``` -The default lookup sizes are larger than for Jelly-RDF (8192 names, 1024 prefixes and 64 datatypes), as in Jelly-JVM: a row of results tends to have more distinct terms than an RDF statement. Readers accept lookups of up to 16384 names, 4096 prefixes and 256 datatypes. +The default lookup sizes are 8192 names, 1024 prefixes and 64 datatypes. Readers accept lookups of up to 16384 names, 4096 prefixes and 256 datatypes. -With `delimited=False`, the results are written as one frame without its size, e.g., to be one message of a message broker. Then the lookups must be large enough to hold all terms of the results. Files and HTTP responses must be delimited, which is the default. To send the frames of the results as separate messages, see [writing frame by frame](#writing-frame-by-frame). +With `delimited=False`, the results are written as one frame without its size, e.g., to be used in Kafka or MQTT. Then the lookups must be large enough to store all terms of the result set. Files and HTTP responses must be delimited, which is the default. To send the frames of the results as separate messages, see [writing frame by frame](#writing-frame-by-frame). ### Sequences of results -A Jelly-SPARQL stream can also hold a sequence of results, e.g., the answers to a stream of queries, or to a query evaluated again and again. This is the `PUNCTUATED` stream type of Jelly-SPARQL. To write and read such a stream, use the functions in `pyjelly.integrations.rdflib.sparql`: +A Jelly-SPARQL stream can also contain a sequence of results, e.g., the answers to a stream of queries, or to a query evaluated again and again. This is the `PUNCTUATED` stream type of Jelly-SPARQL. To write and read such a stream, use the functions in `pyjelly.integrations.rdflib.sparql`: ```python from pyjelly.integrations.rdflib.sparql import ( @@ -106,19 +106,19 @@ with open("answers.jellys", "rb") as file: print(result.type, len(result)) ``` -The lookups are shared by the results of the stream, so the terms they have in common are written once. Blank nodes are not: each result has its own. +The lookups are shared by the results of the stream, so the terms they have in common are written once. An incomplete result doesn't end a sequence, as there may be more results after it. `parse_jelly_sparql_sequence` doesn't raise an error for it, but sets its `error` attribute, so check it. ## Generic API -The [generic API](generic-sink.md) reads and writes results with its own terms (`IRI`, `Literal`, `BlankNode`, and `Triple` for triple terms), so it supports all of RDF 1.2. A row is a tuple with one value per variable, or `None` if the variable is unbound: +The [generic API](generic-sink.md) supports all of RDF 1.2. A row is a tuple with one value per variable, or `None` if the variable is unbound: {{ code_example('generic/10_sparql_results.py') }} -`parse_jelly_sparql` returns a `SparqlResultSet`, with the `variables`, the `boolean` result of an `ASK` query (`None` for a `SELECT` query, see also `is_boolean`), the `links`, and whether it is `truncated`. Its rows are read as you iterate it, so the input must stay open until then, and the rows can be iterated only once. If the writer said that the results are incomplete, iterating the rows raises `JellyIncompleteResultsError` after the last one, and its `error` attribute says why. For an `ASK` query, `parse_jelly_sparql` raises it right away. +`parse_jelly_sparql` returns a `SparqlResultSet`. Its rows are read as you iterate it, so the input must stay open until then, and the rows can be iterated only once. If the results are incomplete, iterating the rows raises `JellyIncompleteResultsError` after the last row. -`serialize_jelly_sparql_ask` writes the result of an `ASK` query. `parse_jelly_sparql_sequence` and `serialize_jelly_sparql_sequence` read and write [sequences of results](#sequences-of-results): each result written is a pair of the variables and the rows, or a boolean result. +`serialize_jelly_sparql_ask` writes the result of an `ASK` query. `parse_jelly_sparql_sequence` and `serialize_jelly_sparql_sequence` read and write [sequences of results](#sequences-of-results). ### Writing frame by frame @@ -141,4 +141,4 @@ If the results can't be completed, end them with `stream.end("why")` instead. ## Conformance -pyjelly passes the [conformance tests of Jelly-SPARQL]({{ proto_link("conformance/sparql-test-cases/") }}), with the generic API. The RDFLib integration passes all tests except those of RDF 1.2 terms. +pyjelly passes the [conformance tests of Jelly-SPARQL]({{ proto_link("conformance/sparql-test-cases/") }}), with the generic API. The RDFLib integration passes all tests except those that use RDF 1.2 terms. diff --git a/pyjelly/integrations/generic/sparql.py b/pyjelly/integrations/generic/sparql.py index 7230988e..a707918a 100644 --- a/pyjelly/integrations/generic/sparql.py +++ b/pyjelly/integrations/generic/sparql.py @@ -1,9 +1,4 @@ -""" -Jelly-SPARQL support for the generic API (SPARQL query results without rdflib). - -Values are generic API terms (IRI, Literal, BlankNode, and Triple for triple -terms). Unbound variables are None. -""" +"""Jelly-SPARQL support for the generic API (without rdflib).""" from __future__ import annotations diff --git a/pyjelly/integrations/rdflib/sparql.py b/pyjelly/integrations/rdflib/sparql.py index 6819672c..ccdfa9e8 100644 --- a/pyjelly/integrations/rdflib/sparql.py +++ b/pyjelly/integrations/rdflib/sparql.py @@ -33,8 +33,8 @@ #: Trailer error written for results that were read from a truncated stream TRUNCATED_ERROR = ( - "the results were read from a stream that ended without saying that they are " - "complete, so some may be missing" + "the results were read from a stream that ended without indicating that they are " + "complete, so some rows may be missing" ) From d22ffeaf9b39663fe98d67cfd55c722fa21e3fdb Mon Sep 17 00:00:00 2001 From: Ostrzyciel Date: Mon, 26 Oct 2026 21:09:18 +0100 Subject: [PATCH 21/24] self-review, fix rdf bug --- pyjelly/integrations/rdflib/parse.py | 37 ++++++++--- pyjelly/serialize/sparql.py | 7 +-- tests/conformance_tests/test_rdf/_common.py | 3 +- .../test_rdflib/test_parse_quads.py | 63 +++++++++++++++++++ 4 files changed, 95 insertions(+), 15 deletions(-) create mode 100644 tests/integration_tests/test_rdflib/test_parse_quads.py diff --git a/pyjelly/integrations/rdflib/parse.py b/pyjelly/integrations/rdflib/parse.py index b33e81f4..4ec1db57 100644 --- a/pyjelly/integrations/rdflib/parse.py +++ b/pyjelly/integrations/rdflib/parse.py @@ -5,7 +5,7 @@ from typing import IO, Final, cast from rdflib import Node -from rdflib.graph import ConjunctiveGraph, Dataset, Graph +from rdflib.graph import DATASET_DEFAULT_GRAPH_ID, ConjunctiveGraph, Dataset, Graph from rdflib.parser import InputSource from rdflib.parser import Parser as RDFLibParser @@ -224,6 +224,7 @@ def parse_jelly_to_graph( dataset_factory: Callable[[], Dataset] = Dataset, *, validate_iris: bool = False, + default_graph: Graph | None = None, ) -> Graph | Dataset: """ Add statements from Generator to provided Graph/Dataset. @@ -238,6 +239,9 @@ def parse_jelly_to_graph( but you can pass something else here. validate_iris (bool): whether rdflib checks the IRIs. An invalid IRI is still read, rdflib only logs a warning. Checking slows down parsing. + default_graph (Graph | None): graph for the quads in the default graph, + in the same store as the Dataset. By default, the Dataset's default + graph. Returns: Dataset | Graph: Dataset or Graph with statements. @@ -260,7 +264,7 @@ def parse_jelly_to_graph( inp=inp, frames=frames, options=options, validate_iris=validate_iris ) if to_store: - _add_to_store(sink, items) + _add_to_store(sink, items, default_graph) return sink # The Graph or Dataset class changes add(), so it gets every statement @@ -271,7 +275,10 @@ def parse_jelly_to_graph( sink.add(item) if isinstance(item, Quad): s, p, o, graph_name = item - context = cast(Dataset, sink).get_context(graph_name) + if default_graph is not None and graph_name == DATASET_DEFAULT_GRAPH_ID: + context = default_graph + else: + context = cast(Dataset, sink).get_context(graph_name) sink.add((s, p, o, context)) # type: ignore[arg-type] return sink @@ -289,8 +296,16 @@ def _context( return context -def _add_to_store(sink: Graph, items: Iterable[Statement | Prefix]) -> None: - """Add statements to the Graph's or Dataset's store directly, in batches.""" +def _add_to_store( + sink: Graph, + items: Iterable[Statement | Prefix], + default_graph: Graph | None = None, +) -> None: + """ + Add statements to the Graph's or Dataset's store directly, in batches. + + Quads in the default graph go to `default_graph`, if given. + """ store = sink.store batch: list[tuple[Node, Node, Node, Graph]] = [] contexts: dict[GraphName, Graph] = {} @@ -302,7 +317,10 @@ def _add_to_store(sink: Graph, items: Iterable[Statement | Prefix]) -> None: if isinstance(item, Quad): s, p, o, graph_name = item if graph_name is not last_graph_name: - context = _context(cast(Dataset, sink), contexts, graph_name) + if default_graph is not None and graph_name == DATASET_DEFAULT_GRAPH_ID: + context = default_graph + else: + context = _context(cast(Dataset, sink), contexts, graph_name) last_graph_name = graph_name batch.append((s, p, o, context)) elif isinstance(item, Triple): @@ -421,9 +439,14 @@ def parse( if inp is None: msg = "expected source to be a stream of bytes" raise TypeError(msg) + # The default graph of a stream of quads goes into the Graph that rdflib + # parses into, as with TriG and N-Quads: for Dataset.parse(), that is the + # Dataset's default graph + target = Graph(store=sink.store, identifier=sink.identifier) parse_jelly_to_graph( inp, - graph_factory=lambda: Graph(store=sink.store, identifier=sink.identifier), + graph_factory=lambda: target, dataset_factory=lambda: Dataset(store=sink.store), validate_iris=validate_iris, + default_graph=target, ) diff --git a/pyjelly/serialize/sparql.py b/pyjelly/serialize/sparql.py index 180cde50..b9f61358 100644 --- a/pyjelly/serialize/sparql.py +++ b/pyjelly/serialize/sparql.py @@ -1,9 +1,4 @@ -""" -Jelly-SPARQL writer: SPARQL query results in frames, with one column per variable. - -The columns use the Jelly-RDF 1.2 column encoding (pyjelly.serialize.columns), -except that any cell may be unbound. -""" +"""Jelly-SPARQL writer: SPARQL query results in frames.""" from __future__ import annotations diff --git a/tests/conformance_tests/test_rdf/_common.py b/tests/conformance_tests/test_rdf/_common.py index c21e8ad5..5ee7a5eb 100644 --- a/tests/conformance_tests/test_rdf/_common.py +++ b/tests/conformance_tests/test_rdf/_common.py @@ -10,8 +10,7 @@ MF = Namespace("http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#") # Requirements of the tests that the rdflib integration can't pass: rdflib has no -# generalized statements, quoted triples, triple terms, or base directions. The same -# for Jelly-SPARQL. +# generalized statements, quoted triples, triple terms, or base directions. NOT_RDFLIB = { JELLYT.requirementRdfStar, JELLYT.requirementGeneralizedRdf, diff --git a/tests/integration_tests/test_rdflib/test_parse_quads.py b/tests/integration_tests/test_rdflib/test_parse_quads.py new file mode 100644 index 00000000..c5dbd0c7 --- /dev/null +++ b/tests/integration_tests/test_rdflib/test_parse_quads.py @@ -0,0 +1,63 @@ +"""Reading a Jelly stream of quads into rdflib's Graph, Dataset and ConjunctiveGraph.""" + +from __future__ import annotations + +import io + +import pytest +from rdflib import Dataset, Graph, URIRef +from rdflib.graph import ConjunctiveGraph + +TRIG = """ + . + { + . +} +""" +DEFAULT_TRIPLE = ( + URIRef("http://example.org/a"), + URIRef("http://example.org/p"), + URIRef("http://example.org/b"), +) +NAMED_TRIPLE = ( + URIRef("http://example.org/c"), + URIRef("http://example.org/p"), + URIRef("http://example.org/d"), +) +NAMED_GRAPH = URIRef("http://example.org/g") + + +def _jelly_quads() -> bytes: + dataset = Dataset() + dataset.parse(data=TRIG, format="trig") + data = dataset.serialize(format="jelly", encoding="utf-8") + assert isinstance(data, bytes) + return data + + +@pytest.mark.parametrize("format_", ["jelly", "trig"]) +def test_quads_into_graph(format_: str) -> None: + data = _jelly_quads() if format_ == "jelly" else TRIG.encode() + graph = Graph() + graph.parse(io.BytesIO(data), format=format_) + assert set(graph) == {DEFAULT_TRIPLE} + named = Graph(store=graph.store, identifier=NAMED_GRAPH) + assert set(named) == {NAMED_TRIPLE} + + +@pytest.mark.parametrize("format_", ["jelly", "trig"]) +def test_quads_into_dataset(format_: str) -> None: + data = _jelly_quads() if format_ == "jelly" else TRIG.encode() + dataset = Dataset() + dataset.parse(io.BytesIO(data), format=format_) + assert set(dataset.default_graph) == {DEFAULT_TRIPLE} + assert set(dataset.graph(NAMED_GRAPH)) == {NAMED_TRIPLE} + + +@pytest.mark.parametrize("format_", ["jelly", "trig"]) +def test_quads_into_conjunctive_graph(format_: str) -> None: + data = _jelly_quads() if format_ == "jelly" else TRIG.encode() + graph = ConjunctiveGraph() + graph.parse(io.BytesIO(data), format=format_) + assert set(graph.default_context) == {DEFAULT_TRIPLE} + assert set(graph.get_context(NAMED_GRAPH)) == {NAMED_TRIPLE} From 2577902112125e10e787ab97c68977b16bff449b Mon Sep 17 00:00:00 2001 From: Ostrzyciel Date: Mon, 26 Oct 2026 21:17:18 +0100 Subject: [PATCH 22/24] fix ci --- pyjelly/parse/sparql.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pyjelly/parse/sparql.py b/pyjelly/parse/sparql.py index 2bb98826..f0180826 100644 --- a/pyjelly/parse/sparql.py +++ b/pyjelly/parse/sparql.py @@ -58,6 +58,11 @@ def tuple_rows(variables: list[str], cells: list[list[Any]], rows: int) -> list[ # every attribute name of a message as a field name first _FRAME_HAS_FIELD: Final = jelly.SparqlResultsFrame.HasField +# What protobuf raises for bytes it can't decode. The pure-Python implementation +# raises UnicodeDecodeError for a string that isn't UTF-8, the others DecodeError. +_DECODE_ERRORS: Final = (DecodeError, UnicodeDecodeError) + + _STREAM_TYPES: Final = ( jelly.SPARQL_STREAM_TYPE_FLAT, jelly.SPARQL_STREAM_TYPE_PUNCTUATED, @@ -406,7 +411,7 @@ def sparql_frames(inp: IO[bytes] | bytes) -> Iterator[jelly.SparqlResultsFrame]: raise JellyConformanceError(msg) try: return iter((parse(jelly.SparqlResultsFrame, data),)) - except DecodeError as error: + except _DECODE_ERRORS as error: raise _not_sparql(error) from None return _delimited_frames(inp) @@ -424,7 +429,7 @@ def _delimited_frames(inp: IO[bytes]) -> Iterator[jelly.SparqlResultsFrame]: try: # protobuf annotates `input_bytes` as io.BytesIO, but any IO[bytes] works. frame = parse_length_prefixed(jelly.SparqlResultsFrame, inp) # type: ignore[arg-type] - except DecodeError as error: + except _DECODE_ERRORS as error: raise _not_sparql(error) from None except ValueError as error: if getattr(inp, "closed", False): From 88399d25554f200c510a0fbae898df2b654ec231 Mon Sep 17 00:00:00 2001 From: Ostrzyciel Date: Mon, 26 Oct 2026 21:47:11 +0100 Subject: [PATCH 23/24] Add hardening tests --- pyjelly/parse/decode.py | 82 +++- pyjelly/parse/ioutils.py | 53 ++- pyjelly/parse/lookup.py | 6 +- pyjelly/parse/sparql.py | 34 +- tests/unit_tests/test_parse/test_decoder.py | 5 +- tests/unit_tests/test_parse/test_ioutils.py | 95 ++++- .../test_parse/test_lookup_decoder.py | 11 +- .../test_parse/test_rdf_hardening.py | 355 ++++++++++++++++ .../test_parse/test_sparql_hardening.py | 399 ++++++++++++++++++ tests/utils/byte_fuzzer.py | 107 +++++ 10 files changed, 1086 insertions(+), 61 deletions(-) create mode 100644 tests/unit_tests/test_parse/test_rdf_hardening.py create mode 100644 tests/unit_tests/test_parse/test_sparql_hardening.py create mode 100644 tests/utils/byte_fuzzer.py diff --git a/pyjelly/parse/decode.py b/pyjelly/parse/decode.py index 87b3af10..28c71cc3 100644 --- a/pyjelly/parse/decode.py +++ b/pyjelly/parse/decode.py @@ -309,6 +309,25 @@ def _adapter_missing(feature: str, *, stream_types: StreamTypes) -> Never: raise NotImplementedError(msg) +def _row_missing( + rows: str, + *, + allowed_in: tuple[jelly.PhysicalStreamType, ...], + stream_types: StreamTypes, +) -> Never: + """ + Raise the error for statement rows that the adapter does not decode. + + The adapter is chosen for the physical type of the stream, so the rows are + either not allowed in the stream, or the adapter is incomplete. + """ + if stream_types.physical_type not in allowed_in: + physical_type_name = jelly.PhysicalStreamType.Name(stream_types.physical_type) + msg = f"a stream of type {physical_type_name} cannot have {rows}" + raise JellyConformanceError(msg) + _adapter_missing(f"decoding {rows}", stream_types=stream_types) + + @mypyc_attr(allow_interpreted_subclasses=True) class Adapter(metaclass=ABCMeta): def __init__( @@ -350,19 +369,34 @@ def directional_literal(self, lex: str, language: str, direction: str) -> Any: # Optional abstract methods--not required to be implemented by all adapters def triple(self, terms: Iterable[Any]) -> Any: # noqa: ARG002 - _adapter_missing("decoding triples", stream_types=self.options.stream_types) + _row_missing( + "triples", + allowed_in=( + jelly.PHYSICAL_STREAM_TYPE_TRIPLES, + jelly.PHYSICAL_STREAM_TYPE_GRAPHS, + ), + stream_types=self.options.stream_types, + ) def quad(self, terms: Iterable[Any]) -> Any: # noqa: ARG002 - _adapter_missing("decoding quads", stream_types=self.options.stream_types) + _row_missing( + "quads", + allowed_in=(jelly.PHYSICAL_STREAM_TYPE_QUADS,), + stream_types=self.options.stream_types, + ) def graph_start(self, graph_id: Any) -> Any: # noqa: ARG002 - _adapter_missing( - "decoding graph start markers", stream_types=self.options.stream_types + _row_missing( + "graph start markers", + allowed_in=(jelly.PHYSICAL_STREAM_TYPE_GRAPHS,), + stream_types=self.options.stream_types, ) def graph_end(self) -> Any: - _adapter_missing( - "decoding graph end markers", stream_types=self.options.stream_types + _row_missing( + "graph end markers", + allowed_in=(jelly.PHYSICAL_STREAM_TYPE_GRAPHS,), + stream_types=self.options.stream_types, ) def namespace_declaration(self, name: str, iri: Any) -> Any: # noqa: ARG002 @@ -504,19 +538,27 @@ def decode_other_row(self, row_owner: jelly.RdfStreamRow, kind: str | None) -> N self.decode_graph_end(row_owner.graph_end) elif kind == "options": self.validate_stream_options(row_owner.options) + elif kind is None: + msg = "a row of the stream is empty" + raise JellyConformanceError(msg) else: msg = f"decoder not implemented for row {kind}" raise TypeError(msg) def validate_stream_options(self, options: jelly.RdfStreamOptions) -> None: + """Check that repeated stream options are the same as the first ones.""" stream_types, lookup_preset, params = self.options - assert stream_types.physical_type == options.physical_type - assert stream_types.logical_type == options.logical_type - assert params.stream_name == options.stream_name - assert params.version >= options.version - assert lookup_preset.max_prefixes == options.max_prefix_table_size - assert lookup_preset.max_datatypes == options.max_datatype_table_size - assert lookup_preset.max_names == options.max_name_table_size + if ( + stream_types.physical_type != options.physical_type + or stream_types.logical_type != options.logical_type + or params.stream_name != options.stream_name + or params.version < options.version + or lookup_preset.max_prefixes != options.max_prefix_table_size + or lookup_preset.max_datatypes != options.max_datatype_table_size + or lookup_preset.max_names != options.max_name_table_size + ): + msg = "the repeated stream options are not the same as the first ones" + raise JellyConformanceError(msg) def ingest_prefix_entry(self, entry: jelly.RdfPrefixEntry) -> None: """ @@ -644,7 +686,7 @@ def decode_literal(self, literal: jelly.RdfLiteral) -> Any: kind = _LITERAL_WHICH_ONEOF(literal, "literalKind") if kind == "langtag": language = literal.langtag or None - elif kind == "datatype" and self.datatypes.lookup_size: + elif kind == "datatype": datatype = self.datatypes.decode_datatype_term_index(literal.datatype) lex = literal.lex cache = self.literal_cache(language, datatype) @@ -705,7 +747,11 @@ def decode_namespace_declaration( return self.adapter.namespace_declaration(declaration.name, iri) def decode_graph_start(self, graph_start: jelly.RdfGraphStart) -> Any: - term = getattr(graph_start, graph_start.WhichOneof("graph")) + kind = graph_start.WhichOneof("graph") + if kind is None: + msg = "a graph start marker has no graph" + raise JellyConformanceError(msg) + term = getattr(graph_start, kind) return self.adapter.graph_start(self.decode_term(term)) def decode_graph_end(self, _: jelly.RdfGraphEnd) -> Any: @@ -722,7 +768,7 @@ def decode_statement(self, fields: list[Any], size: int) -> list[Any]: size (int): 3 for a triple, 4 for a quad Raises: - ValueError: if a missing repeated term is encountered + JellyConformanceError: if a missing repeated term is encountered Returns: Any: a list of decoded terms @@ -756,7 +802,7 @@ def decode_statement(self, fields: list[Any], size: int) -> list[Any]: for i in range(size): if terms[i] is _MISSING: msg = f"missing repeated term {_POSITIONS[i]}" - raise ValueError(msg) + raise JellyConformanceError(msg) return terms def decode_triple(self, triple: jelly.RdfTriple) -> Any: @@ -773,7 +819,7 @@ def decode_quoted_triple(self, triple: jelly.RdfTriple) -> Any: decoded_term = self.decode_term(jelly_term) else: msg = "repeated terms are not allowed in quoted triples" - raise ValueError(msg) + raise JellyConformanceError(msg) terms.append(decoded_term) return self.adapter.quoted_triple(terms) diff --git a/pyjelly/parse/ioutils.py b/pyjelly/parse/ioutils.py index c03e153d..331d37c8 100644 --- a/pyjelly/parse/ioutils.py +++ b/pyjelly/parse/ioutils.py @@ -2,12 +2,14 @@ import os from collections.abc import Generator, Iterator from itertools import chain -from typing import IO, Final, cast +from typing import IO, Final, TypeVar, cast from google.protobuf.descriptor import Descriptor, FieldDescriptor +from google.protobuf.message import DecodeError, Message from google.protobuf.proto import parse, parse_length_prefixed from pyjelly import jelly +from pyjelly.errors import JellyConformanceError from pyjelly.options import MIN_COLUMN_NAME_LOOKUP_SIZE, VERSION_1_2 from pyjelly.parse.decode import ParserOptions, options_from_frame, parser_options @@ -316,9 +318,49 @@ def detect_delimited( return inp, delimited_jelly_hint(header, tags) +# What protobuf raises for bytes it can't decode. The pure-Python implementation +# raises UnicodeDecodeError for a string that isn't UTF-8, the others DecodeError. +DECODE_ERRORS: Final = (DecodeError, UnicodeDecodeError) + +MessageT = TypeVar("MessageT", bound=Message) + + +def parse_delimited(message_class: type[MessageT], inp: IO[bytes]) -> MessageT | None: + """ + Read the next length-prefixed message, or None at the end of the input. + + Raises: + DecodeError: if the bytes are not a message of the class, or the input + ends in the middle of one. Or UnicodeDecodeError, see DECODE_ERRORS. + ValueError: if the input is closed. + + """ + try: + # protobuf annotates `input_bytes` as io.BytesIO, but any IO[bytes] works. + return parse_length_prefixed(message_class, inp) # type: ignore[arg-type] + except DECODE_ERRORS: + raise + except (ValueError, OverflowError) as error: + if getattr(inp, "closed", False): + raise + # The length prefix is cut short, or too large to read + msg = f"invalid length prefix ({error})" + raise DecodeError(msg) from None + + +def _not_jelly(error: Exception) -> JellyConformanceError: + msg = f"the stream is not valid Jelly-RDF ({error}). Is it cut short?" + return JellyConformanceError(msg) + + def frame_iterator(inp: IO[bytes]) -> Generator[jelly.RdfStreamFrame]: - # protobuf annotates `input_bytes` as io.BytesIO, but any IO[bytes] works. - while frame := parse_length_prefixed(jelly.RdfStreamFrame, inp): # type: ignore[arg-type] + while True: + try: + frame = parse_delimited(jelly.RdfStreamFrame, inp) + except DECODE_ERRORS as error: + raise _not_jelly(error) from None + if frame is None: + return yield frame @@ -371,7 +413,10 @@ def get_options_and_frames( options = options_from_frame(first_frame, delimited=True) return options, chain(skipped_frames, (first_frame,), frames) - frame = parse(jelly.RdfStreamFrame, inp.read()) + try: + frame = parse(jelly.RdfStreamFrame, inp.read()) + except DECODE_ERRORS as error: + raise _not_jelly(error) from None if not frame.rows: return _empty_stream_options(delimited=False), iter((frame,)) diff --git a/pyjelly/parse/lookup.py b/pyjelly/parse/lookup.py index 94cc56fe..e9aea12e 100644 --- a/pyjelly/parse/lookup.py +++ b/pyjelly/parse/lookup.py @@ -44,10 +44,10 @@ def assign_entry(self, index: int, value: str) -> None: def at(self, index: int) -> str: self.last_reused_index = index - value = self.data[index - 1] + value = self.data[index - 1] if 0 < index <= self.lookup_size else None if value is None: - msg = f"invalid resolved index {index}" - raise IndexError(msg) + msg = f"a term refers to lookup entry {index}, which is not set" + raise JellyConformanceError(msg) return value def decode_prefix_term_index(self, index: int) -> str: diff --git a/pyjelly/parse/sparql.py b/pyjelly/parse/sparql.py index f0180826..2d7d9acf 100644 --- a/pyjelly/parse/sparql.py +++ b/pyjelly/parse/sparql.py @@ -12,8 +12,7 @@ from collections.abc import Callable, Iterator from typing import IO, Any, Final -from google.protobuf.message import DecodeError -from google.protobuf.proto import parse, parse_length_prefixed +from google.protobuf.proto import parse from pyjelly import jelly from pyjelly.errors import ( @@ -36,7 +35,12 @@ ) from pyjelly.parse.columns import MAX_ROWS, ColumnDecoder from pyjelly.parse.decode import Adapter, Decoder, ParserOptions -from pyjelly.parse.ioutils import SPARQL_FRAME_TAGS, detect_delimited +from pyjelly.parse.ioutils import ( + DECODE_ERRORS, + SPARQL_FRAME_TAGS, + detect_delimited, + parse_delimited, +) #: Metadata key for the links of a result set (head.link in SPARQL JSON results) LINK_KEY: Final = "link" @@ -58,10 +62,6 @@ def tuple_rows(variables: list[str], cells: list[list[Any]], rows: int) -> list[ # every attribute name of a message as a field name first _FRAME_HAS_FIELD: Final = jelly.SparqlResultsFrame.HasField -# What protobuf raises for bytes it can't decode. The pure-Python implementation -# raises UnicodeDecodeError for a string that isn't UTF-8, the others DecodeError. -_DECODE_ERRORS: Final = (DecodeError, UnicodeDecodeError) - _STREAM_TYPES: Final = ( jelly.SPARQL_STREAM_TYPE_FLAT, @@ -411,7 +411,7 @@ def sparql_frames(inp: IO[bytes] | bytes) -> Iterator[jelly.SparqlResultsFrame]: raise JellyConformanceError(msg) try: return iter((parse(jelly.SparqlResultsFrame, data),)) - except _DECODE_ERRORS as error: + except DECODE_ERRORS as error: raise _not_sparql(error) from None return _delimited_frames(inp) @@ -427,18 +427,16 @@ def _not_sparql(error: Exception) -> JellyConformanceError: def _delimited_frames(inp: IO[bytes]) -> Iterator[jelly.SparqlResultsFrame]: while True: try: - # protobuf annotates `input_bytes` as io.BytesIO, but any IO[bytes] works. - frame = parse_length_prefixed(jelly.SparqlResultsFrame, inp) # type: ignore[arg-type] - except _DECODE_ERRORS as error: + frame = parse_delimited(jelly.SparqlResultsFrame, inp) + except DECODE_ERRORS as error: raise _not_sparql(error) from None except ValueError as error: - if getattr(inp, "closed", False): - msg = ( - "the input was closed before all results were read. Read the " - "rows before closing it." - ) - raise ValueError(msg) from error - raise + # parse_delimited() raises it only if the input is closed + msg = ( + "the input was closed before all results were read. Read the rows " + "before closing it." + ) + raise ValueError(msg) from error if frame is None: return yield frame diff --git a/tests/unit_tests/test_parse/test_decoder.py b/tests/unit_tests/test_parse/test_decoder.py index f45f73bf..adb5531a 100644 --- a/tests/unit_tests/test_parse/test_decoder.py +++ b/tests/unit_tests/test_parse/test_decoder.py @@ -1,6 +1,7 @@ import pytest from pyjelly import jelly +from pyjelly.errors import JellyConformanceError from pyjelly.integrations.generic.decode import GenericTriplesAdapter from pyjelly.integrations.generic.generic_sink import IRI, Literal from pyjelly.options import VERSION_1_1, LookupPreset, StreamParameters, StreamTypes @@ -66,7 +67,7 @@ def test_repeated_terms_must_be_set_first() -> None: decoder.ingest_prefix_entry(jelly.RdfPrefixEntry(id=1, value="http://a/")) decoder.ingest_name_entry(jelly.RdfNameEntry(id=1, value="x")) iri = jelly.RdfIri(prefix_id=1, name_id=1) - with pytest.raises(ValueError, match="missing repeated term predicate"): + with pytest.raises(JellyConformanceError, match="missing repeated term predicate"): decoder.decode_triple(jelly.RdfTriple(s_iri=iri, o_iri=iri)) # Terms set once can be repeated @@ -80,5 +81,5 @@ def test_repeated_terms_must_be_set_first() -> None: def test_empty_row_is_an_error() -> None: frame = jelly.RdfStreamFrame(rows=[jelly.RdfStreamRow()]) - with pytest.raises(TypeError, match="decoder not implemented for row None"): + with pytest.raises(JellyConformanceError, match="a row of the stream is empty"): list(make_decoder().iter_rows(frame)) diff --git a/tests/unit_tests/test_parse/test_ioutils.py b/tests/unit_tests/test_parse/test_ioutils.py index ccb5c6ff..278a2ef9 100644 --- a/tests/unit_tests/test_parse/test_ioutils.py +++ b/tests/unit_tests/test_parse/test_ioutils.py @@ -9,15 +9,24 @@ from google.protobuf.descriptor import Descriptor from pyjelly import jelly +from pyjelly.integrations.generic.decode import GenericStatementSinkAdapter +from pyjelly.integrations.generic.generic_sink import IRI from pyjelly.parse.ioutils import ( FRAME_TAGS, + SPARQL_FRAME_TAGS, delimited_jelly_hint, get_options_and_frames, read_hint_header, ) +from pyjelly.parse.sparql import read_result_set from tests.utils import wire_fuzzer FRAME_DESCRIPTOR = cast("Descriptor", jelly.RdfStreamFrame.DESCRIPTOR) +SPARQL_FRAME_DESCRIPTOR = cast("Descriptor", jelly.SparqlResultsFrame.DESCRIPTOR) +FRAME_TYPES = { + "rdf": (FRAME_DESCRIPTOR, FRAME_TAGS), + "sparql": (SPARQL_FRAME_DESCRIPTOR, SPARQL_FRAME_TAGS), +} # A frame of 25 bytes: too long for its first byte to be a size. FRAME_LARGE = jelly.RdfStreamFrame( @@ -37,11 +46,11 @@ ) -def detect(data: bytes) -> tuple[bool, bytes]: +def detect(data: bytes, tags: frozenset[int] = FRAME_TAGS) -> tuple[bool, bytes]: """Run the detection and return its answer plus every byte of the input.""" inp = io.BytesIO(data) - header = read_hint_header(inp) - return delimited_jelly_hint(header), header + inp.read() + header = read_hint_header(inp, tags) + return delimited_jelly_hint(header, tags), header + inp.read() def delimit(*frames: jelly.RdfStreamFrame) -> bytes: @@ -154,15 +163,17 @@ def read(self, size: int = -1) -> bytes: # noqa: ARG002 ITERATIONS = 5_000 -def test_fuzz_detects_random_non_delimited_frames() -> None: +@pytest.mark.parametrize("frame_type", list(FRAME_TYPES)) +def test_fuzz_detects_random_non_delimited_frames(frame_type: str) -> None: + descriptor, tags = FRAME_TYPES[frame_type] rnd = random.Random(1) failures = 0 ambiguous = 0 for _ in range(ITERATIONS): - data = wire_fuzzer.message(FRAME_DESCRIPTOR, rnd) - if data and data[0] in FRAME_TAGS: + data = wire_fuzzer.message(descriptor, rnd) + if data and data[0] in tags: ambiguous += 1 - is_delimited, kept = detect(data) + is_delimited, kept = detect(data, tags) assert kept == data failures += is_delimited # A frame that does not start with the stream options can, by chance, also be a @@ -172,30 +183,86 @@ def test_fuzz_detects_random_non_delimited_frames() -> None: assert ambiguous > ITERATIONS // 2 -def test_fuzz_detects_random_delimited_streams() -> None: +@pytest.mark.parametrize("frame_type", list(FRAME_TYPES)) +def test_fuzz_detects_random_delimited_streams(frame_type: str) -> None: + descriptor, tags = FRAME_TYPES[frame_type] rnd = random.Random(2) ambiguous = 0 for _ in range(ITERATIONS): data = b"" for _ in range(rnd.randrange(1, 4)): - body = wire_fuzzer.message(FRAME_DESCRIPTOR, rnd) + body = wire_fuzzer.message(descriptor, rnd) data += wire_fuzzer.varint(len(body)) + body - if data[0] in FRAME_TAGS: + if data[0] in tags: ambiguous += 1 - assert detect(data) == (True, data) - # The first frame is sometimes 10, 18 or 122 bytes long: sizes that are also tags + assert detect(data, tags) == (True, data) + # Some first frames have a length that is also a tag byte assert ambiguous > 0 -def test_fuzz_keeps_every_byte_of_random_input() -> None: +@pytest.mark.parametrize("frame_type", list(FRAME_TYPES)) +def test_fuzz_keeps_every_byte_of_random_input(frame_type: str) -> None: + _, tags = FRAME_TYPES[frame_type] rnd = random.Random(3) for _ in range(ITERATIONS): length = rnd.randrange(400) if rnd.randrange(10) == 0 else rnd.randrange(24) data = bytes(rnd.randrange(256) for _ in range(length)) - _, kept = detect(data) + _, kept = detect(data, tags) assert kept == data +# Jelly-SPARQL stream options of several serialized sizes +SPARQL_OPTIONS = [ + jelly.SparqlResultsOptions( + max_name_table_size=names, + max_prefix_table_size=prefixes, + max_datatype_table_size=64, + version=1, + ) + for names in (128, 256, 1024, 8192, 16384) + for prefixes in (0, 64, 1024, 4096) +] + + +def sparql_frame(options: jelly.SparqlResultsOptions) -> bytes: + return jelly.SparqlResultsFrame( + options=options, + variables=["x"], + row_count=1, + names=[jelly.RdfLookupEntryPacked(values=["http://example.org/a"])], + columns=[jelly.RdfColumn(name_ids=[1])], + trailer=jelly.SparqlResultsTrailer(), + ).SerializeToString() + + +def test_sparql_options_cover_the_size_of_a_tag() -> None: + # Options of 10 bytes make a non-delimited frame start with 0A 0A, which is + # also how a delimited stream whose first frame is 10 bytes long starts + assert 10 in {options.ByteSize() for options in SPARQL_OPTIONS} + + +@pytest.mark.parametrize( + "options", + SPARQL_OPTIONS, + ids=[ + f"names={o.max_name_table_size},prefixes={o.max_prefix_table_size}" + for o in SPARQL_OPTIONS + ], +) +@pytest.mark.parametrize("delimited", [True, False], ids=["delimited", "non-delimited"]) +def test_detects_sparql_frames( + options: jelly.SparqlResultsOptions, + delimited: bool, # noqa: FBT001 +) -> None: + data = sparql_frame(options) + if delimited: + data = wire_fuzzer.varint(len(data)) + data + assert detect(data, SPARQL_FRAME_TAGS) == (delimited, data) + # Read it back, delimited or not as detected + result_set = read_result_set(data, GenericStatementSinkAdapter) + assert list(result_set) == [(IRI("http://example.org/a"),)] + + # A frame holding only the stream options. OPTIONS_FRAME = b"\x0a\x0f\x0a\x0d\x10\x01\x48\x80\x01\x50\x0f\x58\x0f\x70\x01\x78\x01" diff --git a/tests/unit_tests/test_parse/test_lookup_decoder.py b/tests/unit_tests/test_parse/test_lookup_decoder.py index ab1b0568..d69fe8ff 100644 --- a/tests/unit_tests/test_parse/test_lookup_decoder.py +++ b/tests/unit_tests/test_parse/test_lookup_decoder.py @@ -20,12 +20,19 @@ def test_datatype_index_zero_error() -> None: def test_at_invalid_index() -> None: decoder = LookupDecoder(lookup_size=4) - with pytest.raises(IndexError) as excinfo: + with pytest.raises(JellyConformanceError, match="lookup entry 2, which is not set"): decoder.at(2) - assert "invalid resolved index 2" in str(excinfo.value) assert decoder.last_reused_index == 2 +@pytest.mark.parametrize("index", [0, 5, 2**32 - 1]) +def test_at_outside_lookup(index: int) -> None: + decoder = LookupDecoder(lookup_size=4) + decoder.assign_entry(4, "d") + with pytest.raises(JellyConformanceError, match="which is not set"): + decoder.at(index) + + def test_assign_entry_outside_lookup() -> None: decoder = LookupDecoder(lookup_size=2) decoder.assign_entry(2, "b") diff --git a/tests/unit_tests/test_parse/test_rdf_hardening.py b/tests/unit_tests/test_parse/test_rdf_hardening.py new file mode 100644 index 00000000..920b4a99 --- /dev/null +++ b/tests/unit_tests/test_parse/test_rdf_hardening.py @@ -0,0 +1,355 @@ +""" +Reading invalid Jelly-RDF input, after jelly-jvm's ProtoDecoderHardeningSpec. + +The rows are valid protobuf, but refer to things that are not there or are not +allowed in the stream. The reader must reject them with JellyConformanceError. +""" + +from __future__ import annotations + +import io +from collections.abc import Callable +from typing import IO, Any + +import pytest + +from pyjelly import jelly +from pyjelly.errors import JellyAssertionError, JellyConformanceError +from pyjelly.integrations.generic.decode import GenericStatementSinkAdapter +from pyjelly.integrations.generic.generic_sink import ( + IRI, + BlankNode, + DefaultGraph, + GenericStatementSink, + Literal, + Quad, + Triple, +) +from pyjelly.integrations.generic.parse import parse_jelly_flat +from pyjelly.integrations.generic.serialize import guess_stream, stream_frames +from pyjelly.options import LookupPreset, StreamParameters +from pyjelly.parse.sparql import read_result_sets +from pyjelly.serialize.ioutils import write_delimited, write_single +from pyjelly.serialize.streams import SerializerOptions +from tests.utils import byte_fuzzer, wire_fuzzer + +Row = jelly.RdfStreamRow +DATATYPES = 8 + + +def options(datatypes: int = DATATYPES) -> jelly.RdfStreamRow: + return Row( + options=jelly.RdfStreamOptions( + physical_type=jelly.PHYSICAL_STREAM_TYPE_TRIPLES, + max_name_table_size=128, + max_prefix_table_size=8, + max_datatype_table_size=datatypes, + version=2, + ) + ) + + +def iri(prefix: int, name: int) -> jelly.RdfIri: + return jelly.RdfIri(prefix_id=prefix, name_id=name) + + +def name(index: int, value: str) -> jelly.RdfStreamRow: + return Row(name=jelly.RdfNameEntry(id=index, value=value)) + + +def prefix(index: int, value: str) -> jelly.RdfStreamRow: + return Row(prefix=jelly.RdfPrefixEntry(id=index, value=value)) + + +def datatype(index: int, value: str) -> jelly.RdfStreamRow: + return Row(datatype=jelly.RdfDatatypeEntry(id=index, value=value)) + + +def triple(**terms: Any) -> jelly.RdfStreamRow: + return Row(triple=jelly.RdfTriple(**terms)) + + +def namespace(label: str, value: jelly.RdfIri | None = None) -> jelly.RdfStreamRow: + return Row(namespace=jelly.RdfNamespaceDeclaration(name=label, value=value)) + + +def read(data: bytes) -> list[Any]: + return list(parse_jelly_flat(io.BytesIO(data))) + + +def read_rows(*rows: jelly.RdfStreamRow) -> list[Any]: + out = io.BytesIO() + write_delimited(jelly.RdfStreamFrame(rows=rows), out) + return read(out.getvalue()) + + +def literal(lex: str, index: int) -> jelly.RdfStreamRow: + """Return the triple "lex"^^dt, with dt the datatype at the index.""" + return triple( + s_iri=iri(0, 1), + p_iri=iri(0, 1), + o_literal=jelly.RdfLiteral(lex=lex, datatype=index), + ) + + +REJECTED = { + "datatype entry past the end": ( + [options(), datatype(DATATYPES + 1, "http://dt")], + "the lookup has 8 entries", + ), + "datatype entry id above 2^31": ( + [options(), datatype(2**32 - 1, "http://dt")], + "the lookup has 8 entries", + ), + "datatype entry with no datatype lookup": ( + [options(datatypes=0), datatype(1, "http://dt")], + "the lookup has 0 entries", + ), + "datatype entries that run off the end": ( + [options()] + [datatype(0, f"http://dt{i}") for i in range(DATATYPES + 1)], + "the lookup has 8 entries", + ), + "literal with a datatype never set": ( + [options(), name(1, "http://s"), literal("1", 1)], + "lookup entry 1, which is not set", + ), + "literal with a datatype past the end": ( + [options(), name(1, "http://s"), literal("1", DATATYPES + 1)], + "lookup entry 9, which is not set", + ), + "literal with a datatype, with no datatype lookup": ( + [options(datatypes=0), name(1, "http://s"), literal("1", 1)], + "lookup entry 1, which is not set", + ), + "namespace declaration with no IRI": ( + [options(), namespace("ex")], + "which is not set", + ), + "namespace declaration with a name never set": ( + [options(), prefix(1, "http://example.org/"), namespace("ex", iri(1, 1))], + "lookup entry 1, which is not set", + ), + "namespace declaration with a prefix never set": ( + [options(), name(1, "x"), namespace("ex", iri(1, 1))], + "lookup entry 1, which is not set", + ), + "IRI with a name id above 2^31": ( + [options(), triple(s_iri=iri(0, 2**32 - 1), p_iri=iri(0, 1), o_iri=iri(0, 1))], + "lookup entry 4294967295, which is not set", + ), + "IRI with a prefix id above 2^31": ( + [ + options(), + name(1, "x"), + triple(s_iri=iri(2**32 - 1, 1), p_iri=iri(0, 1), o_iri=iri(0, 1)), + ], + "lookup entry 4294967295, which is not set", + ), + "row with nothing in it": ([options(), Row()], "a row of the stream is empty"), + "repeated stream options that are not the same": ( + [options(), options(datatypes=16)], + "not the same as the first ones", + ), + "quads in a stream of triples": ( + [ + options(), + name(1, "http://s"), + Row( + quad=jelly.RdfQuad( + s_iri=iri(0, 1), p_iri=iri(0, 1), o_iri=iri(0, 1), g_iri=iri(0, 1) + ) + ), + ], + "PHYSICAL_STREAM_TYPE_TRIPLES cannot have quads", + ), + "graph start in a stream of triples": ( + [options(), Row(graph_start=jelly.RdfGraphStart(g_default_graph={}))], + "PHYSICAL_STREAM_TYPE_TRIPLES cannot have graph start markers", + ), + "graph start with no graph": ( + [options(), Row(graph_start=jelly.RdfGraphStart())], + "a graph start marker has no graph", + ), + "quoted triple with a repeated term": ( + [ + options(), + name(1, "http://s"), + triple( + s_iri=iri(0, 1), + p_iri=iri(0, 1), + o_triple_term=jelly.RdfTriple(s_iri=iri(0, 1), p_iri=iri(0, 1)), + ), + ], + "repeated terms are not allowed in quoted triples", + ), + "repeated term with no term before it": ( + [options(), triple()], + "missing repeated term subject", + ), +} + + +@pytest.mark.parametrize(("rows", "error"), list(REJECTED.values()), ids=list(REJECTED)) +def test_rejects(rows: list[jelly.RdfStreamRow], error: str) -> None: + with pytest.raises(JellyConformanceError, match=error): + read_rows(*rows) + + +def test_rejects_namespace_declaration_with_a_name_never_set_in_columns() -> None: + batch = jelly.RdfColumnBatch( + prefixes=[jelly.RdfLookupEntryPacked(values=["http://example.org/"])], + namespaces=[jelly.RdfNamespaceDeclaration(name="ex", value=iri(1, 1))], + ) + stream_options = jelly.RdfStreamOptions( + physical_type=jelly.PHYSICAL_STREAM_TYPE_TRIPLES, + max_name_table_size=128, + max_prefix_table_size=8, + version=3, + ) + out = io.BytesIO() + write_delimited( + jelly.RdfStreamFrame(rows=[Row(options=stream_options)], columns=batch), out + ) + with pytest.raises(JellyConformanceError, match="which is not set"): + read(out.getvalue()) + + +# Length prefixes of delimited streams, after jelly-jvm's DelimitedMessageReaderSpec + +RDF_FRAME = jelly.RdfStreamFrame( + rows=[options(), name(1, "http://example.org/" + "x" * 3000)] +).SerializeToString() +SPARQL_FRAME = jelly.SparqlResultsFrame( + options=jelly.SparqlResultsOptions(max_name_table_size=128, version=1), + variables=["x"], + row_count=1, + names=[jelly.RdfLookupEntryPacked(values=["http://example.org/" + "x" * 3000])], + columns=[jelly.RdfColumn(name_ids=[1])], +).SerializeToString() + + +def read_rdf(inp: IO[bytes]) -> None: + list(parse_jelly_flat(inp)) + + +def read_sparql(inp: IO[bytes]) -> None: + for result_set in read_result_sets(inp, GenericStatementSinkAdapter): + list(result_set) + + +READERS = {"rdf": (RDF_FRAME, read_rdf), "sparql": (SPARQL_FRAME, read_sparql)} + +BAD_LENGTHS: dict[str, Callable[[bytes], bytes]] = { + "frame shorter than its length, cut at 3": lambda delimited: delimited[:3], + "frame shorter than its length, cut at 100": lambda delimited: delimited[:100], + "frame shorter than its length by 1": lambda delimited: delimited[:-1], + "length of 2^31 - 1": lambda _: b"\xff\xff\xff\xff\x07\x01\x02\x03", + "length of 2^32 - 1": lambda _: b"\xff\xff\xff\xff\x0f\x01\x02\x03", + "length of 2^64 - 1": lambda _: b"\xff" * 9 + b"\x01\x01\x02", + "length cut off": lambda _: b"\x80", + "length cut off after a frame": lambda delimited: delimited + b"\x80", + "frame cut off after a frame": lambda delimited: delimited + delimited[:50], +} + + +@pytest.mark.parametrize("reader", list(READERS)) +@pytest.mark.parametrize("bad", list(BAD_LENGTHS)) +def test_rejects_bad_length(reader: str, bad: str) -> None: + frame, read_stream = READERS[reader] + delimited = wire_fuzzer.varint(len(frame)) + frame + with pytest.raises(JellyConformanceError, match="Is it cut short"): + read_stream(io.BytesIO(BAD_LENGTHS[bad](delimited))) + + +@pytest.mark.parametrize("reader", list(READERS)) +def test_closed_input(reader: str) -> None: + frame, read_stream = READERS[reader] + inp = io.BytesIO(wire_fuzzer.varint(len(frame)) + frame) + inp.close() + with pytest.raises(ValueError, match="closed"): + read_stream(inp) + + +def corpus() -> list[bytes]: + """ + Return valid streams to start the fuzzing from. + + Triples and quads in the row layout (Jelly-RDF 1.1) and the column layout + (1.2), delimited and not. They have every term type, lookup entries, repeated + terms, namespace declarations and quoted triples. + """ + ex = "http://example.org/" + s, p, o = IRI(ex + "s"), IRI(ex + "p"), IRI(ex + "o") + triples = [ + Triple(s, p, o), + Triple(s, p, Literal("plain")), + Triple(s, IRI(ex + "q"), Literal("1", datatype="http://dt/int")), + Triple(BlankNode("b1"), p, Literal("hello", langtag="en")), + Triple(BlankNode("b1"), p, Triple(s, p, o)), + Triple(s, p, o), + ] + graph = IRI(ex + "g") + quads = [ + Quad(t.s, t.p, t.o, graph if i % 2 else DefaultGraph) + for i, t in enumerate(triples) + ] + streams = [] + for statements, logical_type in ( + (triples, jelly.LOGICAL_STREAM_TYPE_FLAT_TRIPLES), + (quads, jelly.LOGICAL_STREAM_TYPE_FLAT_QUADS), + ): + for version, frame_size in ((2, 4), (2, 250), (3, 4), (3, 250)): + sink = GenericStatementSink() + sink.bind("ex", IRI(ex)) + for statement in statements: + sink.add(statement) + params = StreamParameters( + version=version, + rdf_star=version == 2, + namespace_declarations=True, + delimited=frame_size == 4, + ) + serializer_options = SerializerOptions( + frame_size=frame_size, + logical_type=logical_type, + params=params, + lookup_preset=LookupPreset( + max_names=128, max_prefixes=8, max_datatypes=8 + ), + ) + out = io.BytesIO() + for frame in stream_frames(guess_stream(serializer_options, sink), sink): + if params.delimited: + write_delimited(frame, out) + else: + write_single(frame, out) + streams.append(out.getvalue()) + return streams + + +def test_corpus_is_valid() -> None: + for data in corpus(): + assert len([s for s in read(data) if isinstance(s, (Triple, Quad))]) == 6 + + +def test_fuzz_mutated_streams() -> None: + reached = 0 + + def consume(data: bytes) -> None: + nonlocal reached + try: + read(data) + except JellyConformanceError as error: + if "Is it cut short" not in str(error): + reached += 1 + raise + reached += 1 + + # Options whose physical and logical types don't match raise + # JellyAssertionError + found = byte_fuzzer.findings( + corpus(), consume, (JellyConformanceError, JellyAssertionError) + ) + assert not found, "\n".join(found) + # Most mutations are rejected by protobuf. At least 5% must get to the decoder + assert reached > byte_fuzzer.ITERATIONS // 20 diff --git a/tests/unit_tests/test_parse/test_sparql_hardening.py b/tests/unit_tests/test_parse/test_sparql_hardening.py new file mode 100644 index 00000000..d9c2e176 --- /dev/null +++ b/tests/unit_tests/test_parse/test_sparql_hardening.py @@ -0,0 +1,399 @@ +""" +Reading invalid Jelly-SPARQL input, after jelly-jvm's SparqlDecoderHardeningSpec. + +The frames are valid protobuf, but declare huge sizes or refer to things that are +not there. The reader must reject them with JellyConformanceError, without +allocating memory for the declared sizes. +""" + +from __future__ import annotations + +import io +import warnings +from typing import Any + +import pytest + +from pyjelly import jelly +from pyjelly.errors import ( + JellyConformanceError, + JellyIncompleteResultsError, + JellyTruncatedResultsWarning, +) +from pyjelly.integrations.generic import sparql as generic_sparql +from pyjelly.integrations.generic.decode import GenericStatementSinkAdapter +from pyjelly.integrations.generic.generic_sink import IRI, BlankNode, Literal +from pyjelly.options import ( + MAX_SPARQL_FRAME_ROWS, + MAX_SPARQL_FRAME_VALUES, + LookupPreset, + SparqlOptions, +) +from pyjelly.parse.sparql import SparqlDecoder, read_result_sets +from tests.utils import byte_fuzzer + +OPTIONS = jelly.SparqlResultsOptions( + max_name_table_size=128, + max_prefix_table_size=16, + max_datatype_table_size=16, + version=1, +) +DATATYPES = OPTIONS.max_datatype_table_size + +Entry = jelly.RdfLookupEntryPacked +Column = jelly.RdfColumn + +KIND_IRI = 0 +KIND_LITERAL = 1 +KIND_BNODE = 2 + + +def decoder( + *, max_rows: int = MAX_SPARQL_FRAME_ROWS, max_values: int = MAX_SPARQL_FRAME_VALUES +) -> SparqlDecoder: + return SparqlDecoder( + GenericStatementSinkAdapter, max_rows=max_rows, max_values=max_values + ) + + +def frame( + rows: int, *columns: jelly.RdfColumn, **fields: Any +) -> jelly.SparqlResultsFrame: + """Return a frame with the stream options and the variable x, unless given.""" + fields.setdefault("variables", ["x"]) + return jelly.SparqlResultsFrame( + options=OPTIONS, row_count=rows, columns=list(columns), **fields + ) + + +def kinds(values: list[int]) -> bytes: + """Pack the kinds of the values, 2 bits each.""" + out = bytearray((len(values) + 3) // 4) + for i, kind in enumerate(values): + out[i >> 2] |= kind << ((i & 3) << 1) + return bytes(out) + + +def datatype(index: int) -> int: + """Return the literal kind of a literal with this datatype.""" + return 2 * index - 1 + + +def langtag(index: int) -> int: + """Return the literal kind of a literal with this language tag.""" + return 2 * index + 2 + + +def decode(data: jelly.SparqlResultsFrame, **limits: int) -> list[Any]: + return decoder(**limits).decode(data).rows + + +REJECTED = { + "row count above 2^27": (frame(1 << 27, variables=[]), "at most 134217727 rows"), + "row count above 2^31": (frame(2**31 - 1, Column()), "at most 134217727 rows"), + "row count above 2^32 - 1": ( + frame(2**32 - 1, variables=[]), + "not 4294967295", + ), + "row count above the default limit": ( + frame(MAX_SPARQL_FRAME_ROWS + 1, variables=[]), + f"more than the {MAX_SPARQL_FRAME_ROWS} allowed", + ), + "datatype entry past the end": ( + frame(0, Column(), datatypes=[Entry(id=DATATYPES + 1, values=["http://dt"])]), + "the lookup has 16 entries", + ), + "datatype entry id above 2^31": ( + frame(0, Column(), datatypes=[Entry(id=2**32 - 1, values=["http://dt"])]), + "the lookup has 16 entries", + ), + "datatype entries that run off the end": ( + frame( + 0, + Column(), + datatypes=[ + Entry(id=1, values=[f"http://dt{i}" for i in range(DATATYPES + 1)]) + ], + ), + "the lookup has 16 entries", + ), + "literal with a datatype past the end": ( + frame(1, Column(lex_values=["1"], literal_kinds=[datatype(DATATYPES + 1)])), + "outside the datatype lookup", + ), + "literal with a datatype never set": ( + frame(1, Column(lex_values=["1"], literal_kinds=[datatype(1)])), + "lookup entry 1, which is not set", + ), + "literal kind per value with a datatype never set": ( + frame(2, Column(lex_values=["1", "2"], literal_kinds=[0, datatype(1)])), + "lookup entry 1, which is not set", + ), + "literal with a language tag the column does not have": ( + frame( + 2, + Column( + lex_values=["a", "b"], + literal_kinds=[langtag(0), langtag(1)], + langtags=["en"], + ), + ), + "language tag 1", + ), + "uniform literal kind with no language tags": ( + frame(1, Column(lex_values=["a"], literal_kinds=[langtag(0)])), + "language tag 0", + ), + "more than one literal kind, but not one per literal": ( + frame(3, Column(lex_values=["a", "b", "c"], literal_kinds=[0, 0])), + "2 literal kinds for 3 literals", + ), + "base directions for only some language tags": ( + frame( + 2, + Column( + lex_values=["a", "b"], + literal_kinds=[langtag(0), langtag(1)], + langtags=["en", "fr"], + langtag_directions=[jelly.RDF_BASE_DIRECTION_LTR], + ), + ), + "1 base directions for 2 language tags", + ), + "unknown base direction": ( + frame( + 1, + Column( + lex_values=["a"], + literal_kinds=[langtag(0)], + langtags=["en"], + langtag_directions=[7], + ), + ), + "unknown base direction: 7", + ), + "kinds of the wrong length": ( + frame(2, Column(kinds=b"\x06\x00", bnodes=["a"], lex_values=["x"])), + "2 bytes of kinds for 2 values", + ), + "kinds that do not match the values": ( + frame( + 2, + Column( + kinds=kinds([KIND_BNODE, KIND_BNODE]), bnodes=["a"], lex_values=["x"] + ), + ), + "more blank nodes than it has", + ), + "kinds with unused bits set": ( + frame(2, Column(kinds=bytes([0x06 | 0x40]), bnodes=["a"], lex_values=["x"])), + "unused bits", + ), + "IRI with a name never set": ( + frame( + 1, + Column(name_ids=[1], prefix_ids=[1]), + prefixes=[Entry(id=1, values=["http://example.org/"])], + ), + "name 1 or prefix 1, which is not set", + ), + "IRI with a prefix never set": ( + frame( + 1, + Column(name_ids=[1], prefix_ids=[1]), + names=[Entry(id=1, values=["x"])], + ), + "name 1 or prefix 1, which is not set", + ), + "IRI with kinds, with a name never set": ( + frame( + 1, + Column(kinds=kinds([KIND_IRI]), name_ids=[1], prefix_ids=[1]), + prefixes=[Entry(id=1, values=["http://example.org/"])], + ), + "name 1 or prefix 1, which is not set", + ), + "IRI with a name id above 2^31": ( + frame(1, Column(name_ids=[2**32 - 1])), + "outside the lookups", + ), + "IRI with a prefix id above 2^31": ( + frame( + 1, + Column(name_ids=[1], prefix_ids=[2**32 - 1]), + names=[Entry(id=1, values=["x"])], + ), + "outside the lookups", + ), +} + + +@pytest.mark.parametrize(("data", "error"), list(REJECTED.values()), ids=list(REJECTED)) +def test_rejects(data: jelly.SparqlResultsFrame, error: str) -> None: + with pytest.raises(JellyConformanceError, match=error): + decode(data) + + +def test_row_count_up_to_the_limit() -> None: + data = frame( + 5, Column(name_ids=[1]), names=[Entry(id=1, values=["http://example.org/x"])] + ) + rows = decode(data, max_rows=5) + assert rows == [(IRI("http://example.org/x"),)] + [(None,)] * 4 + with pytest.raises(JellyConformanceError, match="more than the 4 allowed"): + decode(data, max_rows=4) + + +def test_value_count_above_the_default_limit() -> None: + # 1000 empty columns of 2^20 rows each: a few kilobytes of input that would + # need gigabytes of cells. The row count is within the row limit. + variables = [f"v{i}" for i in range(1000)] + data = frame( + MAX_SPARQL_FRAME_ROWS, *[Column()] * len(variables), variables=variables + ) + with pytest.raises( + JellyConformanceError, match=f"the {MAX_SPARQL_FRAME_VALUES} values allowed" + ): + decode(data) + + +def test_value_count_up_to_the_limit() -> None: + def data(rows: int) -> jelly.SparqlResultsFrame: + return frame(rows, Column(), Column(), variables=["x", "y"]) + + assert len(decode(data(5), max_values=10)) == 5 + with pytest.raises(JellyConformanceError, match="6 rows of 2 variables"): + decode(data(6), max_values=10) + + +# A column long enough to check that the kinds are read to the end +LONG_COLUMN = 70_001 + + +def long_mixed_column() -> tuple[jelly.RdfColumn, list[int]]: + """Return a column of blank nodes and literals, and its kinds.""" + values = [KIND_BNODE if i % 3 == 0 else KIND_LITERAL for i in range(LONG_COLUMN)] + column = Column( + kinds=kinds(values), + bnodes=[f"b{i}" for i in range(0, LONG_COLUMN, 3)], + lex_values=[f"l{i}" for i in range(LONG_COLUMN) if i % 3], + ) + return column, values + + +def test_long_mixed_column() -> None: + column, _ = long_mixed_column() + rows = decode(frame(LONG_COLUMN, column)) + assert len(rows) == LONG_COLUMN + for i in (0, 1, 65531, 65532, 65533, 70000): + expected = BlankNode(f"b{i}") if i % 3 == 0 else Literal(f"l{i}") + assert str(rows[i][0]) == str(expected) + + +def test_long_mixed_column_with_a_mismatch_near_the_end() -> None: + column, values = long_mixed_column() + # Value 69998 is a literal, the kinds say it is a blank node + values[69998] = KIND_BNODE + column.kinds = kinds(values) + with pytest.raises(JellyConformanceError, match="more blank nodes than it has"): + decode(frame(LONG_COLUMN, column)) + + +def test_kinds_on_a_column_of_one_type() -> None: + column = Column(kinds=kinds([KIND_BNODE, KIND_BNODE]), bnodes=["a", "b"]) + rows = decode(frame(2, column)) + assert [str(row[0]) for row in rows] == ["_:a", "_:b"] + + +def test_base_direction_checked_only_where_used() -> None: + # The second tag has an unknown direction, but no value refers to it + column = Column( + lex_values=["a"], + literal_kinds=[langtag(0)], + langtags=["en", "fr"], + langtag_directions=[jelly.RDF_BASE_DIRECTION_UNSPECIFIED, 7], + ) + assert decode(frame(1, column)) == [(Literal("a", langtag="en"),)] + + +def test_column_with_no_values() -> None: + assert decode(frame(1, Column())) == [(None,)] + + +def corpus() -> list[bytes]: + """ + Return valid streams to start the fuzzing from. + + They have every term type, columns with and without kinds, lookup entries, + repeated and unbound cells, delimited and non-delimited streams, a trailer + and a boolean result. + """ + a = IRI("http://example.org/a") + rows: list[tuple[Any, ...]] = [ + (a, Literal("plain"), BlankNode("b1")), + # Repeats the row before it + (a, Literal("plain"), BlankNode("b1")), + (IRI("http://example.org/b"), Literal("1", datatype="http://dt/int"), None), + # Mixes term types in the columns, so that they get kinds + (BlankNode("b2"), Literal("hello", langtag="en"), IRI("http://example.org/c")), + (None, None, None), + ( + IRI("http://example.org/d"), + Literal("x", langtag="ar", direction="rtl"), + BlankNode("b3"), + ), + ] + small = LookupPreset(max_names=128, max_prefixes=8, max_datatypes=8) + streams = [] + for options in ( + SparqlOptions(lookup_preset=small, delimited=False), + SparqlOptions(lookup_preset=small, frame_values=6), + ): + out = io.BytesIO() + generic_sparql.serialize_jelly_sparql(["x", "y", "z"], rows, out, options) + streams.append(out.getvalue()) + out = io.BytesIO() + generic_sparql.serialize_jelly_sparql_ask( + value=True, out=out, options=SparqlOptions(lookup_preset=small, delimited=False) + ) + streams.append(out.getvalue()) + return streams + + +def read_all(data: bytes) -> None: + with warnings.catch_warnings(): + warnings.simplefilter("ignore", JellyTruncatedResultsWarning) + # A low row limit, so that mutated row counts are rejected quickly + for result_set in read_result_sets( + data, GenericStatementSinkAdapter, max_rows=64 + ): + if result_set.is_boolean: + result_set.boolean # noqa: B018 + else: + list(result_set) + + +def test_corpus_is_valid() -> None: + for data in corpus(): + read_all(data) + + +def test_fuzz_mutated_streams() -> None: + reached = 0 + + def consume(data: bytes) -> None: + nonlocal reached + try: + read_all(data) + except JellyConformanceError as error: + if "Is it cut short" not in str(error): + reached += 1 + raise + reached += 1 + + found = byte_fuzzer.findings( + corpus(), consume, (JellyConformanceError, JellyIncompleteResultsError) + ) + assert not found, "\n".join(found) + # Most mutations are rejected by protobuf. At least 5% must get to the decoder + assert reached > byte_fuzzer.ITERATIONS // 20 diff --git a/tests/utils/byte_fuzzer.py b/tests/utils/byte_fuzzer.py new file mode 100644 index 00000000..4245ab62 --- /dev/null +++ b/tests/utils/byte_fuzzer.py @@ -0,0 +1,107 @@ +""" +Byte-level mutational fuzzing of serialized messages. + +Ported from jelly-jvm's `ByteFuzzer`. Valid messages are edited at random. Some +of them still parse as protobuf, but their contents are invalid, and the reader +must reject them with its own error. + +Runs are seeded, so a finding can be reproduced from the seed and iteration. Set +the JELLY_FUZZ_ITERATIONS and JELLY_FUZZ_SEED environment variables to change +them, e.g. for a longer run. +""" + +from __future__ import annotations + +import os +import random +import traceback +from collections.abc import Callable, Sequence +from typing import Final + +ITERATIONS: Final = int(os.environ.get("JELLY_FUZZ_ITERATIONS", "30000")) +SEED: Final = int(os.environ.get("JELLY_FUZZ_SEED", "20260918")) + +MAX_REPORTED: Final = 5 +MAX_SIZE: Final = 1 << 16 + +# Bytes with a special meaning in protobuf: the smallest and largest one-byte +# varints, a continuation bit, and the tag of field 1 +INTERESTING: Final = (0x00, 0x01, 0x7F, 0x80, 0xFF, 0x0A) + + +def mutate(data: bytes, rnd: random.Random) -> bytes: + """Apply 1-4 random edits to the bytes.""" + for _ in range(rnd.randrange(1, 5)): + edited = _edit(data, rnd) + if len(edited) <= MAX_SIZE: + data = edited + return data + + +def _edit(data: bytes, rnd: random.Random) -> bytes: # noqa: PLR0911 + if not data: + return data + size = len(data) + at = rnd.randrange(size) + edit = rnd.randrange(6) + if edit == 0: + # Flip one bit + return data[:at] + bytes([data[at] ^ (1 << rnd.randrange(8))]) + data[at + 1 :] + if edit == 1: + # Replace one byte with a random value + return data[:at] + bytes([rnd.randrange(256)]) + data[at + 1 :] + if edit == 2: # noqa: PLR2004 + # Replace one byte with one of INTERESTING + return data[:at] + bytes([rnd.choice(INTERESTING)]) + data[at + 1 :] + if edit == 3: # noqa: PLR2004 + # Cut the message short + return data[:at] + if edit == 4: # noqa: PLR2004 + # Repeat a slice, so repeated fields and submessages get longer + end = at + rnd.randrange(size - at) + 1 + return data[:end] + data[at:end] + data[end:] + # Insert 1-4 random bytes + return data[:at] + rnd.randbytes(rnd.randrange(1, 5)) + data[at:] + + +def findings( + corpus: Sequence[bytes], + consume: Callable[[bytes], object], + expected: tuple[type[BaseException], ...], + iterations: int = ITERATIONS, + seed: int = SEED, +) -> list[str]: + """ + Feed mutated copies of the corpus to `consume`. + + Returns: + list[str]: one description per exception that is not `expected`, with + enough to reproduce it. + + """ + rnd = random.Random(seed) # noqa: S311 + found: list[str] = [] + count = 0 + for i in range(iterations): + data = mutate(rnd.choice(corpus), rnd) + try: + consume(data) + except expected: + pass + except Exception as error: # noqa: BLE001 + count += 1 + if len(found) < MAX_REPORTED: + found.append(_describe(seed, i, data, error)) + if count > len(found): + found.append(f"...and {count - len(found)} more") + return found + + +def _describe(seed: int, iteration: int, data: bytes, error: Exception) -> str: + where = traceback.extract_tb(error.__traceback__)[-1] + more = f"... ({len(data)} bytes)" if len(data) > 96 else "" # noqa: PLR2004 + return ( + f"seed {seed}, iteration {iteration}: {type(error).__name__}: {error}\n" + f" at {where.filename}:{where.lineno}\n" + f" input: {data[:96].hex()}{more}" + ) From 40d7105b40b82887c726883936a9de20889e0a3a Mon Sep 17 00:00:00 2001 From: Ostrzyciel Date: Tue, 27 Oct 2026 09:30:25 +0100 Subject: [PATCH 24/24] Add final benchmarks Also update the default frame size for Jelly-RDF to 1024, which was found in Jelly-JVM to work best. --- benchmarks/README.md | 3 ++- benchmarks/measure.py | 13 ++++++++----- benchmarks/methods.py | 20 +++++++++++++++----- benchmarks/suites.py | 18 +++++++++++++----- pyjelly/serialize/flows.py | 2 +- 5 files changed, 39 insertions(+), 17 deletions(-) diff --git a/benchmarks/README.md b/benchmarks/README.md index ddf7b0f3..be0dc605 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -87,8 +87,9 @@ Method names are `-`, plus `-` for Jelly, which has two APIs | `rdflib-jelly-graph` | Jelly through rdflib's plugin API: `Graph.serialize` | `Graph.parse` | | `rdflib-jelly-stream` | Jelly with pyjelly's flat streaming API, from the list | `parse_jelly_flat` | | `generic-jelly-stream` | The same, with the generic integration's terms instead of rdflib's | `parse_jelly_flat` | +| `rdflib-jelly11-stream`, `generic-jelly11-stream` | The same two, writing Jelly-RDF 1.1 instead of 1.2 | `parse_jelly_flat` | -All Jelly methods use pyjelly's default options: the "big" lookup preset (4000 names, 150 prefixes, 32 datatypes) and frames of 250 rows. The other formats' writers get the dataset's prefixes. +All Jelly methods use pyjelly's default options: the "big" lookup preset (4000 names, 150 prefixes, 32 datatypes) and frames of 1024 rows. The other formats' writers get the dataset's prefixes. ### Tasks diff --git a/benchmarks/measure.py b/benchmarks/measure.py index 6ddf0081..6306c44f 100644 --- a/benchmarks/measure.py +++ b/benchmarks/measure.py @@ -151,11 +151,6 @@ def operation() -> None: def operation() -> int: return int(method.read(written, quads=data.quads)) - # Everything made so far lives until the end. Freezing it keeps the garbage - # collector from going through it again and again during the measurement. - gc.collect() - gc.freeze() - _measure( operation, result, @@ -177,6 +172,13 @@ def _measure( # noqa: PLR0913 profile: Path | None, ) -> None: """Run the warmup and measured iterations, and fill in the result.""" + # Everything made so far lives until the end. Freezing it keeps the garbage + # collector from going through it again and again during the measurement. + # PyPy has no gc.freeze(), and doesn't need it: its collector doesn't go + # through old objects again unless they change. + gc.collect() + if hasattr(gc, "freeze"): + gc.freeze() result.warmup = [ _iteration(operation, result.statements, seconds) for _ in range(warmup) ] @@ -212,6 +214,7 @@ def environment() -> dict[str, Any]: "platform": platform.platform(), "machine": platform.machine(), "cpu_affinity": sorted(affinity) if affinity else None, + "pyjelly": importlib.metadata.version("pyjelly"), "pyjelly_commit": commit, "pyjelly_compiled_with_mypyc": compiled, "pyjelly_max_jelly_version": MAX_VERSION, diff --git a/benchmarks/methods.py b/benchmarks/methods.py index 7d08204c..0a906057 100644 --- a/benchmarks/methods.py +++ b/benchmarks/methods.py @@ -28,6 +28,8 @@ from pyjelly.integrations.generic import serialize as generic_serialize from pyjelly.integrations.rdflib import parse as rdflib_parse from pyjelly.integrations.rdflib import serialize as rdflib_serialize +from pyjelly.options import StreamParameters +from pyjelly.serialize.streams import SerializerOptions class NullOutput(io.BytesIO): @@ -254,12 +256,14 @@ class PyjellyFlat(Method): Jelly with pyjelly's flat streaming API, using pyjelly's default options. `integration` is "rdflib" for statements made of rdflib terms, or "generic" - for pyjelly's own terms. + for pyjelly's own terms. With `version`, the stream is written in that protocol + version instead of the default one, e.g. 2 for Jelly-RDF 1.1. """ - def __init__(self, integration: str) -> None: - self.name = f"{integration}-jelly-stream" + def __init__(self, integration: str, version: int | None = None) -> None: + self.name = f"{integration}-jelly{'11' if version == 2 else ''}-stream" # noqa: PLR2004 self.generic = integration == "generic" + self.version = version @override def statements(self, data: BenchData) -> list[Any]: @@ -269,10 +273,13 @@ def statements(self, data: BenchData) -> list[Any]: def write(self, data: BenchData, out: IO[bytes]) -> None: # The API takes a generator, so the list is wrapped in one statements = (statement for statement in self.statements(data)) + options = None + if self.version is not None: + options = SerializerOptions(params=StreamParameters(version=self.version)) if self.generic: - generic_serialize.flat_stream_to_file(statements, out) + generic_serialize.flat_stream_to_file(statements, out, options) else: - rdflib_serialize.flat_stream_to_file(statements, out) + rdflib_serialize.flat_stream_to_file(statements, out, options) def _parse(self, data: bytes) -> Iterable[Any]: if self.generic: @@ -305,6 +312,9 @@ def read_all(self, data: bytes, *, quads: bool) -> list[tuple[Any, ...]]: # Jelly through pyjelly's flat streaming API PyjellyFlat("rdflib"), PyjellyFlat("generic"), + # The same, writing Jelly-RDF 1.1, to compare it with 1.2 + PyjellyFlat("rdflib", version=2), + PyjellyFlat("generic", version=2), ) } diff --git a/benchmarks/suites.py b/benchmarks/suites.py index 3beab54e..2c069684 100644 --- a/benchmarks/suites.py +++ b/benchmarks/suites.py @@ -6,9 +6,8 @@ from dataclasses import dataclass from typing import Any -from benchmarks import data, sparql_data +from benchmarks import data from benchmarks.methods import METHODS, round_trip -from benchmarks.sparql_methods import SPARQL_METHODS, sparql_round_trip @dataclass(frozen=True) @@ -33,7 +32,17 @@ class Suite: data.load, round_trip, ), - "sparql": Suite( +} + +try: + from benchmarks import sparql_data + from benchmarks.sparql_methods import SPARQL_METHODS, sparql_round_trip +except ImportError: + # pyjelly 0.8 and older have no Jelly-SPARQL, but the rdf suite still runs on + # them, to compare them with the current version + pass +else: + SUITES["sparql"] = Suite( "sparql", "cells", sparql_data.DATASETS, @@ -41,5 +50,4 @@ class Suite: SPARQL_METHODS, sparql_data.load, sparql_round_trip, - ), -} + ) diff --git a/pyjelly/serialize/flows.py b/pyjelly/serialize/flows.py index 8fb5f785..10e8237a 100644 --- a/pyjelly/serialize/flows.py +++ b/pyjelly/serialize/flows.py @@ -8,7 +8,7 @@ if TYPE_CHECKING: from pyjelly.serialize.columns import ColumnBatch -DEFAULT_FRAME_SIZE = 250 +DEFAULT_FRAME_SIZE = 1024 class FrameFlow: