From 455094e2d345dd0bf760a7642b91c233644c5a11 Mon Sep 17 00:00:00 2001 From: SkyeAv Date: Thu, 15 Oct 2026 13:02:22 -0700 Subject: [PATCH 1/5] feat: [US-001] add no_map opt-out to NodeEncoding with config-time validation Add no_map: bool = False to NodeEncoding so statement subjects and objects whose data already carries CURIEs can skip fullmap entity resolution and use the encoded value verbatim as the node id. - reject no_map combined with resolution-only knobs (prioritize, avoid, exclude_prefixes, exclude_regex, explicit non-default taxon) as dead config - validate no_map literals are CURIE-shaped (after prefix/suffix), so a bad literal fails at config time instead of silently emptying the build - Qualifier.resolved becomes the single routing predicate: enum-ranged qualifiers stay verbatim, CURIE-ranged ones resolve unless no_map opts out; no_map on an enum-ranged qualifier is rejected as redundant --- src/tablassert/errors.py | 2 + src/tablassert/models.py | 113 +++++++++++++++++++++++++++++++++++++-- tests/test_models.py | 86 +++++++++++++++++++++++++++++ 3 files changed, 198 insertions(+), 3 deletions(-) diff --git a/src/tablassert/errors.py b/src/tablassert/errors.py index 0414023..41041dc 100644 --- a/src/tablassert/errors.py +++ b/src/tablassert/errors.py @@ -63,6 +63,8 @@ def redact_secrets(text: str, secrets: Sequence[str] = ()) -> str: "provenance-missing-publication", "provenance-publication-and-override", "encoding-list-method-removed", + "no-map-dead-config", + "no-map-bad-curie", "annotation-split-by-requires-column", "annotation-split-by-empty", "field-disabled", diff --git a/src/tablassert/models.py b/src/tablassert/models.py index 138c3c4..81689d8 100644 --- a/src/tablassert/models.py +++ b/src/tablassert/models.py @@ -288,6 +288,16 @@ def polars_compatible_replacement(cls, remove: list[int | float | str] | None) - ) +#: CURIE shape enforced on verbatim (``no_map``) node values: non-empty ``PREFIX:local-id``. +#: Shared by config-time literal validation (models) and build-time cell audits (lib). +CURIE_PATTERN: re.Pattern[str] = re.compile(r"^[A-Za-z][A-Za-z0-9_.-]*:[A-Za-z0-9_.\-/]+$") + + +def is_curie(value: object) -> bool: + """Whether ``value`` is a CURIE-shaped string (``PREFIX:local-id``, both halves non-empty).""" + return isinstance(value, str) and CURIE_PATTERN.match(value.strip()) is not None + + class NodeEncoding(Encoding): taxon: PositiveInt | None = Field( 9606, @@ -312,6 +322,87 @@ class NodeEncoding(Encoding): examples=[["^OMIM:\\d+$"]], ) + no_map: bool = Field( + False, + description=( + "When True, skip fullmap entity resolution: the encoded value is used verbatim as the node CURIE, " + "for data that already carries its own CURIEs. No NLP normalization, taxon constraint, or category " + "lookup is applied, and cells that are blank or not CURIE-shaped are dropped like unresolved nodes." + ), + examples=[True], + ) + + @model_validator(mode="before") + @classmethod + def no_map_rejects_explicit_taxon(cls, data: Any) -> Any: + """Reject ``no_map`` with an explicit non-default ``taxon`` (dead resolution config). + + ``taxon`` defaults to 9606 and only steers fullmap ranking, so an explicit + non-default taxon alongside ``no_map: true`` is config that can never take + effect. The default itself stays indistinguishable from an explicit 9606 and + is therefore allowed. Checked ``before`` precisely so "explicitly set" is + detectable from the raw input dict. + """ + if isinstance(data, dict) and data.get("no_map") and "taxon" in data and data["taxon"] not in (None, 9606): + raise TablassertValidationError( + f"`no_map: true` skips entity resolution, so `taxon: {data['taxon']}` can never take effect; remove the taxon constraint.", + code="no-map-dead-config", + ) + return data + + @model_validator(mode="after") + def no_map_rejects_resolution_knobs(self: Self) -> Self: + """Reject ``no_map`` combined with resolution-only knobs (dead config). + + ``prioritize``, ``avoid``, ``exclude_prefixes``, and ``exclude_regex`` only + steer fullmap ranking and filtering; with ``no_map: true`` no lookup happens, + so these fields mislead the reader. Fail loudly at config time, like the + ``nullable``-on-literal check on :class:`Qualifier`. + """ + if not self.no_map: + return self + dead: list[str] = [ + name + for name, value in ( + ("prioritize", self.prioritize), + ("avoid", self.avoid), + ("exclude_prefixes", self.exclude_prefixes), + ("exclude_regex", self.exclude_regex), + ) + if value + ] + if dead: + raise TablassertValidationError( + f"`no_map: true` skips entity resolution, so {', '.join(f'`{name}`' for name in dead)} can never take effect; remove them.", + code="no-map-dead-config", + ) + return self + + @model_validator(mode="after") + def no_map_literal_must_be_curie(self: Self) -> Self: + """Validate that a ``no_map`` literal encodes a CURIE at config time. + + A ``method: value`` node with ``no_map: true`` becomes the node id verbatim, + so the literal (after ``prefix``/``suffix`` are applied, matching the build's + ``encoding()``) must already be CURIE-shaped; otherwise every row would be + dropped at build time. ``method: column`` cells are audited per row at build + time instead. + """ + if not self.no_map or self.method != EncodingMethods.VALUE: + return self + # Enum-ranged qualifier literals are closed-vocabulary tokens (audited against + # the vocabulary, and `no_map` on them is rejected as dead config), not CURIEs. + if getattr(self, "vocabulary", None) is not None: + return self + literal: str = f"{self.prefix or ''}{self.encoding}{self.suffix or ''}".strip() + if not is_curie(literal): + raise TablassertValidationError( + f"`no_map: true` uses the encoded value verbatim as the node CURIE, but the literal {literal!r} is not CURIE-shaped " + "(expected `PREFIX:local-id`). Fix the literal, or add `prefix`/`suffix` to form a CURIE.", + code="no-map-bad-curie", + ) + return self + @field_validator("exclude_regex", mode="after") @classmethod def polars_compatible_exclude_regex(cls, exclude_regex: list[str] | None) -> list[str] | None: @@ -359,10 +450,26 @@ def vocabulary(self: Self) -> frozenset[str] | None: def resolved(self: Self) -> bool: """Whether this qualifier's values go through fullmap entity resolution. - Only CURIE-ranged qualifiers are resolved. Enum-ranged ones carry a literal - token from a closed Biolink vocabulary and must be passed through verbatim. + Enum-ranged qualifiers carry a literal token from a closed Biolink vocabulary + and must be passed through verbatim. CURIE-ranged qualifiers are resolved + unless they opt out with ``no_map: true`` (their values are already CURIEs). + """ + return self.vocabulary is None and not self.no_map + + @model_validator(mode="after") + def reject_no_map_on_enum_ranged(self: Self) -> Self: + """Reject ``no_map: true`` on enum-ranged qualifiers as dead config. + + Enum-ranged qualifiers already pass their literal tokens through verbatim, so + ``no_map`` would be a no-op that misleads the reader; fail loudly at config + time, like the ``nullable``-on-literal check below. """ - return self.vocabulary is None + if self.no_map and self.vocabulary is not None: + raise TablassertValidationError( + f"{self.qualifier} has a closed Biolink vocabulary and already passes values through verbatim, so `no_map` is redundant; remove it.", + code="no-map-dead-config", + ) + return self @model_validator(mode="after") def reject_disabled_qualifiers(self: Self) -> Self: diff --git a/tests/test_models.py b/tests/test_models.py index 2cde591..4c15017 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -351,6 +351,92 @@ def test_node_encoding_explicit_null_disables_taxon() -> None: assert node.taxon is None +def test_node_encoding_no_map_defaults_false() -> None: + """no_map defaults to False so every existing config resolves exactly as before.""" + node: NodeEncoding = NodeEncoding(method="value", encoding="BRCA1") # pyright: ignore + assert node.no_map is False + + +def test_node_encoding_no_map_literal_must_be_curie() -> None: + """A no_map literal becomes the node id verbatim, so a non-CURIE literal is rejected at config time. + + Without this check every row would build-drop at the CURIE audit, turning one + config typo into a silently empty node file. + """ + with pytest.raises(ValidationError) as exc_info: + NodeEncoding(method="value", encoding="BRCA1", no_map=True) # pyright: ignore + assert "no-map-bad-curie" in str(exc_info.value) + + +def test_node_encoding_no_map_literal_curie_accepted() -> None: + """A CURIE-shaped no_map literal passes config validation.""" + node: NodeEncoding = NodeEncoding(method="value", encoding="HGNC:1100", no_map=True) # pyright: ignore + assert node.no_map is True + + +def test_node_encoding_no_map_literal_prefix_suffix_form_curie() -> None: + """prefix/suffix participate in the literal CURIE check, matching the build's encoding().""" + node: NodeEncoding = NodeEncoding(method="value", encoding="1100", prefix="HGNC:", no_map=True) # pyright: ignore + assert node.no_map is True + with pytest.raises(ValidationError) as exc_info: + NodeEncoding(method="value", encoding="1100", prefix="HGNC", no_map=True) # pyright: ignore + assert "no-map-bad-curie" in str(exc_info.value) + + +def test_node_encoding_no_map_column_needs_no_literal_curie() -> None: + """method: column no_map nodes skip literal validation; cells are audited per row at build time.""" + node: NodeEncoding = NodeEncoding(method="column", encoding="A", no_map=True) # pyright: ignore + assert node.no_map is True + + +@pytest.mark.parametrize( + "kwargs", + [ + {"prioritize": [Categories.GENE]}, + {"avoid": [Categories.DISEASE]}, + {"exclude_prefixes": ["OMIM"]}, + {"exclude_regex": ["^OMIM:\\d+$"]}, + {"taxon": 10090}, + ], +) +def test_node_encoding_no_map_rejects_resolution_knobs(kwargs: dict[str, Any]) -> None: + """Resolution-only knobs alongside no_map: true are dead config and fail loudly at config time. + + Each knob only steers fullmap ranking/filtering; with no_map no lookup happens, + so keeping them would mislead the reader into thinking they take effect. + """ + with pytest.raises(ValidationError) as exc_info: + NodeEncoding(method="column", encoding="A", no_map=True, **kwargs) # pyright: ignore + assert "no-map-dead-config" in str(exc_info.value) + + +def test_node_encoding_no_map_allows_default_taxon() -> None: + """An explicit default taxon (9606) is indistinguishable from the unset default and stays allowed.""" + node: NodeEncoding = NodeEncoding(method="column", encoding="A", no_map=True, taxon=9606) # pyright: ignore + assert node.no_map is True + + +def test_qualifier_resolved_property_truth_table() -> None: + """resolved reflects vocabulary presence AND the no_map opt-out. + + Enum-ranged qualifiers never resolve; CURIE-ranged ones resolve unless no_map + opts them out. This property is the single routing predicate in lib._node_ops. + """ + enum_ranged: Qualifier = Qualifier(qualifier="object_direction_qualifier", method="value", encoding="increased") # pyright: ignore + assert enum_ranged.resolved is False + curie_ranged: Qualifier = Qualifier(qualifier="anatomical_context_qualifier", method="column", encoding="A") # pyright: ignore + assert curie_ranged.resolved is True + curie_no_map: Qualifier = Qualifier(qualifier="anatomical_context_qualifier", method="column", encoding="A", no_map=True) # pyright: ignore + assert curie_no_map.resolved is False + + +def test_qualifier_no_map_rejected_on_enum_ranged() -> None: + """no_map on an enum-ranged qualifier is redundant (already verbatim) and is rejected as dead config.""" + with pytest.raises(ValidationError) as exc_info: + models.Qualifier(qualifier="object_direction_qualifier", method="value", encoding="increased", no_map=True) # pyright: ignore + assert "no-map-dead-config" in str(exc_info.value) + + def test_qualifier_rejects_disabled_species_context_qualifier() -> None: """species_context_qualifier is disabled and cannot be manually declared.""" with pytest.raises(ValidationError) as exc_info: From b9d0940428849cf55742c1dd8ce97ce3130a95b7 Mon Sep 17 00:00:00 2001 From: SkyeAv Date: Thu, 15 Oct 2026 13:08:02 -0700 Subject: [PATCH 2/5] feat: [US-002] route no_map nodes around fullmap resolution in the section pipeline _node_ops now splits node columns into resolved (resolve_batch specs), verbatim (no_map subject/object and no_map CURIE-ranged qualifiers), and literal (enum-ranged qualifiers) groups. - new lib.curie_audit op: drops blank/null-like rows (a missing CURIE cannot form a node), raises no-map-curie-violation on non-CURIE cells, and synthesizes _category = biolink:NamedThing so edge_category degrades to biolink:Association like the documented no-role fallback - resolve_batch receives zero specs for an all-verbatim section, so the fullmap redb is never opened; mixed mapped/verbatim statements resolve the mapped side as before - fullmap_audit naturally skips verbatim columns (it iterates the resolved node_columns list) - register curie_audit in PHASE_OF (encode) and the runcache op-cost table --- src/tablassert/errors.py | 1 + src/tablassert/lib.py | 85 +++++++++++++++++++++++--- src/tablassert/runcache.py | 3 + tests/test_lib.py | 118 +++++++++++++++++++++++++++++++++++++ 4 files changed, 200 insertions(+), 7 deletions(-) diff --git a/src/tablassert/errors.py b/src/tablassert/errors.py index 41041dc..8d1b1de 100644 --- a/src/tablassert/errors.py +++ b/src/tablassert/errors.py @@ -65,6 +65,7 @@ def redact_secrets(text: str, secrets: Sequence[str] = ()) -> str: "encoding-list-method-removed", "no-map-dead-config", "no-map-bad-curie", + "no-map-curie-violation", "annotation-split-by-requires-column", "annotation-split-by-empty", "field-disabled", diff --git a/src/tablassert/lib.py b/src/tablassert/lib.py index bae8cb2..6063ab2 100644 --- a/src/tablassert/lib.py +++ b/src/tablassert/lib.py @@ -48,7 +48,7 @@ from tablassert.errors import TablassertError from tablassert.fullmap import ResolveSpec, fullmap_db_path, resolve, resolve_batch from tablassert.log import cat -from tablassert.models import EDGE_ID_PLACEHOLDER, Encoding, NodeEncoding, Qualifier, RIGConfig, Section +from tablassert.models import EDGE_ID_PLACEHOLDER, Encoding, NodeEncoding, Qualifier, RIGConfig, Section, is_curie from tablassert.nlp import level_one, level_two from tablassert.qc import fullmap_audit from tablassert.rig import ( @@ -1215,6 +1215,61 @@ def qualifier_vocabulary_audit(lf: pl.LazyFrame, col: str, vocabulary: Iterable[ ) +def curie_audit(lf: pl.LazyFrame, col: str, section: str, config_file: str) -> pl.LazyFrame: + """Finalize a ``no_map`` node column: blank rows drop, non-CURIE values fail, category is synthesized. + + Columns configured with ``no_map: true`` bypass entity resolution, so nothing else + (a) drops rows whose cell is blank (a missing node id can never form an edge, exactly + like an unresolved subject/object), (b) rejects cells that are not CURIE-shaped -- the + encoded value becomes the KGX node id verbatim, so garbage would ship as an identifier -- + or (c) produces the ``_category`` column downstream stages require: without a + fullmap lookup there is no role to derive, so the column is pinned to + ``biolink:NamedThing`` and edge derivation degrades to ``biolink:Association`` exactly + like the documented no-role-lookup fallback in :func:`edge_category`. + + Args: + lf: Source LazyFrame. + col: The no_map node column (``subject``/``object`` or a CURIE-ranged qualifier key). + section: Short section hash (for log context). + config_file: Originating config file (for log context). + + Returns: + LazyFrame with blank/null-like rows dropped and ``_category`` added. + + Notes: + A non-blank value that is not CURIE-shaped raises a coded error naming the + offending values (mirroring ``qualifier_vocabulary_audit``); blank and null-like + cells (``""``, ``"NA"``, ...) are dropped with a loud per-column count instead, + mirroring how an unresolvable subject/object row is dropped, never emitted. + """ + values: pl.Series = lf.select(pl.col(col)).collect().get_column(col) + if values.dtype.base_type() in (pl.List, pl.Array): + values = values.explode() + blank_mask: pl.Series = values.is_null() | values.str.strip_chars().str.to_lowercase().is_in(list(_NULL_LIKE_TEXT)) + offending: pl.Series = values.filter(~blank_mask & ~values.str.strip_chars().map_elements(is_curie, return_dtype=pl.Boolean)).unique().sort() + if offending.len() > 0: + shown: list[str] = offending.head(10).to_list() + more: str = "..." if offending.len() > len(shown) else "" + raise TablassertError( + f"{col} is configured with `no_map: true` so its values are used verbatim as node CURIEs, but " + f"{offending.len()} non-blank value(s) in the built column are not CURIE-shaped: " + f"{', '.join(repr(x) for x in shown)}{more}. Fix the source table or the encoding " + "(prefix/suffix can form a CURIE from raw cells).", + code="no-map-curie-violation", + ) + dropped: int = int(blank_mask.sum()) + if dropped: + logger.info( + "no_map column {col} in {config} ({hash}) dropped {n} blank row(s): a missing CURIE cannot form a node", + col=col, + config=config_file, + hash=section, + n=dropped, + ) + present: pl.Expr = pl.col(col).is_not_null() & ~pl.col(col).str.strip_chars().str.to_lowercase().is_in(list(_NULL_LIKE_TEXT)) + return lf.filter(present).with_columns(pl.lit("biolink:NamedThing").alias(f"{col}_category")) + + def to_store(lf: pl.LazyFrame, p: Path, config_name: str) -> Path: """Collect and write a section LazyFrame to a parquet file. @@ -1369,16 +1424,25 @@ def _node_ops(self: Self, db: Path) -> list[Any]: enabled. """ # Subject/object/qualifiers share one resolve_batch call instead of one per column. - # Enum-ranged qualifiers are excluded from resolution: their range is a closed - # Biolink vocabulary, so sending them through the fullmap would turn the required - # token `increased` into the CURIE `UMLS:C0205217`, which the slot rejects. + # Two kinds of columns never reach it: enum-ranged qualifiers, whose range is a + # closed Biolink vocabulary (sending them through the fullmap would turn the + # required token `increased` into the CURIE `UMLS:C0205217`, which the slot + # rejects), and no_map nodes, whose values are already CURIEs used verbatim. qualifiers: list[Qualifier] = self.statement.qualifiers or [] + subj_obj: list[tuple[NodeEncoding, str]] = [(self.statement.subject, "subject"), (self.statement.object, "object")] node_columns: list[tuple[NodeEncoding, str]] = [ - (self.statement.subject, "subject"), - (self.statement.object, "object"), + *[(x, col) for x, col in subj_obj if not x.no_map], *[(x, x.qualifier) for x in qualifiers if x.resolved], ] - literals: list[Qualifier] = [x for x in qualifiers if not x.resolved] + # no_map subject/object plus CURIE-ranged qualifiers that opted out (a no_map + # enum-ranged qualifier is rejected at config time). Their encoded values become + # node ids verbatim: encode only, then curie_audit drops blank rows, fails on + # non-CURIE cells, and synthesizes the category column edge derivation needs. + verbatim: list[tuple[NodeEncoding, str]] = [ + *[(x, col) for x, col in subj_obj if x.no_map], + *[(x, x.qualifier) for x in qualifiers if not x.resolved and x.no_map], + ] + literals: list[Qualifier] = [x for x in qualifiers if not x.resolved and not x.no_map] # A nullable qualifier keeps its edge when the cell is blank or unresolvable (the # column stays null and the null-stripper omits the key); subject/object and # strict qualifiers drop the row as before. @@ -1401,6 +1465,12 @@ def _node_ops(self: Self, db: Path) -> list[Any]: # ranged column ends with a vocabulary audit: nothing else validates its cell # values against the closed Biolink range before KGX emission. [[*self.encoding(x, x.qualifier), (qualifier_vocabulary_audit, (str(x.qualifier), sorted(x.vocabulary or ())))] for x in literals], + # no_map columns are encoded exactly like enum-ranged literals (no + # pre-resolution snapshot, no NLP normalization -- both exist only to feed + # entity resolution these columns never undergo), then finalized by + # curie_audit: blank rows dropped, non-CURIE cells failed loudly, and the + # pinned `biolink:NamedThing` category added for downstream edge derivation. + [[*self.encoding(x, col), (curie_audit, (col, self.store.stem, self.config.name))] for x, col in verbatim], # Raw ``column_`` columns are dead weight once the encodings above have # copied them into named slots; trim before resolution so the frame that # resolve_batch materializes and joins stays narrow. @@ -1558,6 +1628,7 @@ def collect(self: Self, db: Path) -> list[tuple[Callable, tuple[Any]]] | Path: fullmap_audit: "qc", column: "encode", qualifier_vocabulary_audit: "encode", + curie_audit: "encode", edge_category: "edge", publications: "provenance", retrieval_sources: "provenance", diff --git a/src/tablassert/runcache.py b/src/tablassert/runcache.py index 45b98c0..40b45b8 100644 --- a/src/tablassert/runcache.py +++ b/src/tablassert/runcache.py @@ -272,6 +272,9 @@ def prefix_digest(ops: list[tuple[Callable, tuple[Any, ...]]]) -> str: lib.split_list: 1, # Vocabulary audit: one eager distinct-value pass over a single enum column. lib.qualifier_vocabulary_audit: 1, + # CURIE audit: one eager pass over a single no_map column (distinct-value check, + # blank-row filter, category column synthesis). + lib.curie_audit: 1, # Clean: coerce_columns applies five coercion rules (schema resolve + renames + value # coercion) in one pass, so it outprices a single-expression clean op. lib.coerce_columns: 3, diff --git a/tests/test_lib.py b/tests/test_lib.py index 5c74d04..61042d6 100644 --- a/tests/test_lib.py +++ b/tests/test_lib.py @@ -36,6 +36,7 @@ coerce_pvalue_columns, coerce_study_size_columns, coerced_target, + curie_audit, drop_low_number_of_cases, drop_not_significant, drop_zero_effect_size, @@ -621,6 +622,123 @@ def test_tcode_collect_audits_enum_ranged_qualifier_vocabulary(fixtures_path: Pa assert idx < trim_idx +def test_curie_audit_adds_named_thing_category_and_keeps_curies() -> None: + """curie_audit passes CURIE cells through verbatim and synthesizes the category column. + + Without the synthesized `_category`, edge_category would crash on a missing + column for every no_map node; `biolink:NamedThing` is the valid KGX category that + degrades edge derivation to `biolink:Association` like the documented no-role fallback. + """ + lf: pl.LazyFrame = pl.LazyFrame({"subject": ["CHEBI:1234", "HGNC:1100"]}) + + result: pl.DataFrame = curie_audit(lf, "subject", "sectionhash", "config.yaml").collect() + + assert result.get_column("subject").to_list() == ["CHEBI:1234", "HGNC:1100"] + assert result.get_column("subject_category").to_list() == ["biolink:NamedThing", "biolink:NamedThing"] + + +def test_curie_audit_drops_blank_and_null_like_rows() -> None: + """Blank/null-like cells in a no_map column drop the row, like an unresolved subject/object. + + A missing CURIE can never form a node, so the edge is dropped -- never emitted with a + garbage id. Null-like text ("NA", "none") is treated as blank, matching strip_nulls. + """ + lf: pl.LazyFrame = pl.LazyFrame({"subject": ["CHEBI:1234", None, "", "NA"], "row": [1, 2, 3, 4]}) + + result: pl.DataFrame = curie_audit(lf, "subject", "sectionhash", "config.yaml").collect() + + assert result.get_column("row").to_list() == [1] + assert result.get_column("subject").to_list() == ["CHEBI:1234"] + + +def test_curie_audit_rejects_non_curie_values() -> None: + """Non-blank cells that are not CURIE-shaped fail the build with a coded error. + + The encoded value becomes the KGX node id verbatim; shipping free text as an + identifier would fail validation far from the cause, so curie_audit raises where + the qualifier vocabulary audit does. + """ + lf: pl.LazyFrame = pl.LazyFrame({"subject": ["CHEBI:1234", "BRCA1"]}) + + with pytest.raises(TablassertError) as exc_info: + curie_audit(lf, "subject", "sectionhash", "config.yaml") + assert "no-map-curie-violation" in str(exc_info.value) + assert "BRCA1" in str(exc_info.value) + + +def test_tcode_collect_routes_no_map_nodes_around_resolve_batch(fixtures_path: Path) -> None: + """no_map subject is excluded from resolve_batch specs and finalized by curie_audit instead. + + The fullmap is never queried for the no_map column (it is absent from the batch + specs), while the mapped object keeps its ResolveSpec. + """ + data: Any = from_yaml(fixtures_path / "minimal_section.yaml") + store: Path = Path("/tmp/sectionhash_no_map.parquet") + data["statement"]["subject"] = {"method": "column", "encoding": "A", "no_map": True} + + tcode_model: Tcode = Tcode.model_validate( # pyright: ignore + {**data, "config": fixtures_path / "minimal_section.yaml", "store": store} + ) + + collected: list[tuple[Any, tuple[Any]]] = tcode_model.collect(Path("/tmp/fullmap.redb")) # pyright: ignore + specs: list[ResolveSpec] = next(op[1][0] for op in collected if op[0].__name__ == "resolve_batch") + spec_cols: list[str] = [spec.col for spec in specs] + audits: list[tuple[Any, ...]] = [op[1] for op in collected if op[0].__name__ == "curie_audit"] + + assert "subject" not in spec_cols + assert "object" in spec_cols + assert audits == [("subject", "sectionhash_no_map", "minimal_section.yaml")] + + +def test_tcode_collect_no_map_on_both_nodes_empties_resolve_batch(fixtures_path: Path) -> None: + """With subject and object both no_map, resolve_batch receives zero specs and early-returns. + + This is the proof that the fullmap redb is never touched for an all-verbatim + section: resolve_batch returns the frame untouched before any redb access when + `not specs`. + """ + data: Any = from_yaml(fixtures_path / "minimal_section.yaml") + store: Path = Path("/tmp/sectionhash_all_no_map.parquet") + data["statement"]["subject"] = {"method": "column", "encoding": "A", "no_map": True} + data["statement"]["object"] = {"method": "column", "encoding": "B", "no_map": True} + + tcode_model: Tcode = Tcode.model_validate( # pyright: ignore + {**data, "config": fixtures_path / "minimal_section.yaml", "store": store} + ) + + collected: list[tuple[Any, tuple[Any]]] = tcode_model.collect(Path("/tmp/does-not-exist.redb")) # pyright: ignore + specs: list[ResolveSpec] = next(op[1][0] for op in collected if op[0].__name__ == "resolve_batch") + audits: list[tuple[Any, ...]] = [op[1] for op in collected if op[0].__name__ == "curie_audit"] + + assert specs == [] + assert [args[0] for args in audits] == ["subject", "object"] + + +def test_tcode_collect_no_map_curie_ranged_qualifier_is_verbatim(fixtures_path: Path) -> None: + """A CURIE-ranged qualifier with no_map skips resolution like an enum-ranged literal does. + + It must land on curie_audit (not qualifier_vocabulary_audit, which would reject every + CURIE against the empty vocabulary, and not resolve_batch). + """ + data: Any = from_yaml(fixtures_path / "minimal_section.yaml") + store: Path = Path("/tmp/sectionhash_no_map_qual.parquet") + data["statement"]["object"] = {"method": "column", "encoding": "B", "no_map": True} + data["statement"]["qualifiers"] = [{"qualifier": "anatomical_context_qualifier", "method": "column", "encoding": "C", "no_map": True}] + + tcode_model: Tcode = Tcode.model_validate( # pyright: ignore + {**data, "config": fixtures_path / "minimal_section.yaml", "store": store} + ) + + collected: list[tuple[Any, tuple[Any]]] = tcode_model.collect(Path("/tmp/fullmap.redb")) # pyright: ignore + names: list[str] = [op[0].__name__ for op in collected] + specs: list[ResolveSpec] = next(op[1][0] for op in collected if op[0].__name__ == "resolve_batch") + audits: list[tuple[Any, ...]] = [op[1] for op in collected if op[0].__name__ == "curie_audit"] + + assert "anatomical_context_qualifier" not in [spec.col for spec in specs] + assert "qualifier_vocabulary_audit" not in names + assert [args[0] for args in audits] == ["object", "anatomical_context_qualifier"] + + def test_tcode_collect_edge_ops_follow_resolve_batch(fixtures_path: Path) -> None: """tcode collect runs predicate/edge_category after the single resolve_batch op.""" data: Any = from_yaml(fixtures_path / "minimal_section.yaml") From c4445b7356e63d51631650c73a5cfae176b35773 Mon Sep 17 00:00:00 2001 From: SkyeAv Date: Thu, 15 Oct 2026 13:12:57 -0700 Subject: [PATCH 3/5] feat: [US-003] e2e: build-kg with no_map statement emits verbatim CURIE nodes Add offline build_pipeline smokes for the no_map path: - verbatim CUSTOM: CURIEs absent from the redb appear in nodes and edges, proving no lookup happened; blank-subject row dropped, never emitted - emitted nodes carry the synthesized biolink:NamedThing category and edges reference the emitted node ids - negative smoke: a non-CURIE cell fails the build with no-map-curie-violation Also emit the _pre_resolution snapshot for no_map columns: graph compile discovers node columns through that snapshot, so without it a verbatim subject/object never split out of the edges into nodes and the RIG audit rejected the empty category lists. --- src/tablassert/lib.py | 18 ++++--- tests/test_e2e_smoke.py | 104 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 116 insertions(+), 6 deletions(-) diff --git a/src/tablassert/lib.py b/src/tablassert/lib.py index 6063ab2..e9c6f09 100644 --- a/src/tablassert/lib.py +++ b/src/tablassert/lib.py @@ -1465,12 +1465,18 @@ def _node_ops(self: Self, db: Path) -> list[Any]: # ranged column ends with a vocabulary audit: nothing else validates its cell # values against the closed Biolink range before KGX emission. [[*self.encoding(x, x.qualifier), (qualifier_vocabulary_audit, (str(x.qualifier), sorted(x.vocabulary or ())))] for x in literals], - # no_map columns are encoded exactly like enum-ranged literals (no - # pre-resolution snapshot, no NLP normalization -- both exist only to feed - # entity resolution these columns never undergo), then finalized by - # curie_audit: blank rows dropped, non-CURIE cells failed loudly, and the - # pinned `biolink:NamedThing` category added for downstream edge derivation. - [[*self.encoding(x, col), (curie_audit, (col, self.store.stem, self.config.name))] for x, col in verbatim], + # no_map columns are encoded exactly like enum-ranged literals (no NLP + # normalization -- it exists only to feed entity resolution these columns + # never undergo), then finalized by curie_audit: blank rows dropped, + # non-CURIE cells failed loudly, and the pinned `biolink:NamedThing` + # category added for downstream edge derivation. The `_pre_resolution` + # snapshot rides along because `_collect_subframes` discovers node columns + # through it at graph-compile time; without it a verbatim subject/object + # would never be split out of the edges into nodes. + [ + [*self.encoding(x, col), (column, (f"{col}_pre_resolution", col)), (curie_audit, (col, self.store.stem, self.config.name))] + for x, col in verbatim + ], # Raw ``column_`` columns are dead weight once the encodings above have # copied them into named slots; trim before resolution so the frame that # resolve_batch materializes and joins stays narrow. diff --git a/tests/test_e2e_smoke.py b/tests/test_e2e_smoke.py index 572d84c..4a1f5dd 100644 --- a/tests/test_e2e_smoke.py +++ b/tests/test_e2e_smoke.py @@ -113,6 +113,110 @@ def test_build_pipeline_against_real_redb(tmp_path: Path, monkeypatch: pytest.Mo assert "HGNC:6871" in edge_text +def test_build_pipeline_with_no_map_nodes_passes_curies_through_verbatim(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, rig_factory: Any) -> None: + """SMOKE: ``no_map: true`` subject/object build KGX from verbatim CURIEs, no lookup. + + The source cells carry ``CUSTOM:...`` CURIEs that the tiny real redb cannot resolve + (it only knows ``brca1``/``mapk1``), so their presence in the output proves the + pipeline used the values as-is. The blank subject row must vanish: a missing CURIE + cannot form a node, exactly like an unresolved term. Nodes carry the synthesized + ``biolink:NamedThing`` category so consumers still see a valid KGX category. + """ + monkeypatch.chdir(tmp_path) + (tmp_path / ".tablassert" / "store").mkdir(parents=True) + + fullmap: Path = _build_real_redb(tmp_path / "fullmap") + + # Two data rows: the second has a blank subject and must be dropped, not emitted. + data: Path = tmp_path / "data.tsv" + data.write_text("CUSTOM:1001\tCUSTOM:2001\n\tCUSTOM:2002\n") + + table: Path = tmp_path / "table.yaml" + table_config: dict[str, Any] = { + "template": { + "source": {"kind": "text", "local": str(data), "url": ["https://example.com/data.tsv"], "delimiter": "\t"}, + "statement": { + "subject": {"method": "column", "encoding": "A", "no_map": True}, + "predicate": "associated_with", + "object": {"method": "column", "encoding": "B", "no_map": True}, + }, + "provenance": {"repo": "PMC", "publication": "PMC0000000"}, + } + } + to_yaml(table, table_config) + + graph: Path = tmp_path / "graph.yaml" + graph_config: dict[str, Any] = { + "name": "NOMAP_KG", + "version": "1.0.0", + "tables": [str(table)], + "fullmap": str(fullmap), + "rig": rig_factory(tmp_path, infores_id="infores:nomap-kg", source_info={"description": "no_map e2e graph"}), + } + to_yaml(graph, graph_config) + + build_pipeline(graph, PipelineProgress(total_stages=6)) + + nodes: Path = tmp_path / "NOMAP_KG_1.0.0.nodes.ndjson" + edges: Path = tmp_path / "NOMAP_KG_1.0.0.edges.ndjson" + node_text: str = nodes.read_text() + edge_text: str = edges.read_text() + + # Verbatim CURIEs absent from the redb: only a true pass-through can produce them. + assert "CUSTOM:1001" in node_text + assert "CUSTOM:2001" in node_text + assert "CUSTOM:1001" in edge_text + assert "CUSTOM:2001" in edge_text + # Synthesized category keeps the nodes valid KGX without a fullmap role lookup. + assert "biolink:NamedThing" in node_text + # The blank-subject row was dropped, never emitted with an empty id. + assert "CUSTOM:2002" not in edge_text + assert "CUSTOM:2002" not in node_text + + +def test_build_pipeline_with_no_map_rejects_non_curie_cells(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, rig_factory: Any) -> None: + """SMOKE (negative): a non-CURIE cell in a no_map column fails the build loudly. + + The encoded value would become the KGX node id verbatim; free text must not ship + as an identifier, so the build raises the coded no-map-curie-violation instead of + emitting KGX that fails validation far from the cause. + """ + monkeypatch.chdir(tmp_path) + (tmp_path / ".tablassert" / "store").mkdir(parents=True) + + fullmap: Path = _build_real_redb(tmp_path / "fullmap") + + data: Path = tmp_path / "data.tsv" + data.write_text("not-a-curie\tCUSTOM:2001\n") + + table: Path = tmp_path / "table.yaml" + table_config: dict[str, Any] = { + "template": { + "source": {"kind": "text", "local": str(data), "url": ["https://example.com/data.tsv"], "delimiter": "\t"}, + "statement": { + "subject": {"method": "column", "encoding": "A", "no_map": True}, + "predicate": "associated_with", + "object": {"method": "column", "encoding": "B", "no_map": True}, + }, + "provenance": {"repo": "PMC", "publication": "PMC0000000"}, + } + } + to_yaml(table, table_config) + + graph: Path = tmp_path / "graph.yaml" + graph_config: dict[str, Any] = { + "name": "NOMAPBAD_KG", + "version": "1.0.0", + "tables": [str(table)], + "fullmap": str(fullmap), + "rig": rig_factory(tmp_path, infores_id="infores:nomapbad-kg", source_info={"description": "no_map negative e2e graph"}), + } + to_yaml(graph, graph_config) + + with pytest.raises(Exception, match="no-map-curie-violation"): + build_pipeline(graph, PipelineProgress(total_stages=6)) + + def test_build_pipeline_with_linkouts_resolves_edge_ids(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, rig_factory: Any) -> None: """SMOKE: ``build_pipeline(linkouts=True)`` emits the resolved EdgeLinkouts record URL. From 5985cf91e644af248279d0de885d8bed5cbf191d Mon Sep 17 00:00:00 2001 From: SkyeAv Date: Thu, 15 Oct 2026 13:13:59 -0700 Subject: [PATCH 4/5] docs: [US-004] document no_map in the node-encoding reference Add the no_map row to the NodeEncoding field table (keeps the docs source-of-truth binding green) and a No-map mode subsection covering the verbatim pass-through, blank-drop/non-CURIE-fail behavior, prefix/suffix CURIE formation, and the rejected resolution-filter combinations. --- docs/configuration/table.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/docs/configuration/table.md b/docs/configuration/table.md index 76e5b2a..011f8c1 100644 --- a/docs/configuration/table.md +++ b/docs/configuration/table.md @@ -227,6 +227,7 @@ Defines how to extract and resolve entities. |-------|------|----------|-------------| | `method` | String | No | `"value"` (literal) or `"column"` (source column letters). Defaults to `"value"`. | | `encoding` | String\|Int\|Float | Yes | Literal value or source column letters, depending on `method` | +| `no_map` | Bool | No | Skip fullmap entity resolution: the encoded value is used verbatim as the node CURIE, for data that already carries its own CURIEs. No NLP normalization, taxon constraint, or category lookup is applied; cells that are blank or not CURIE-shaped are dropped like unresolved nodes. Combines with nothing resolution-only (`prioritize`, `avoid`, `exclude_prefixes`, `exclude_regex`, non-default `taxon` are rejected as dead config). See [No-map mode](#no-map-mode). | | `taxon` | PositiveInt | No | NCBI Taxon ID for filtering (e.g., `9606` for human) | | `prioritize` | List[String] | No | Preferred Biolink categories (must be valid `Categories` enum values such as `Gene`, `Protein`) | | `avoid` | List[String] | No | Excluded Biolink categories (must be valid `Categories` enum values). Setting `avoid` makes the column an allow-list by complement: categories the `Categories` enum cannot name (fullmap-emitted Biolink mixins; see `biolink.CATEGORY_OVERRIDES`) are dropped as well, so the guard has no silent hole | @@ -258,6 +259,29 @@ subject: encoding: A # Read from column A ``` +#### No-map mode + +**`no_map: true`** - Use the data's own CURIEs as-is + +Most sources name entities in plain text and need entity resolution against the fullmap to turn a +mention like `BRCA1` into a CURIE. When the table already carries CURIEs (your own identifiers or +an upstream resource's), `no_map: true` skips that lookup entirely: the encoded value becomes the +node id verbatim, no NLP normalization or taxon/category filtering is applied, and the node is +emitted with the generic `biolink:NamedThing` category (so its edges derive `biolink:Association`). + +```yaml +subject: + method: column + encoding: A + no_map: true # Column A already holds CURIEs like "CHEBI:41774" +``` + +Blank cells are dropped like unresolved nodes (an edge needs a node id), and a non-blank cell that +is not CURIE-shaped fails the build with `no-map-curie-violation`. `prefix`/`suffix` still apply, +so raw identifiers without a namespace can form CURIEs (`prefix: "CHEBI:"`). Resolution filters +(`taxon`, `prioritize`, `avoid`, `exclude_prefixes`, `exclude_regex`) are rejected alongside +`no_map` as dead config: with no lookup, nothing reads them. + At runtime those letters are converted internally to Polars column names such as `column_1`, but those internal names are not valid configuration values. #### `split_by` From 1c363fbe554761fc631f962db607791455453d3c Mon Sep 17 00:00:00 2001 From: SkyeAv Date: Thu, 15 Oct 2026 13:17:02 -0700 Subject: [PATCH 5/5] fix: align no_map descriptions with the fail-loud audit and audit distinct values only The field description and docs table claimed non-CURIE cells are dropped; the shipped curie_audit raises no-map-curie-violation on them (mirroring qualifier_vocabulary_audit), so both now state: blanks drop, non-CURIE cells fail the build. Also judge CURIE shape on distinct non-blank values so the per-element check does not scale with row count. --- docs/configuration/table.md | 2 +- src/tablassert/lib.py | 6 +++++- src/tablassert/models.py | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/configuration/table.md b/docs/configuration/table.md index 011f8c1..e3342d0 100644 --- a/docs/configuration/table.md +++ b/docs/configuration/table.md @@ -227,7 +227,7 @@ Defines how to extract and resolve entities. |-------|------|----------|-------------| | `method` | String | No | `"value"` (literal) or `"column"` (source column letters). Defaults to `"value"`. | | `encoding` | String\|Int\|Float | Yes | Literal value or source column letters, depending on `method` | -| `no_map` | Bool | No | Skip fullmap entity resolution: the encoded value is used verbatim as the node CURIE, for data that already carries its own CURIEs. No NLP normalization, taxon constraint, or category lookup is applied; cells that are blank or not CURIE-shaped are dropped like unresolved nodes. Combines with nothing resolution-only (`prioritize`, `avoid`, `exclude_prefixes`, `exclude_regex`, non-default `taxon` are rejected as dead config). See [No-map mode](#no-map-mode). | +| `no_map` | Bool | No | Skip fullmap entity resolution: the encoded value is used verbatim as the node CURIE, for data that already carries its own CURIEs. No NLP normalization, taxon constraint, or category lookup is applied; blank cells are dropped like unresolved nodes and a non-blank non-CURIE cell fails the build. Combines with nothing resolution-only (`prioritize`, `avoid`, `exclude_prefixes`, `exclude_regex`, non-default `taxon` are rejected as dead config). See [No-map mode](#no-map-mode). | | `taxon` | PositiveInt | No | NCBI Taxon ID for filtering (e.g., `9606` for human) | | `prioritize` | List[String] | No | Preferred Biolink categories (must be valid `Categories` enum values such as `Gene`, `Protein`) | | `avoid` | List[String] | No | Excluded Biolink categories (must be valid `Categories` enum values). Setting `avoid` makes the column an allow-list by complement: categories the `Categories` enum cannot name (fullmap-emitted Biolink mixins; see `biolink.CATEGORY_OVERRIDES`) are dropped as well, so the guard has no silent hole | diff --git a/src/tablassert/lib.py b/src/tablassert/lib.py index e9c6f09..8ea7200 100644 --- a/src/tablassert/lib.py +++ b/src/tablassert/lib.py @@ -1246,7 +1246,11 @@ def curie_audit(lf: pl.LazyFrame, col: str, section: str, config_file: str) -> p if values.dtype.base_type() in (pl.List, pl.Array): values = values.explode() blank_mask: pl.Series = values.is_null() | values.str.strip_chars().str.to_lowercase().is_in(list(_NULL_LIKE_TEXT)) - offending: pl.Series = values.filter(~blank_mask & ~values.str.strip_chars().map_elements(is_curie, return_dtype=pl.Boolean)).unique().sort() + # Judge CURIE shape on the distinct non-blank values only: the audit runs once per + # section, and map_elements is a per-element Python call that must not scale with + # row count when one bad cell repeats a million times. + distinct: pl.Series = values.filter(~blank_mask).unique() + offending: pl.Series = distinct.filter(~distinct.str.strip_chars().map_elements(is_curie, return_dtype=pl.Boolean)).sort() if offending.len() > 0: shown: list[str] = offending.head(10).to_list() more: str = "..." if offending.len() > len(shown) else "" diff --git a/src/tablassert/models.py b/src/tablassert/models.py index 81689d8..0cfc672 100644 --- a/src/tablassert/models.py +++ b/src/tablassert/models.py @@ -327,7 +327,7 @@ class NodeEncoding(Encoding): description=( "When True, skip fullmap entity resolution: the encoded value is used verbatim as the node CURIE, " "for data that already carries its own CURIEs. No NLP normalization, taxon constraint, or category " - "lookup is applied, and cells that are blank or not CURIE-shaped are dropped like unresolved nodes." + "lookup is applied; blank cells are dropped like unresolved nodes and non-CURIE cells fail the build." ), examples=[True], )