From 425b2777a238d01d32f82f55fcb3b8d2262e414b Mon Sep 17 00:00:00 2001 From: SkyeAv Date: Thu, 15 Oct 2026 13:06:26 -0700 Subject: [PATCH 1/6] feat!: re-home derived-path source record URLs onto the upstream supporting entry Sections without a provenance.override now emit their source record URLs on the supporting entry of the publication repository (infores:pubmed-central for PMC sections) instead of the primary graph-infores entry. The primary knowledge source is the transforming resource, not the downloadable record; the --linkouts per-edge template stays on the primary entry and still resolves via the post-dedup sweep. Override sections are byte-identical: upstream_source_record_urls mappings re-home as specified, other overrides keep the legacy primary-entry emission. --- src/tablassert/lib.py | 56 ++++++++++++++++++++++--------------- tests/test_e2e_smoke.py | 8 ++++-- tests/test_lib.py | 61 ++++++++++++++++++++++++++++++++--------- 3 files changed, 88 insertions(+), 37 deletions(-) diff --git a/src/tablassert/lib.py b/src/tablassert/lib.py index bae8cb2..2d26fbc 100644 --- a/src/tablassert/lib.py +++ b/src/tablassert/lib.py @@ -608,11 +608,14 @@ def retrieval_sources( Mirrors ``build_association_knowledge_sources()`` from ``translator-ingests/util/biolink.py``: the primary knowledge source lists the upstream resources, and each upstream resource additionally appears as its own - ``supporting_data_source`` entry. By default the primary entry carries the - section's source record URLs; when ``upstream_urls`` is given, URL placement is - fully determined by that mapping instead -- each listed upstream entry carries - its own ``source_record_urls`` and the primary entry emits none (the primary is - the transforming resource, not a downloadable record). + ``supporting_data_source`` entry. URL placement is determined by ``upstream_urls`` + whenever it is given (the derived build path passes it unconditionally): each + listed upstream entry carries its own ``source_record_urls``, because the section's + record URLs describe the upstream repository's downloadable record, not the + transforming primary. The primary entry still emits ``urls`` that are not source + records -- the per-edge EdgeLinkouts template when ``--linkouts`` is set. Without + ``upstream_urls`` (override sections that set no explicit mapping), the primary + entry carries ``urls`` verbatim and the supporting entries stay bare. When ``explicit`` is given, exactly those entries are emitted, in order, and ``primary``/``upstream``/``urls``/``upstream_urls`` are ignored. Each entry @@ -639,7 +642,7 @@ def retrieval_sources( for entry in explicit ] elif upstream_urls is not None: - entries = [_retrieval_source(primary, "primary_knowledge_source", upstream)] + entries = [_retrieval_source(primary, "primary_knowledge_source", upstream, urls)] entries.extend(_retrieval_source(x, "supporting_data_source", urls=upstream_urls.get(x)) for x in upstream) else: entries = [_retrieval_source(primary, "primary_knowledge_source", upstream, urls)] @@ -1454,19 +1457,28 @@ def _provenance_ops(self: Self) -> list[Any]: # (rig.source_info.infores_id); there is no implicit derivation from the graph # name, so a RIG and its edges can never disagree about the source identity. primary_knowledge_source: str | None = self.infores - # Per-edge record URLs for the primary entry: the section's declared source URLs, - # plus the EdgeLinkouts template when --linkouts is set. A section with ANY - # provenance.override is author-controlled (DAKP hand-writes the same template - # into override.sources), so the flag injects nothing there. - primary_urls: list[str] = [str(u) for u in self.source.url] - if self.linkouts and override is None: - primary_urls.append(EDGE_LINKOUT_URL_TEMPLATE) + # Record URLs: the section's declared source URLs. On the derived path they + # describe the UPSTREAM record the data was extracted from, so they are emitted + # on the supporting entry of the publication repository (infores:pubmed-central + # for PMC sections) and the primary entry -- the transforming resource -- + # carries only the EdgeLinkouts template when --linkouts is set. A section with + # ANY provenance.override is author-controlled (DAKP hand-writes the same + # template into override.sources): an upstream_source_record_urls mapping + # re-homes URLs exactly as specified (primary gets nothing), any other override + # keeps the legacy primary-entry emission, and the flag injects nothing there. + record_urls: list[str] = [str(u) for u in self.source.url] upstream_ids = override.upstream_resource_ids if override else upstream_resource_ids(self.provenance.repo) - upstream_urls: dict[str, list[str]] | None = ( - {key: [str(u) for u in urls] for key, urls in override.upstream_source_record_urls.items()} - if override and override.upstream_source_record_urls is not None - else None - ) + primary_urls: list[str] + upstream_urls: dict[str, list[str]] | None + if override is None: + upstream_urls = {upstream_ids[0]: record_urls} + primary_urls = [EDGE_LINKOUT_URL_TEMPLATE] if self.linkouts else [] + elif override.upstream_source_record_urls is not None: + upstream_urls = {key: [str(u) for u in urls] for key, urls in override.upstream_source_record_urls.items()} + primary_urls = [] + else: + upstream_urls = None + primary_urls = record_urls # An explicit `sources` template replaces the derived primary/upstream # emission entirely; the model already forbids combining it with # `upstream_resource_ids`/`upstream_source_record_urls`. @@ -1500,10 +1512,10 @@ def _provenance_ops(self: Self) -> list[Any]: (value, ("agent_type", agent_type)), # Retrieval provenance lives only in the nested `sources` list (Biolink # RetrievalSource); current translator-ingests emits no flat - # `primary_knowledge_source` scalar, so neither do we. A per-upstream URL - # mapping (override.upstream_source_record_urls) re-homes the record URLs - # from the primary entry onto the matching supporting entries; an explicit - # `sources` template (override.sources) replaces the whole derivation. + # `primary_knowledge_source` scalar, so neither do we. The derived path maps + # the section's record URLs onto the upstream repository's supporting entry + # (see above); an explicit `sources` template (override.sources) replaces + # the whole derivation. (retrieval_sources, (primary_knowledge_source, upstream_ids, primary_urls, upstream_urls, explicit_sources)) if primary_knowledge_source else None, diff --git a/tests/test_e2e_smoke.py b/tests/test_e2e_smoke.py index 572d84c..476d212 100644 --- a/tests/test_e2e_smoke.py +++ b/tests/test_e2e_smoke.py @@ -164,8 +164,12 @@ def test_build_pipeline_with_linkouts_resolves_edge_ids(tmp_path: Path, monkeypa primary: dict[str, Any] = next(s for s in record["sources"] if s["resource_role"] == "primary_knowledge_source") assert primary["resource_id"] == "infores:linkouts-kg" urls: list[str] = primary["source_record_urls"] - # The section's own record URL survives and the linkout resolves to THIS edge's id. - assert urls == ["https://example.com/data.tsv", f"https://linkouts.skyelanegoetz.com/edges/{record['id']}"] + # The linkout resolves to THIS edge's id and stays on the primary entry; the + # section's own record URL moved to the PMC upstream supporting entry. + assert urls == [f"https://linkouts.skyelanegoetz.com/edges/{record['id']}"] + supporting: dict[str, Any] = next(s for s in record["sources"] if s["resource_role"] == "supporting_data_source") + assert supporting["resource_id"] == "infores:pubmed-central" + assert supporting["source_record_urls"] == ["https://example.com/data.tsv"] # No unresolved placeholder may survive anywhere in the emitted file. assert "{edge_id}" not in edges.read_text() diff --git a/tests/test_lib.py b/tests/test_lib.py index 5c74d04..9aefbbf 100644 --- a/tests/test_lib.py +++ b/tests/test_lib.py @@ -363,12 +363,13 @@ def test_section_store_path_suffixes_every_build_mode() -> None: def test_tcode_collect_appends_edge_linkout_when_linkouts(fixtures_path: Path) -> None: - """--linkouts appends the EdgeLinkouts template to the primary entry's record URLs. + """--linkouts keeps the EdgeLinkouts template on the primary entry while record URLs move upstream. Why: the flag's whole contract is that the graph-infores primary ``sources`` entry - gains ``https://linkouts.skyelanegoetz.com/edges/{edge_id}`` AFTER the section's own - source URLs; the template must reach ``retrieval_sources`` unresolved so the - post-dedup sweep can substitute each edge's own id. + gains ``https://linkouts.skyelanegoetz.com/edges/{edge_id}``; the section's own + record URLs belong on the upstream repository's supporting entry. The template + must reach ``retrieval_sources`` unresolved so the post-dedup sweep can substitute + each edge's own id. """ data: Any = from_yaml(fixtures_path / "minimal_section.yaml") tcode_model: Tcode = Tcode.model_validate( # pyright: ignore @@ -386,14 +387,16 @@ def test_tcode_collect_appends_edge_linkout_when_linkouts(fixtures_path: Path) - assert len(linkout_ops) == 1 urls: list[str] = linkout_ops[0][1][2] - assert urls == ["https://example.com/test.tsv", "https://linkouts.skyelanegoetz.com/edges/{edge_id}"] + assert urls == ["https://linkouts.skyelanegoetz.com/edges/{edge_id}"] + # The section's own record URLs ride on the PMC upstream supporting entry (arg 4). + assert linkout_ops[0][1][3] == {"infores:pubmed-central": ["https://example.com/test.tsv"]} def test_tcode_collect_omits_edge_linkout_by_default(fixtures_path: Path) -> None: - """Without --linkouts the primary entry's record URLs are exactly the section's source URLs. + """Without --linkouts the primary entry carries no record URLs and no linkout template. - Why: the flag is opt-in; default-off emission must be byte-identical to a build - without the flag, so the op list must carry no linkout template whatsoever. + Why: the flag is opt-in; default-off emission must carry no linkout template + whatsoever, and the section's record URLs ride on the upstream supporting entry. """ data: Any = from_yaml(fixtures_path / "minimal_section.yaml") tcode_model: Tcode = Tcode.model_validate( # pyright: ignore @@ -401,9 +404,10 @@ def test_tcode_collect_omits_edge_linkout_by_default(fixtures_path: Path) -> Non ) collected: list[tuple[Any, tuple[Any, ...]]] = tcode_model.collect(Path("/tmp/fullmap.redb")) # pyright: ignore - urls: list[str] = next(op for op in collected if op[0].__name__ == "retrieval_sources")[1][2] + retrieval_op: tuple[Any, tuple[Any, ...]] = next(op for op in collected if op[0].__name__ == "retrieval_sources") - assert urls == ["https://example.com/test.tsv"] + assert retrieval_op[1][2] == [] + assert retrieval_op[1][3] == {"infores:pubmed-central": ["https://example.com/test.tsv"]} def test_tcode_collect_omits_edge_linkout_when_override(fixtures_path: Path) -> None: @@ -920,7 +924,7 @@ def test_tcode_collect_uses_graph_infores_when_no_section_override(fixtures_path def test_tcode_collect_nests_source_record_urls_in_sources(fixtures_path: Path) -> None: - """Source record URLs hang off the primary ``RetrievalSource``, not the edge.""" + """Source record URLs hang off the upstream ``RetrievalSource``, not the edge or the primary.""" data: Any = from_yaml(fixtures_path / "minimal_section.yaml") store: Path = Path("/tmp/sectionhash.parquet") tcode_model: Tcode = Tcode.model_validate( # pyright: ignore @@ -935,9 +939,40 @@ def test_tcode_collect_nests_source_record_urls_in_sources(fixtures_path: Path) assert len(source_ops) == 1 assert url_ops == [] assert "source_record_urls" not in result.columns - primary: dict[str, Any] = next(s for s in result["sources"].to_list()[0] if s["resource_role"] == "primary_knowledge_source") - assert primary["source_record_urls"] == ["https://example.com/test.tsv"] + sources: list[dict[str, Any]] = result["sources"].to_list()[0] + primary: dict[str, Any] = next(s for s in sources if s["resource_role"] == "primary_knowledge_source") + assert primary["source_record_urls"] is None assert "id" not in primary + supporting: dict[str, Any] = next(s for s in sources if s["resource_role"] == "supporting_data_source") + assert supporting["resource_id"] == "infores:pubmed-central" + assert supporting["source_record_urls"] == ["https://example.com/test.tsv"] + + +def test_tcode_collect_rehomes_pmc_record_urls_onto_pubmed_central(fixtures_path: Path) -> None: + """A PMC-hosted table URL lands on the infores:pubmed-central entry, not the graph-infores primary. + + Why: the primary knowledge source is the transforming resource (e.g. + infores:multiomics-kp); the downloadable record lives at the upstream publication + repository, so that supporting entry is where ``source_record_urls`` belongs. + """ + data: Any = from_yaml(fixtures_path / "minimal_section.yaml") + data["source"]["url"] = ["https://pmc-oa-opendata.s3.amazonaws.com/PMC10010437.1/Table3.xlsx"] + store: Path = Path("/tmp/sectionhash_pmc.parquet") + tcode_model: Tcode = Tcode.model_validate( # pyright: ignore + {**data, "config": fixtures_path / "minimal_section.yaml", "store": store, "name": "MULTIOMICS_KG", "infores": "infores:multiomics-kp"} + ) + + collected: list[tuple[Any, tuple[Any]]] = tcode_model.collect(Path("/tmp/fullmap.redb")) # pyright: ignore + source_ops: list[tuple[Any, tuple[Any]]] = [op for op in collected if op[0] is retrieval_sources] + result: pl.DataFrame = source_ops[0][0](pl.LazyFrame({"subject": ["A"]}), *source_ops[0][1]).collect() + + sources: list[dict[str, Any]] = result["sources"].to_list()[0] + primary: dict[str, Any] = next(s for s in sources if s["resource_role"] == "primary_knowledge_source") + assert primary["resource_id"] == "infores:multiomics-kp" + assert primary["source_record_urls"] is None + assert primary["upstream_resource_ids"] == ["infores:pubmed-central"] + pmc: dict[str, Any] = next(s for s in sources if s["resource_id"] == "infores:pubmed-central") + assert pmc["source_record_urls"] == ["https://pmc-oa-opendata.s3.amazonaws.com/PMC10010437.1/Table3.xlsx"] def test_tcode_collect_upstream_source_record_urls_rehome_urls(fixtures_path: Path) -> None: From 0adc52ed9a1110d8bc8c4f570b08cb6445a9d5a2 Mon Sep 17 00:00:00 2001 From: SkyeAv Date: Thu, 15 Oct 2026 13:12:19 -0700 Subject: [PATCH 2/6] feat!: name inlined Study after table config file and 1-based section index The has_supporting_studies Study name now reads " SECTION " (document order) instead of the worksheet or source data filename, so edge provenance points back to the exact configuration that produced it. ingests.to_sections stamps the 1-based section_index into each expanded section; Tcode carries it and _provenance_ops composes the name. Section-level direct validation is unaffected: the stamp is a Tcode-only key, popped alongside the existing config stamp. --- src/tablassert/ingests.py | 11 ++++++++--- src/tablassert/lib.py | 22 +++++++++++++--------- tests/test_e2e_smoke.py | 2 +- tests/test_lib.py | 30 +++++++++++++++++++++++++----- tests/test_models.py | 7 +++++-- 5 files changed, 52 insertions(+), 20 deletions(-) diff --git a/src/tablassert/ingests.py b/src/tablassert/ingests.py index 0ad5341..ce71402 100644 --- a/src/tablassert/ingests.py +++ b/src/tablassert/ingests.py @@ -82,12 +82,14 @@ def to_yaml(p: Path, data: object) -> None: yaml.dump(data, f, Dumper=_IndentedSafeDumper, sort_keys=False, default_flow_style=False, allow_unicode=True, width=120) -def to_sections(instructions: dict[str, Any], table: Path) -> list[list[dict[str, Any]]]: +def to_sections(instructions: dict[str, Any], table: Path) -> list[dict[str, Any]]: """Expand a table config dict into a list of section dicts. Each section is the deep merge of the table-level ``template`` over a per-section entry. The originating ``table`` path is stamped into each - section as ``config``. + section as ``config``, and the section's 1-based position in the file + (document order) as ``section_index`` so provenance can name the inlined + Study after its origin (" SECTION "). Args: instructions: Parsed YAML config with optional ``template`` and @@ -100,4 +102,7 @@ def to_sections(instructions: dict[str, Any], table: Path) -> list[list[dict[str template: dict[str, Any] = instructions.get("template", {}) template["config"] = table sections: list[dict[str, Any]] = instructions.get("sections", [{}]) - return [fastmerge(deepcopy(template), x) for x in sections] + merged: list[dict[str, Any]] = [fastmerge(deepcopy(template), x) for x in sections] + for index, section in enumerate(merged, start=1): + section["section_index"] = index + return merged diff --git a/src/tablassert/lib.py b/src/tablassert/lib.py index 2d26fbc..e88033a 100644 --- a/src/tablassert/lib.py +++ b/src/tablassert/lib.py @@ -471,8 +471,8 @@ def inline_supporting_study(lf: pl.LazyFrame, study_id: str, study_name: str | N The struct is shaped by the current Biolink Model (4.4.4+): * ``Study.id`` is a real identifier -- the section's publication CURIE -- and - ``Study.name`` a DISJOINT human label (worksheet or source filename); the two - are never composed into one string. + ``Study.name`` a DISJOINT human label (the table config file plus the section's + 1-based index); the two are never composed into one string. * Study-level metadata declared as annotations (``study_size``, ``study_cohort``, ``study_context``, ``study_date_range``, ``study_method_description``, ``study_method_types`` -- see :data:`biolink.STUDY_METADATA_FIELDS`) lands as @@ -504,8 +504,8 @@ def inline_supporting_study(lf: pl.LazyFrame, study_id: str, study_name: str | N lf: Edges LazyFrame after annotation and provenance ops. study_id: Stable study identifier (publication CURIE, or the config stem when unidentified). - study_name: Human-readable study label (worksheet or source filename), or - ``None`` to omit it. + study_name: Human-readable study label (table config file plus 1-based section + index), or ``None`` to omit it. identified: Whether ``study_id`` names a real publication rather than falling back to the config stem. ``False`` lets a contentless study be skipped. @@ -1253,6 +1253,10 @@ class Tcode(Section): name: str | None = Field(None) infores: str | None = Field(None) linkouts: bool = Field(False) + section_index: int = Field( + 1, + description="1-based position of this section within its table config file (document order); stamped by ingests.to_sections and used to name the inlined Study.", + ) qc_similarity_threshold: float = Field(0.5) qc_fuzzy_ratio_threshold: float = Field(70.0) qc_fuzzy_partial_threshold: float = Field(80.0) @@ -1488,17 +1492,17 @@ def _provenance_ops(self: Self) -> list[Any]: knowledge_level = override.knowledge_level if override else self.provenance.knowledge_level agent_type = override.agent_type if override else self.provenance.agent_type publication_values = override.publications if override else [publication_curie(self.provenance.repo, self.provenance.publication or "")] - # The study is the table itself: one publication, one worksheet/table. The - # publication CURIE is the study's IDENTIFIER; the worksheet or source filename - # is its human-readable NAME. The two stay disjoint fields and are never composed + # The study is the table section itself: one publication, one config section. + # The publication CURIE is the study's IDENTIFIER; the originating table config + # file plus the section's 1-based index is its human-readable NAME (e.g. + # "KANG12.yaml SECTION 2"). The two stay disjoint fields and are never composed # into one string, so neither ever duplicates the other. - sheet: str | None = self.source.sheet if self.source.kind == Files.EXCEL else None # pyright: ignore # No publication means no study to name: the fallback id is the config stem (the # YAML filename without its extension), not an identifier, so # `inline_supporting_study` skips the struct unless it has content to carry. identified: bool = bool(publication_values) study_id: str = publication_values[0] if publication_values else self.config.stem - study_name: str | None = sheet if sheet is not None else Path(self.source.local).name + study_name: str | None = f"{self.config.name} SECTION {self.section_index}" if study_name == study_id: study_name = None # `use_enum_values` stores the override's enum keys/values as plain strings. diff --git a/tests/test_e2e_smoke.py b/tests/test_e2e_smoke.py index 476d212..b73fef7 100644 --- a/tests/test_e2e_smoke.py +++ b/tests/test_e2e_smoke.py @@ -403,7 +403,7 @@ def test_build_pipeline_coerces_statistical_annotations(tmp_path: Path, monkeypa assert "sample_size" not in edge_text study: dict[str, Any] = edge["has_supporting_studies"]["PMCID:PMC0000000"] assert study["id"] == "PMCID:PMC0000000" - assert study["name"] == "data.tsv" + assert study["name"] == "table.yaml SECTION 1" assert study["study_size"] == 450 assert study["has_study_results"] == [{"id": "row:1"}] diff --git a/tests/test_lib.py b/tests/test_lib.py index 9aefbbf..a25879d 100644 --- a/tests/test_lib.py +++ b/tests/test_lib.py @@ -26,7 +26,7 @@ from tablassert.enums import Repositories from tablassert.errors import TablassertError from tablassert.fullmap import ResolveSpec -from tablassert.ingests import from_yaml +from tablassert.ingests import from_yaml, to_sections from tablassert.lib import ( HEAD_ROWS, Tcode, @@ -410,6 +410,26 @@ def test_tcode_collect_omits_edge_linkout_by_default(fixtures_path: Path) -> Non assert retrieval_op[1][3] == {"infores:pubmed-central": ["https://example.com/test.tsv"]} +def test_tcode_collect_names_study_after_config_file_and_section_index(fixtures_path: Path) -> None: + """The inlined Study name identifies the table config file and the 1-based section index. + + Why: a source data filename like data.parquet says nothing about which configuration + produced the edge; "KANG12.yaml SECTION 2" pinpoints exactly the section, which is + what a reader of has_supporting_studies needs to find the configuration back. + """ + raw: Any = from_yaml(fixtures_path / "minimal_section_with_sections.yaml") + sections: list[dict[str, Any]] = to_sections(raw, fixtures_path / "KANG12.yaml") # pyright: ignore + assert len(sections) == 2 + second: Tcode = Tcode.model_validate( # pyright: ignore + {**sections[1], "store": Path("/tmp/sectionhash_kang12.parquet"), "infores": "infores:test-kg"} + ) + + collected: list[tuple[Any, tuple[Any]]] = second.collect(Path("/tmp/fullmap.redb")) # pyright: ignore + study_op: tuple[Any, tuple[Any, ...]] = next(op for op in collected if op[0].__name__ == "inline_supporting_study") + + assert study_op[1][1] == "KANG12.yaml SECTION 2" + + def test_tcode_collect_omits_edge_linkout_when_override(fixtures_path: Path) -> None: """--linkouts never injects into a section that authors its own provenance.override. @@ -3422,7 +3442,7 @@ def test_compile_subgraph_e2e_column_cleanup_and_numeric_annotations(monkeypatch "tp53": [fake_fullmap_row("tp53", "HGNC:11998", "TP53", "Gene", 9606)], } install_fake_fullmap(monkeypatch, rows) - table_path, source_path = write_text_section( + table_path, _ = write_text_section( tmp_path, "column_cleanup", { @@ -3458,7 +3478,7 @@ def test_compile_subgraph_e2e_column_cleanup_and_numeric_annotations(monkeypatch # field on the inlined Study; the StudyResult only anchors the source row. study: dict[str, Any] = result["has_supporting_studies"]["PMID:12345"] assert study["id"] == "PMID:12345" - assert study["name"] == source_path.name + assert study["name"] == f"{table_path.name} SECTION 1" assert study["study_size"] == 1200 # The result only anchors the row; the parquet-level null description is stripped # by the NDJSON writer before emission. @@ -3479,7 +3499,7 @@ def test_compile_subgraph_e2e_release_drops_rows_before_fullmap_lookup(monkeypat "droppeddiseas": [fake_fullmap_row("droppeddiseas", "MONDO:2", "Dropped disease", "Disease", 0)], } calls: list[list[str]] = install_fake_fullmap(monkeypatch, rows) - table_path, source_path = write_text_section( + table_path, _ = write_text_section( tmp_path, "release_drop", { @@ -3505,7 +3525,7 @@ def test_compile_subgraph_e2e_release_drops_rows_before_fullmap_lookup(monkeypat # Published wrapper with no study metadata remains, but an empty rescue column # cannot justify a row-only StudyResult. study: dict[str, Any] = result["has_supporting_studies"].to_list()[0]["PMCID:PMC0000000"] - assert study["name"] == source_path.name + assert study["name"] == f"{table_path.name} SECTION 1" assert not study.get("has_study_results") assert "droppedgen" not in looked_up assert "droppeddiseas" not in looked_up diff --git a/tests/test_models.py b/tests/test_models.py index 2cde591..9b05f81 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -325,6 +325,7 @@ def test_merged_section_drops_a_template_level_unpaired_effect_type(fixtures_pat sections: list[dict[str, Any]] = to_sections(raw, fixtures_path / "minimal_section_with_sections.yaml") # pyright: ignore for merged in sections: merged.pop("config", None) + merged.pop("section_index", None) with pytest.warns(UnpairedEffectAnnotationWarning, match="Dropped unpaired `effect_type`"): section: Section = Section.model_validate(merged) # The merged list was [effect_type (template), p_value (section)]; only the unpaired @@ -1108,13 +1109,15 @@ def test_no_deprecation_warnings_for_current_fixtures(fixtures_path: Path, recwa # Single-section fixture validates directly. Section.model_validate(from_yaml(fixtures_path / "minimal_section.yaml")) - # Template/sections configs expand via to_sections; the stamped `config` path is a - # Tcode-only key, so drop it before validating the pure Section schema. + # Template/sections configs expand via to_sections; the stamped `config` path and + # `section_index` position are Tcode-only keys, so drop them before validating the + # pure Section schema. for path in (fixtures_path / "minimal_section_with_sections.yaml", EXAMPLES / "tutorial-table.yaml"): raw: Any = from_yaml(path) sections: list[dict[str, Any]] = to_sections(raw, path) # pyright: ignore for section in sections: section.pop("config", None) + section.pop("section_index", None) Section.model_validate(section) # Graph example validates directly. From c33e3ce6c91b913b823e03e25aea6c0866f34076 Mon Sep 17 00:00:00 2001 From: SkyeAv Date: Thu, 15 Oct 2026 13:14:33 -0700 Subject: [PATCH 3/6] docs: sync provenance placement and Study naming with the new build output BaseSource.url description, the table/graph configuration reference, and the advanced example now state that derived sections emit source record URLs on the upstream repository supporting entry (infores:pubmed-central for PMC sections), that --linkouts leaves only the per-edge template on the primary entry, and that the inlined Study name is the table config file plus the 1-based section index. --- docs/configuration/advanced-example.md | 2 +- docs/configuration/table.md | 10 +++++----- src/tablassert/models.py | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/configuration/advanced-example.md b/docs/configuration/advanced-example.md index a67b889..5bde5ff 100644 --- a/docs/configuration/advanced-example.md +++ b/docs/configuration/advanced-example.md @@ -103,7 +103,7 @@ whole table or `method: column` to pull per-row notes from the source (see top-level edge fields (`p_value` in controlled scientific-notation string form, `effect_size` as a real JSON number since biolink-model 4.4.4). Study metadata (`study_size`, here declared as a constant) lands on the edge's inlined supporting `Study`; the `Study.id` is the publication CURIE, `Study.name` -the worksheet, and the single `StudyResult` anchors the source row as its `row:` id. Any +the table config file and 1-based section index (e.g. `KANG12.yaml SECTION 2`), and the single `StudyResult` anchors the source row as its `row:` id. Any non-Biolink-slot name such as `assertion_method`, `multiple_testing_correction_method`, and `miscellaneous_notes` folds into the edge's `supporting_text` list as `"name: value"` entries (sorted alphabetically): diff --git a/docs/configuration/table.md b/docs/configuration/table.md index 76e5b2a..cc0f2f6 100644 --- a/docs/configuration/table.md +++ b/docs/configuration/table.md @@ -90,7 +90,7 @@ Defines the data file location and format. |-------|------|----------|-------------| | `kind` | String | No | Source kind. Model default is `"excel"`, but specify it explicitly in configs. | | `local` | Path | Yes | Local file path the source is read from. The file must already exist here; Tablassert does not download it. | -| `url` | List[URL] | Yes | One or more source URLs recorded as provenance (emitted as the primary `sources` entry's `source_record_urls` list and in the RIG; when `provenance.override.upstream_source_record_urls` is set, RIG only; the per-upstream mapping determines edge placement). At least one URL is required; supply multiple to back a single section with several links. Format-validated only; not fetched. | +| `url` | List[URL] | Yes | One or more source URLs recorded as provenance (derived sections emit them as the upstream repository supporting entry's `source_record_urls` list, infores:pubmed-central for PMC sections, and in the RIG; when `provenance.override.upstream_source_record_urls` is set, RIG only; the per-upstream mapping determines edge placement instead). At least one URL is required; supply multiple to back a single section with several links. Format-validated only; not fetched. | | `sheet` | String | No | Sheet name. Defaults to `"Sheet1"`. | | `row_slice` | List[PositiveInt\|"auto"] | No | Two-value zero-based crop bounds: `[start, stop]`. Each value may be a positive integer or `"auto"`. Mutually exclusive with `rows`. | | `rows` | List[PositiveInt] | No | Zero-based row indices to keep after any `row_slice` crop. Mutually exclusive with `row_slice`. | @@ -115,7 +115,7 @@ source: |-------|------|----------|-------------| | `kind` | String | No | Source kind. Model default is `"text"`, but specify it explicitly in configs. | | `local` | Path | Yes | Local file path the source is read from. The file must already exist here; Tablassert does not download it. | -| `url` | List[URL] | Yes | One or more source URLs recorded as provenance (emitted as the primary `sources` entry's `source_record_urls` list and in the RIG; when `provenance.override.upstream_source_record_urls` is set, RIG only; the per-upstream mapping determines edge placement). At least one URL is required; supply multiple to back a single section with several links. Format-validated only; not fetched. | +| `url` | List[URL] | Yes | One or more source URLs recorded as provenance (derived sections emit them as the upstream repository supporting entry's `source_record_urls` list, infores:pubmed-central for PMC sections, and in the RIG; when `provenance.override.upstream_source_record_urls` is set, RIG only; the per-upstream mapping determines edge placement instead). At least one URL is required; supply multiple to back a single section with several links. Format-validated only; not fetched. | | `delimiter` | String | No | Field delimiter. Defaults to `","`. | | `row_slice` | List[PositiveInt\|"auto"] | No | Two-value zero-based crop bounds: `[start, stop]`. Each value may be a positive integer or `"auto"`. Mutually exclusive with `rows`. | | `rows` | List[PositiveInt] | No | Zero-based row indices to keep after any `row_slice` crop. Mutually exclusive with `row_slice`. | @@ -146,7 +146,7 @@ as-is. |-------|------|----------|-------------| | `kind` | String | No | Source kind. Model default is `"parquet"`, but specify it explicitly in configs. | | `local` | Path | Yes | Local file path the source is read from. The file must already exist here; Tablassert does not download it. | -| `url` | List[URL] | Yes | One or more source URLs recorded as provenance (emitted as the primary `sources` entry's `source_record_urls` list and in the RIG; when `provenance.override.upstream_source_record_urls` is set, RIG only; the per-upstream mapping determines edge placement). At least one URL is required; supply multiple to back a single section with several links. Format-validated only; not fetched. | +| `url` | List[URL] | Yes | One or more source URLs recorded as provenance (derived sections emit them as the upstream repository supporting entry's `source_record_urls` list, infores:pubmed-central for PMC sections, and in the RIG; when `provenance.override.upstream_source_record_urls` is set, RIG only; the per-upstream mapping determines edge placement instead). At least one URL is required; supply multiple to back a single section with several links. Format-validated only; not fetched. | | `row_slice` | List[PositiveInt\|"auto"] | No | Two-value zero-based crop bounds: `[start, stop]`. Each value may be a positive integer or `"auto"`. Mutually exclusive with `rows`. | | `rows` | List[PositiveInt] | No | Zero-based row indices to keep after any `row_slice` crop. Mutually exclusive with `row_slice`. | | `reindex` | List[Reindex] | No | Conditional row filtering | @@ -522,7 +522,7 @@ Override fields: | `knowledge_level` | String | No | Override-specific KL value. Defaults to `statistical_association`. | | `agent_type` | String | No | Override-specific AT value. Defaults to `data_analysis_pipeline`. | -Tablassert emits the graph-level infores (or `infores:` when unset) as the primary entry of the Biolink `sources` list on each edge, `{resource_id: "infores:multiomics-kg", resource_role: "primary_knowledge_source", upstream_resource_ids: [...], source_record_urls: [...]}`, with one additional `supporting_data_source` entry per upstream. When `override.upstream_source_record_urls` is set, the primary entry emits no `source_record_urls` and each mapped supporting entry carries its own instead. When `override.sources` is set, that derivation is skipped entirely and the template entries are emitted verbatim, in order. No flat `primary_knowledge_source` scalar is emitted: current translator-ingests practice carries retrieval provenance only in `sources`, and the Biolink `RetrievalSource` class is where `resource_id` / `upstream_resource_ids` / `source_record_urls` are defined. Each retrieval-source entry uses `resource_id` as its sole identifier. Without `override.sources`, the override cannot set a per-section primary source; manual infores CURIEs belong in `upstream_resource_ids`. Older flat `resource_id` / `primary_knowledge_source` output has been removed so generated KGX matches the Biolink edge contract. +Tablassert emits the graph-level infores (or `infores:` when unset) as the primary entry of the Biolink `sources` list on each edge, `{resource_id: "infores:multiomics-kg", resource_role: "primary_knowledge_source", upstream_resource_ids: [...]}`, with one additional `supporting_data_source` entry per upstream. On the derived path the primary entry emits no section record URLs: the section's `source.url` values ride on the upstream repository's supporting entry (infores:pubmed-central for PMC sections), because that is where the downloadable record lives, and `build-kg --linkouts` puts only the per-edge linkout template on the primary entry. When `override.upstream_source_record_urls` is set, the primary entry emits no `source_record_urls` and each mapped supporting entry carries its own instead; sections with any other `provenance.override` keep the legacy primary-entry emission. When `override.sources` is set, that derivation is skipped entirely and the template entries are emitted verbatim, in order. No flat `primary_knowledge_source` scalar is emitted: current translator-ingests practice carries retrieval provenance only in `sources`, and the Biolink `RetrievalSource` class is where `resource_id` / `upstream_resource_ids` / `source_record_urls` are defined. Each retrieval-source entry uses `resource_id` as its sole identifier. Without `override.sources`, the override cannot set a per-section primary source; manual infores CURIEs belong in `upstream_resource_ids`. Older flat `resource_id` / `primary_knowledge_source` output has been removed so generated KGX matches the Biolink edge contract. #### Explicit sources template @@ -607,7 +607,7 @@ This means nothing in your source data is silently dropped: context that doesn't In addition to user-declared annotations, every edge automatically carries `extracted_from_row_number`, a 1-based index into the original source table (matching Excel-style row numbering). It is not declared as an annotation. Tablassert emits it internally so each edge always carries its source-row provenance, and the inlined supporting study consumes it: the `StudyResult` that anchors the study's row is identified by the scoped CURIE `row:`. The row and sheet columns are never folded into `supporting_text`. -The supporting study's identity is disjoint by construction: `Study.id` is the section's publication CURIE (`PMID:123`, `PMCID:PMC11708054`); `Study.name` is a human label, the worksheet name for spreadsheet sources and the source filename otherwise. It is never a composition of the two. A section declaring **no `publications`** falls back to the config stem (the YAML filename without its extension) as the id, and the struct is then emitted **only when it carries something to preserve**, such as study metadata, routed unsatisfiable values, or class-pruned values. Biolink defines `has supporting studies` as "studies that produced information used as evidence", so a contentless unpublished section emits no `has_supporting_studies` at all; a section with a real publication always keeps the wrapper: +The supporting study's identity is disjoint by construction: `Study.id` is the section's publication CURIE (`PMID:123`, `PMCID:PMC11708054`); `Study.name` is a human label naming the originating table config file and the section's 1-based position within it (e.g. `KANG12.yaml SECTION 2`). It is never a composition of the two. A section declaring **no `publications`** falls back to the config stem (the YAML filename without its extension) as the id, and the struct is then emitted **only when it carries something to preserve**, such as study metadata, routed unsatisfiable values, or class-pruned values. Biolink defines `has supporting studies` as "studies that produced information used as evidence", so a contentless unpublished section emits no `has_supporting_studies` at all; a section with a real publication always keeps the wrapper: ```json "has_supporting_studies": { diff --git a/src/tablassert/models.py b/src/tablassert/models.py index 138c3c4..2d18206 100644 --- a/src/tablassert/models.py +++ b/src/tablassert/models.py @@ -138,7 +138,7 @@ class BaseSource(TablaBase): url: list[HttpUrl] = Field( ..., min_length=1, - description="One or more remote source URL(s) recorded as provenance; emitted in the edge `sources` list under the primary entry's `source_record_urls` list and in the RIG. When `provenance.override.upstream_source_record_urls` is set, these URLs serve the RIG only and the per-upstream mapping determines edge placement instead. Format-validated only; not fetched.", + description="One or more remote source URL(s) recorded as provenance. Derived sections emit them in the edge `sources` list under the upstream repository supporting entry's `source_record_urls` list (infores:pubmed-central for PMC sections) and in the RIG. When `provenance.override.upstream_source_record_urls` is set, these URLs serve the RIG only and the per-upstream mapping determines edge placement instead; any other provenance.override keeps the primary-entry emission. Format-validated only; not fetched.", ) rows: list[NonNegativeInt] | None = Field(None, description="Zero-based row indices kept after any row_slice crop.", examples=[[0, 2, 5]]) From 0b11f94d7ba43c3a87da973ecd2287ea334853b9 Mon Sep 17 00:00:00 2001 From: SkyeAv Date: Thu, 15 Oct 2026 13:29:48 -0700 Subject: [PATCH 4/6] test: align to_sections annotations with the corrected return type --- tests/test_ingests.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_ingests.py b/tests/test_ingests.py index b92fb4e..92df62b 100644 --- a/tests/test_ingests.py +++ b/tests/test_ingests.py @@ -95,7 +95,7 @@ def test_to_sections_expands_template(fixtures_path: Path) -> None: """to_sections expands template with sections.""" data: Any = from_yaml(fixtures_path / "minimal_section_with_sections.yaml") table: Path = Path("test_config.yaml") - sections: list[list[dict[str, Any]]] = to_sections(data, table) + sections: list[dict[str, Any]] = to_sections(data, table) assert len(sections) == 2 @@ -103,7 +103,7 @@ def test_to_sections_merges_template(fixtures_path: Path) -> None: """to_sections merges template into each section.""" data: Any = from_yaml(fixtures_path / "minimal_section_with_sections.yaml") table: Path = Path("test_config.yaml") - sections: list[list[dict[str, Any]]] = to_sections(data, table) + sections: list[dict[str, Any]] = to_sections(data, table) # each section should inherit provenance from template for section in sections: assert "provenance" in section @@ -113,7 +113,7 @@ def test_to_sections_adds_config(fixtures_path: Path) -> None: """to_sections adds config path.""" data: Any = from_yaml(fixtures_path / "minimal_section_with_sections.yaml") table: Path = Path("test_config.yaml") - sections: list[list[dict[str, Any]]] = to_sections(data, table) + sections: list[dict[str, Any]] = to_sections(data, table) for section in sections: assert section["config"] == table # pyright: ignore @@ -122,7 +122,7 @@ def test_to_sections_unique_objects(fixtures_path: Path) -> None: """to_sections each section has unique object.""" data: Any = from_yaml(fixtures_path / "minimal_section_with_sections.yaml") table: Path = Path("test_config.yaml") - sections: list[list[dict[str, Any]]] = to_sections(data, table) + sections: list[dict[str, Any]] = to_sections(data, table) # sections override object encoding from the sections list obj_encodings: list[str] = [] for section in sections: From 4338f7f1990169f0687e4335e51e55e37f74ebb9 Mon Sep 17 00:00:00 2001 From: SkyeAv Date: Thu, 15 Oct 2026 13:36:39 -0700 Subject: [PATCH 5/6] refactor: apply review fixes for provenance re-homing - retrieval_sources Args doc now matches the upstream_urls branch, which passes primary urls through (the --linkouts template). - Drop the dead study_name == study_id guard in _provenance_ops: a " SECTION " name can never equal the id. - Rename the flattened-results local in the build loop (flat_index) so it cannot be confused with the new per-file Tcode.section_index. - Pin the PMID re-homing branch and the override-section Study name in new tests. --- src/tablassert/cli.py | 4 ++-- src/tablassert/lib.py | 8 ++++---- tests/test_lib.py | 38 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 6 deletions(-) diff --git a/src/tablassert/cli.py b/src/tablassert/cli.py index 33663cf..1c18a58 100644 --- a/src/tablassert/cli.py +++ b/src/tablassert/cli.py @@ -420,12 +420,12 @@ def build_graph_pipeline( advance() with RunCache() as cache: for planned in plan.execution_order: - section_index, x, op = pending[planned] + flat_index, x, op = pending[planned] entry: PlanEntry = plan.entries[planned] start(format_section_compact(x)) # on_phase drives the per-op sub-step indicator (cache → resolve → write ...); # resume/snapshot are the producer/consumer seams the plan assigned this section. - results[section_index] = compile_subgraph( + results[flat_index] = compile_subgraph( op, on_phase=sub_step, resume=_run_cache_resume(cache, entry), snapshot=_run_cache_snapshot(cache, entry) ) advance() diff --git a/src/tablassert/lib.py b/src/tablassert/lib.py index e88033a..fb540a9 100644 --- a/src/tablassert/lib.py +++ b/src/tablassert/lib.py @@ -629,7 +629,9 @@ def retrieval_sources( lf: Source LazyFrame. primary: Infores CURIE of the primary knowledge source. upstream: Infores CURIEs of upstream/supporting data sources. - urls: Source record URLs for the primary entry (ignored when ``upstream_urls`` is set). + urls: URLs for the primary entry (when ``upstream_urls`` is set these must not be + source records -- e.g. the per-edge EdgeLinkouts template -- since record URLs + ride the supporting entries in that mode). upstream_urls: Optional per-upstream source record URLs keyed by infores CURIE. explicit: Optional explicit entry templates emitted verbatim, in order. @@ -1502,9 +1504,7 @@ def _provenance_ops(self: Self) -> list[Any]: # `inline_supporting_study` skips the struct unless it has content to carry. identified: bool = bool(publication_values) study_id: str = publication_values[0] if publication_values else self.config.stem - study_name: str | None = f"{self.config.name} SECTION {self.section_index}" - if study_name == study_id: - study_name = None + study_name: str = f"{self.config.name} SECTION {self.section_index}" # `use_enum_values` stores the override's enum keys/values as plain strings. category_override: dict[str, str] | None = ( {obj: f"biolink:{pinned}" for obj, pinned in self.statement.category_override.items()} if self.statement.category_override else None diff --git a/tests/test_lib.py b/tests/test_lib.py index a25879d..fa084fe 100644 --- a/tests/test_lib.py +++ b/tests/test_lib.py @@ -995,6 +995,44 @@ def test_tcode_collect_rehomes_pmc_record_urls_onto_pubmed_central(fixtures_path assert pmc["source_record_urls"] == ["https://pmc-oa-opendata.s3.amazonaws.com/PMC10010437.1/Table3.xlsx"] +def test_tcode_collect_rehomes_pmid_record_urls_onto_pubmed(fixtures_path: Path) -> None: + """A PMID-backed section emits record URLs on the infores:pubmed supporting entry. + + Why: the re-homing keys off ``upstream_resource_ids(repo)``, so the PMID branch must + pin the same placement contract as the PMC branch (infores:pubmed-central). + """ + data: Any = from_yaml(fixtures_path / "minimal_section.yaml") + data["provenance"] = {"repo": "PMID", "publication": "12345678"} + store: Path = Path("/tmp/sectionhash_pmid.parquet") + tcode_model: Tcode = Tcode.model_validate( # pyright: ignore + {**data, "config": fixtures_path / "minimal_section.yaml", "store": store, "infores": "infores:test-kg"} + ) + + collected: list[tuple[Any, tuple[Any]]] = tcode_model.collect(Path("/tmp/fullmap.redb")) # pyright: ignore + source_args: tuple[Any, ...] = next(op[1] for op in collected if op[0] is retrieval_sources) + + assert source_args[3] == {"infores:pubmed": ["https://example.com/test.tsv"]} + + +def test_tcode_collect_names_override_study_after_config_section(fixtures_path: Path) -> None: + """Override sections get the same " SECTION " Study name as derived sections. + + Why: the rename in ``_provenance_ops`` is override-agnostic and the docs state the + naming generally; this pins it so an override carve-out cannot silently change it. + """ + data: Any = from_yaml(fixtures_path / "minimal_section.yaml") + data["provenance"] = {"override": {"upstream_resource_ids": ["infores:upstream-source"]}} + store: Path = Path("/tmp/sectionhash_override_name.parquet") + tcode_model: Tcode = Tcode.model_validate( # pyright: ignore + {**data, "config": fixtures_path / "minimal_section.yaml", "store": store, "infores": "infores:test-kg"} + ) + + collected: list[tuple[Any, tuple[Any]]] = tcode_model.collect(Path("/tmp/fullmap.redb")) # pyright: ignore + study_op: tuple[Any, tuple[Any, ...]] = next(op for op in collected if op[0].__name__ == "inline_supporting_study") + + assert study_op[1][1] == "minimal_section.yaml SECTION 1" + + def test_tcode_collect_upstream_source_record_urls_rehome_urls(fixtures_path: Path) -> None: """``override.upstream_source_record_urls`` leaves the primary bare and attaches URLs to supporting entries. From aeef8cf0a35b903cdb7440bd37836c0c1956e022 Mon Sep 17 00:00:00 2001 From: SkyeAv Date: Thu, 15 Oct 2026 13:37:49 -0700 Subject: [PATCH 6/6] docs: add Unreleased breaking-changes entries for PR #220 --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 34f57ca..b244d4c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project are documented in this file. +## Unreleased + +### Breaking Changes +- **Derived-path source record URLs now ride the upstream repository's supporting `sources` entry, not the primary.** Sections without a `provenance.override` emit their `source.url` values as `source_record_urls` on the supporting entry matching the publication repository (`infores:pubmed-central` for PMC sections, `infores:pubmed` for PMID sections), because the primary knowledge source is the transforming resource, not the downloadable record; the primary entry keeps `upstream_resource_ids` and, with `build-kg --linkouts`, the per-edge linkout template (`{edge_id}` still resolves in the post-dedup sweep). Override sections are byte-identical: `override.upstream_source_record_urls` re-homes exactly as specified, other overrides keep the legacy primary-entry emission, and `--linkouts` still never injects into override sections. Downstream consumers keying off the primary entry's `source_record_urls` must read the supporting entry instead. ([#220](https://github.com/SkyeAv/Tablassert/pull/220)) + **Migration:** read record URLs from the `supporting_data_source` entry whose `resource_id` matches the publication repository; no config change is required. +- **The inlined supporting Study is named after the table config file and section index.** `Study.name` reads ` SECTION ` (1-based position within that file, stamped by `ingests.to_sections`) instead of the worksheet name or source data filename, so edge provenance points back to the exact configuration; `Study.id` is unchanged (publication CURIE, or the config stem when unidentified). Section cache parquets content-address the new stamp, so pre-upgrade caches miss once and rebuild. ([#220](https://github.com/SkyeAv/Tablassert/pull/220)) + ## 20.0.0 - 2026-10-14 ### Breaking Changes