diff --git a/.gitignore b/.gitignore index 5f1c1695..121f3484 100644 --- a/.gitignore +++ b/.gitignore @@ -154,6 +154,8 @@ pytest.ini # Misc *.bak misc/clinvar +misc/refgetstore/store/ +misc/refgetstore/downloads/ notebooks/scratch/ notes lint diff --git a/2026-04-15T211800Z-refgetstore-seqrepo-comparison.md b/2026-04-15T211800Z-refgetstore-seqrepo-comparison.md new file mode 100644 index 00000000..5ee5bdd4 --- /dev/null +++ b/2026-04-15T211800Z-refgetstore-seqrepo-comparison.md @@ -0,0 +1,329 @@ +# Refgetstore vs seqrepo 2024-12-20 — coverage comparison snapshot + +Snapshot date: 2026-04-15. Captured after the `ensembl_human_cdna`, +`ensembl_human_ncrna`, and `ensembl_human_pep` seqsets landed on top of +the existing 4 assemblies + 32 NCBI RefSeq mRNA/Prot shards. + +This doc inventories what's in our custom refgetstore +(`misc/refgetstore/store/`), compares it alias-by-alias to the seqrepo +instance at `/Users/kferrite/dev/data/seqrepo/2024-12-20`, and flags +the one investigation thread that's still open (Ensembl ENSP digest +mismatches). + +## 1. Refgetstore inventory + +Raw stats (`RefgetStore.stats()`): + +``` +n_sequences: 783 625 unique sha512t24u digests across all sources +n_collections: 39 4 assemblies + 32 RefSeq shards + 3 Ensembl FASTAs +storage_mode: Encoded (on-disk, lazy sequence byte loading) +``` + +Sources loaded (from [assemblies.toml](misc/refgetstore/assemblies.toml)): + +| Source | Collection count | File count | Fresh aliases written | +|----------------------------------------------------------|-----------------:|-----------:|----------------------:| +| NCBI GRCh38 major (`GCF_000001405.26`) | 1 | 1 | 1 820 | +| NCBI GRCh38.p14 (`GCF_000001405.40`) | 1 | 1 | 1 918 | +| NCBI GRCh37 major (`GCF_000001405.13`) | 1 | 1 | 344 | +| NCBI GRCh37.p13 (`GCF_000001405.25`) | 1 | 1 | 694 | +| NCBI RefSeq human mRNA (`human.1..16.rna.fna.gz`) | 16 | 16 | 273 558 | +| NCBI RefSeq human protein (`human.1..16.protein.faa.gz`) | 16 | 16 | 197 939 | +| Ensembl r113 cdna (`Homo_sapiens.GRCh38.cdna.all.fa.gz`) | 1 | 1 | 207 133 | +| Ensembl r113 ncrna (`Homo_sapiens.GRCh38.ncrna.fa.gz`) | 1 | 1 | 203 776 | +| Ensembl r113 pep (`Homo_sapiens.GRCh38.pep.all.fa.gz`) | 1 | 1 | 123 845 | + +Per-namespace alias counts (`list_sequence_alias_namespaces()`): + +| Namespace | Count | Source | +|-----------------|--------:|-----------------------------------------------| +| `GRCh38` | 910 | GRCh38 major assembly report — chr*/CM*/etc. | +| `GRCh38.p14` | 1 410 | GRCh38.p14 assembly report | +| `GRCh37` | 184 | GRCh37 major assembly report | +| `GRCh37.p13` | 390 | GRCh37.p13 assembly report | +| `insdc` | 941 | GenBank-Accn column across all assemblies | +| `refseq` | 472 438 | RefSeq-Accn across assemblies + NM/NR/XM/XR/NP/XP from mRNA/Prot shards | +| `ensembl` | 534 754 | ENST/ENSP from Ensembl r113 cdna+ncrna+pep | + +Collection namespaces: + +- `refseq:GCF_*` — 4 entries (the 4 NCBI assemblies) +- `insdc:GCA_*` — 4 entries (the 4 GenBank counterparts) + +The seqset FASTAs (RefSeq shards and Ensembl files) each become their +own collection but do **not** get collection-level aliases — no +`refseq:GCF_*` equivalent exists for a transcript shard, and we don't +synthesize one. + +## 2. Seqrepo inventory for comparison + +Seqrepo namespaces at `/Users/kferrite/dev/data/seqrepo/2024-12-20` +(from `SELECT namespace, COUNT(*) FROM seqalias WHERE is_current=1 GROUP BY namespace`): + +``` +Ensembl 388 235 +NCBI 885 113 +GRCh37 184 +GRCh37.p2 .. GRCh37.p13 256 .. 390 +GRCh38 910 +GRCh38.p1 .. GRCh38.p12 926 .. 998 +CHM1_1.1 23 +NCBI34 3 +NCBI35 26 +NCBI36 149 +JRGv1, JRGv2 25 each +hs37-1kg 84 +hs37d5 86 +MD5, SEGUID, SHA1, VMC 1 144 093 each (digest-form aliases) +``` + +Things seqrepo has that our refgetstore does **not**: + +- `GRCh38.p1` through `GRCh38.p13` — older patch levels (we pinned p14 + only since each patch is a superset of the prior). +- `GRCh37.p2`, `p5`, `p9`, `p10`, `p11`, `p12` — intermediate patches + (we pinned p13 only). +- `CHM1_1.1`, `NCBI34`, `NCBI35`, `NCBI36`, `JRGv1`, `JRGv2`, + `hs37-1kg`, `hs37d5` — legacy assemblies predating GRCh37/38, plus + alt-contig sets. +- `MD5`, `SEGUID`, `SHA1`, `VMC` — digest-form aliases under non-SHA512t24u + algorithms. These are derivable at query time, so we deliberately + don't store them (see + [build_store.py:add_aliases_for_row](misc/refgetstore/build_store.py)). + +Things our refgetstore has that seqrepo does **not**: + +- `GRCh38.p14` — our pinned patch level postdates what seqrepo has. +- `ensembl` **ncRNA biotype** (XIST, lincRNAs, miRNAs, snoRNAs, + pseudogene transcripts, etc.) — seqrepo only loaded Ensembl cdna + historically, so ~200 k ncRNA ENSTs exist only in our store. + +## 3. Alias-level cross-check (refget vs seqrepo) + +### 3.1 Assembly namespaces (genomic) + +No systematic comparison was run; the canonical digests we care about +(chr1, chr22, chrX, chrY, chrM, NC_000001.11, CM000663.2) were already +verified by +[misc/refgetstore/verify_store.py](misc/refgetstore/verify_store.py)'s +sections [C] and [D]. 41/41 checks pass. + +### 3.2 `refseq` / seqrepo `NCBI` namespace + +Seqrepo calls this `NCBI`; our store calls it `refseq`. They contain +NM/NR/NP + NC/NG/NT/NW accessions. + +| Bucket | count | % of seqrepo NCBI | +|--------------------------------------------------------|---------:|------------------:| +| Both resolve, digests agree | 464 830 | 52.52 % | +| Digest mismatch on the same versioned accession | 0 | 0.00 % | +| In seqrepo, not in refgetstore | 420 283 | 47.48 % | +| In refgetstore, not in seqrepo | 7 608 | n/a | + +**Zero digest mismatches**. The 420 k "seqrepo-only" rows are dominated +by historical versions that seqrepo has accumulated across loads but +that the current RefSeq FTP `mRNA_Prot/human.*.rna.fna.gz` / +`human.*.protein.faa.gz` shards no longer publish (NCBI ships "current" +only, the way Ensembl does). Our store matches whatever is in the +shards as of the download date. + +Our `refseq` composition (by accession prefix): + +``` +XM_ : 127 028 # predicted mRNAs +XP_ : 127 028 # predicted proteins +NM_ : 70 898 # curated mRNAs +NP_ : 70 898 # curated proteins +XR_ : 51 132 # predicted ncRNAs +NR_ : 24 500 # curated ncRNAs +NW_ : 477 # genomic contigs from assemblies +NT_ : 415 # genomic contigs from assemblies +NC_ : 49 # chromosome/organelle entries from assemblies +YP_ : 13 # proteins from GenBank WGS, assembly-embedded +``` + +The `X*_` / `N*_` split matches the upstream RefSeq curation tiers +exactly: "model" (X*) vs "curated" (N*). + +### 3.3 `ensembl` namespace + +| Bucket | count | % of seqrepo Ensembl | +|-------------------------------------------------------------|---------:|---------------------:| +| Both resolve, digests agree | 322 286 | 83.01 % | +| Digest mismatch on the same versioned accession (ENSP 115, ENST 1) | 116 | 0.03 % | +| In seqrepo, not in refgetstore | 65 833 | 16.96 % | +| In refgetstore, not in seqrepo | 212 352 | n/a | + +83% exact overlap without any deliberate release alignment — that's +good confirmation that the seqset loader + our release-113 source is +sane. The 116 digest mismatches are discussed separately below. + +Stem-level breakdown of the asymmetries (helpful for understanding +what's missing where): + +**seqrepo-only (65 833 rows)** — versions or stems seqrepo has but +release 113 does not: + +``` + ENST ENSP +same stem, diff version: 43 881 6 474 ← older transcript/protein + versions (BRCA1 .7 alongside .9) +stem not in refgetstore: 7 987 6 852 ← entire stems retired or never + in release 113 +``` + +**refget-only (212 352 rows)** — versions or stems we have that +seqrepo never loaded: + +``` + ENST ENSP +same stem, newer version: 22 862 4 732 ← normal bumps between seqrepo's + load date and release 113 +stem not in seqrepo: 167 490 17 268 ← Ensembl ncRNA FASTA + new + genes/proteins added since 2024 +``` + +The ~167 k "ENST stem not in seqrepo at all" rows are almost entirely +the `ncrna/` biotype, which seqrepo's historical loads omit — compare +with the 203 776 entries in `Homo_sapiens.GRCh38.ncrna.fa.gz`. About +30 k of those ncRNAs are also in seqrepo under some version (matching +the `same_stem` buckets); the rest (~167 k) are new coverage. + +Breakdown of `ensembl` aliases by prefix: + +| Prefix | refgetstore | seqrepo | +|--------|------------:|---------:| +| `ENST` | 410 909 | 272 425 | +| `ENSP` | 123 845 | 115 171 | +| other | 0 | 639 | + +The ~639 "other" seqrepo Ensembl aliases are things like `CHR_*` and +`KI27*` — these are assembly contig aliases that somehow ended up +under the Ensembl namespace in an older seqrepo load. We don't +reproduce them. + +## 4. Resolved — 116 Ensembl digest mismatches + +**Status**: Investigated and root-caused (2026-04-16). See +[`misc/refgetstore/diagnose_ensembl_mismatches.py`](misc/refgetstore/diagnose_ensembl_mismatches.py) +and [`misc/refgetstore/diagnosis_report.json`](misc/refgetstore/diagnosis_report.json). + +**Distribution**: 115 ENSP + 1 ENST. + +### Root cause: two distinct mechanisms + +#### 4a. `*` (stop codon) normalization — 115 ENSP mismatches + +The underlying sequence bytes are **identical** in both stores. +The digest difference is caused by `bioutils.sequences.normalize_sequence()` +stripping `*` characters (and whitespace) before hashing: + +- **SeqRepo** computes its `seq_id` via `bioutils.digests.seq_seqhash(seq)`, + which calls `normalize_sequence(seq)` → strips `*` → uppercases → + SHA-512 → truncate to 24 bytes → base64url. The resulting digest + becomes the `ga4gh:SQ.{digest}` alias. +- **RefgetStore** (gtars) computes `sha512t24u` directly from the raw + FASTA bytes, including `*`. This is the GA4GH VRS-conformant approach. + +For protein sequences without `*`, the two algorithms produce the same +digest (as confirmed by the 322,286 agreeing aliases). Only the 115 +ENSPs that contain embedded stop codons diverge. + +Example (ENSP00000340011.8, KIR2DS4, 297 aa, 3 internal `*`): + +``` +seqrepo seq_id (normalized): fv4uQW4AfrwGj3dx9D4xfpccU7Qnqtia ← sha512t24u("...TEQ TARILM...") +refget sha512t24u (raw): 8Co4fICavUFj9qH_GGXd5cbWbzh8oSZ0 ← sha512t24u("...TEQ*TARILM...") + ^ this * is the difference +``` + +**Conclusion**: refgetstore's digest is correct per the GA4GH VRS +specification (sha512t24u of the raw bytes). SeqRepo's digest is a +legacy artifact of `bioutils.normalize_sequence()` stripping `*` before +hashing. This does NOT affect DNA sequences. + +#### 4b. Sequence content divergence — 1 ENST mismatch + +ENST00000668831.1 has genuinely different content: seqrepo loaded +1,214 bp (added 2020-04-12, from an older Ensembl release), while +refgetstore has 1,120 bp from release 113. The sequences share no +homology at position 0 — Ensembl re-annotated this transcript between +releases under the same version number. + +Both stores are self-consistent (sha512t24u matches their own stored +bytes). The refgetstore carries the r113 version, which is +authoritative for that release. + +### Impact on VRS interoperability + +For vrs-python's primary use case (genomic DNA variants), this has +**zero impact** — DNA sequences never contain `*` and the digest +algorithms agree exactly. The divergence only affects callers who: + +1. Look up protein sequences with embedded `*` (stop codons) +2. AND use the resulting `ga4gh:SQ.*` identifier for cross-system + comparison between seqrepo and refgetstore + +The 115 affected ENSPs are listed in +[`misc/refgetstore/ensembl_known_divergent.txt`](misc/refgetstore/ensembl_known_divergent.txt). + +## 5. What this snapshot does NOT include + +- No comparison of sequence bytes themselves — only digest equality + was checked. If seqrepo and refgetstore both report the same + sha512t24u, the bytes are identical by construction. +- No latency/throughput comparison beyond the earlier + [bulk_equivalence.py](misc/refgetstore/bulk_equivalence.py) / + [vcf_equivalence.py](misc/refgetstore/vcf_equivalence.py) runs, + which showed refget ~1.15× faster than seqrepo on the translator + path and ~1.14× on the VCF annotator path. +- No attempt to load GRCh37 Ensembl, Ensembl regulatory features + (`ENSR*`), or Ensembl gene sequences (`ENSG*`). Those were + explicitly out of scope for the release-113 seqset plan + (`~/.claude/plans/rippling-splashing-cerf.md`). +- No check of seqrepo's non-current (`is_current=0`) rows. All counts + use `WHERE is_current=1`. + +## 6. Files of interest + +- [misc/refgetstore/assemblies.toml](misc/refgetstore/assemblies.toml) — + declarative source list, now with 4 assemblies + 2 sharded RefSeq + seqsets + 3 Ensembl seqsets. +- [misc/refgetstore/build_store.py](misc/refgetstore/build_store.py) — + loader; `process_seqset` uses the bulk `load_sequence_aliases` path + that makes ~500 k new aliases finish in sub-second time. +- [misc/refgetstore/verify_store.py](misc/refgetstore/verify_store.py) — + 55 checks (was 41 before Ensembl) including a `[H]` Ensembl + coverage section with VHL / BRCA1 / XIST / TP53 ground-truth. +- [misc/refgetstore/compare_against_seqrepo.py](misc/refgetstore/compare_against_seqrepo.py) — + 11 canonical-identifier checks (spot-check, not bulk); still 74/74 + passing after this round. +- `/Users/kferrite/dev/data/seqrepo/2024-12-20/aliases.sqlite3` — the + seqrepo reference used for all cross-checks in this doc. + +## 7. Suggested next steps (in rough priority order) + +1. ~~**Investigate the 115 ENSP digest mismatches**~~ — **Done** (§4). + Root cause: `bioutils.normalize_sequence()` strips `*` before + hashing. RefgetStore digest is GA4GH-conformant; seqrepo's is not. + No code fix needed — documented and accessions listed. +2. **Run a translator-path bulk_equivalence comparison that + specifically stresses ENST/ENSP inputs** (e.g. feed + `NM_*:c.1A>T`-style HGVS expressions that coerce into Ensembl via + the translator's namespace fallback). The gnomAD chr1 VCF we + currently use is purely genomic and doesn't exercise any of the + Ensembl alias code paths. +3. **Decide whether to load GRCh37 Ensembl** (archive release 87). + Probably still not needed, but worth revisiting if any caller + reports KeyError on a GRCh37-only ENST. +4. **Document the version-drift reality in the README**: make clear + that refgetstore ships "latest-per-release" only, not the + cumulative version history that seqrepo accumulates. Callers that + depend on older transcript versions need to load additional + release snapshots. +5. **Consider a periodic rebuild script**. Pinning release 113 in + `assemblies.toml` is good for reproducibility, but we should have + a clear story for bumping it (update the URL, rerun, verify + digests against ground-truth, update `ENSEMBL_GROUND_TRUTH` in + `verify_store.py`). diff --git a/2026-04-16T135514Z-ensembl-digest-mismatch-investigation.md b/2026-04-16T135514Z-ensembl-digest-mismatch-investigation.md new file mode 100644 index 00000000..88f867cc --- /dev/null +++ b/2026-04-16T135514Z-ensembl-digest-mismatch-investigation.md @@ -0,0 +1,333 @@ +# Investigation: 116 Ensembl Digest Mismatches Between SeqRepo and RefgetStore + +**Date**: 2026-04-16 +**Branch**: `kf/dataproxy-backend-refactor` +**Investigator**: Kyle Ferriter + Claude + +## 1. Background + +The `kf/dataproxy-backend-refactor` branch introduces a refget-backed +data proxy as an alternative to seqrepo. A cross-check of the `ensembl` +namespace between seqrepo (2024-12-20) and the custom refgetstore +(Ensembl r113) found **116 aliases** where the stored digest identifiers +diverge despite both systems being loaded from Ensembl protein/transcript +FASTAs. + +Totals from the alias-level cross-check: + +``` +total aliases checked: 600,587 +both resolve, agree: 322,286 +both resolve, MISMATCH: 116 (115 ENSP + 1 ENST) +only in seqrepo: 65,833 +only in refgetstore: 212,352 +``` + +This report documents the investigation, root cause, and impact. + +## 2. Initial Hypotheses + +Five hypotheses were ranked before investigation (from the +[comparison report](2026-04-15T211800Z-refgetstore-seqrepo-comparison.md)): + +1. Ensembl silently re-published proteins under the same version number +2. **Seqrepo's loader stripped something (e.g. `*`) that refgetstore kept** +3. Cross-release contamination in seqrepo +4. Isoform equivalence class ambiguity +5. Bug in the ingest path + +## 3. Investigation Method + +### 3.1 Direct byte comparison + +The first diagnostic step was to fetch the sequence bytes from both stores +for a sample mismatch (ENSP00000340011.8, KIR2DS4) and compare them: + +```python +from biocommons.seqrepo import SeqRepo +from gtars.refget import RefgetStore +from ga4gh.core.digests import sha512t24u + +sr = SeqRepo('/path/to/seqrepo/2024-12-20') +sr_seq = sr.fetch_uri('Ensembl:ENSP00000340011.8') + +store = RefgetStore.open_local('misc/refgetstore/store') +rec = store.get_sequence_by_alias('ensembl', 'ENSP00000340011.8') +store.load_sequence(rec.metadata.sha512t24u) +rg_seq = store.get_substring(rec.metadata.sha512t24u, 0, rec.metadata.length) +``` + +**Result**: The sequences are **byte-identical** (297 aa, both uppercase, +both containing `*` at positions 239, 269, 280). Yet the stored +identifiers differ: + +``` +seqrepo seq_id: fv4uQW4AfrwGj3dx9D4xfpccU7Qnqtia +refgetstore sha512t24u: 8Co4fICavUFj9qH_GGXd5cbWbzh8oSZ0 +``` + +Computing `sha512t24u` from the raw bytes confirms only the refgetstore +digest: + +```python +>>> sha512t24u(sr_seq.encode('ascii')) +'8Co4fICavUFj9qH_GGXd5cbWbzh8oSZ0' # matches refgetstore, NOT seqrepo +``` + +### 3.2 Tracing the digest divergence + +Two different functions compute what appear to be the same hash but +produce different results: + +**GA4GH VRS digest** (`src/ga4gh/core/digests.py:12-34`): + +```python +def sha512t24u(blob: bytes) -> str: + digest_size = 24 + digest = hashlib.sha512(blob).digest() + tdigest_b64us = base64.urlsafe_b64encode(digest[:digest_size]) + return tdigest_b64us.decode("ascii") +``` + +This takes raw bytes and hashes them directly. No normalization. + +**bioutils digest** (`bioutils.digests.seq_seqhash`): + +```python +def seq_seqhash(seq, normalize=True): + seq = normalize_sequence(seq) if normalize else seq + return str(vmc_digest(seq, digest_size=24)) +``` + +This calls `normalize_sequence()` before hashing. + +**The normalization function** (`bioutils.sequences.normalize_sequence`): + +```python +def normalize_sequence(seq): + """Removes whitespace and asterisks, and uppercases the string.""" + nseq = re.sub(r"[\s\*]", "", seq).upper() + m = re.search("[^A-Z]", nseq) + if m: + raise RuntimeError("Normalized sequence contains non-alphabetic characters") + return nseq +``` + +The key line: `re.sub(r"[\s\*]", "", seq)` — **strips all `*` characters** +before hashing. + +### 3.3 How SeqRepo uses the normalized digest + +SeqRepo computes its internal `seq_id` using `seq_seqhash()` (with +normalization enabled by default). The `seq_id` becomes: +- The primary key for sequence storage +- The value behind the `ga4gh:SQ.*` alias namespace + +When `SeqRepoDataProxy.get_aliases()` (`dataproxy_seqrepo.py:51-59`) +returns aliases for a sequence, the `ga4gh:SQ.{seq_id}` entry uses this +normalized digest. Any caller that calls `derive_refget_accession()` gets +back `SQ.{normalized_digest}`. + +### 3.4 How RefgetStore uses the raw digest + +gtars `RefgetStore` computes `sha512t24u` directly from the FASTA bytes. +When `RefgetAliasDataProxy.get_aliases()` (`dataproxy_refget.py:216-251`) +synthesizes the `ga4gh:SQ.*` alias, it uses this raw digest: + +```python +# dataproxy_refget.py:249-250 +aliases.append(f"sha512t24u:{digest}") +aliases.append(f"ga4gh:SQ.{digest}") +``` + +### 3.5 Proof: DNA sequences are unaffected + +For a DNA sequence like NC_000001.11 (chr1), `normalize_sequence()` has +no effect — DNA bases have no `*` to strip, and uppercasing is a no-op +for already-uppercase FASTA. Both algorithms produce identical digests: + +``` +NC_000001.11 seqrepo seq_id: Ya6Rs7DHhDeg7YaOSg1EoNi3U_nQ9SvO +NC_000001.11 ga4gh sha512t24u: Ya6Rs7DHhDeg7YaOSg1EoNi3U_nQ9SvO ← identical +``` + +This is why all 464,830 refseq namespace aliases and all 322,286 +non-`*`-containing Ensembl aliases agree perfectly. + +## 4. Automated Verification + +A diagnostic script (`misc/refgetstore/diagnose_ensembl_mismatches.py`) +was written to verify the hypothesis across all 116 mismatches. It +operates in 4 phases: + +1. **Phase A**: Enumerate all `ensembl` namespace aliases from both stores + (reading seqrepo's SQLite directly and refgetstore's + `aliases/sequences/ensembl.tsv`), identify where digests differ. + +2. **Phase B**: For each mismatch, fetch full sequences from both stores, + compare byte-by-byte, compute `sha512t24u` from each side's raw bytes, + and check whether seqrepo's stored `seq_id` matches + `seq_seqhash(seq, normalize=True)`. + +3. **Phase C**: Classify each mismatch as `star_normalization` (identical + bytes + `*` present) or `sequence_divergence` (different bytes). + +4. **Phase D**: Output JSON report + human-readable summary. + +### Results + +``` +Total mismatches: 116 +Cause: * normalization (same bytes): 115 +Cause: sequence divergence (diff bytes): 1 +Fetch errors: 0 +seqrepo id matches normalized digest: 116/116 +refget id self-consistent: 116/116 +``` + +**Every** seqrepo `seq_id` matches the result of `seq_seqhash(bytes)`. +**Every** refgetstore digest matches `sha512t24u(bytes)`. The two hash +functions simply disagree on what to hash (raw bytes vs `*`-stripped bytes). + +### Stop codon distribution + +The 115 `star_normalization` mismatches contain between 1 and 23 internal +stop codons: + +| `*` count | Proteins | +|-----------|----------| +| 1 | 54 | +| 2 | 19 | +| 3 | 17 | +| 4 | 3 | +| 6-23 | 22 | + +### Representative examples + +``` +ENSP00000375609.1 len= 225 stars= 1 pos=[222] + seqrepo (normalized): 0wFdmvBugnimf4tOFjCVdckpw3P6GmOx + refget (raw): Qg7FS7qyO-pfHp7HYd4wvr26DEx1oov4 + sha512t24u(bytes): Qg7FS7qyO-pfHp7HYd4wvr26DEx1oov4 ← matches refget + +ENSP00000340011.8 len= 297 stars= 3 pos=[239, 269, 280] + seqrepo (normalized): fv4uQW4AfrwGj3dx9D4xfpccU7Qnqtia + refget (raw): 8Co4fICavUFj9qH_GGXd5cbWbzh8oSZ0 + sha512t24u(bytes): 8Co4fICavUFj9qH_GGXd5cbWbzh8oSZ0 ← matches refget + +ENSP00000434417.1 len=2459 stars=17 pos=[2130, 2153, ..., 2418] + seqrepo (normalized): zKGzcqTwIi1cpsh_25jTR-g7cycsIEpf + refget (raw): Ih9FQHBcIpxLvdvSmQwf1B_uW07yCbjE + sha512t24u(bytes): Ih9FQHBcIpxLvdvSmQwf1B_uW07yCbjE ← matches refget +``` + +In every case, `sha512t24u(actual_bytes)` matches the refgetstore digest, +not the seqrepo `seq_id`. + +## 5. The Sequence-Divergent Outlier + +ENST00000668831.1 is a genuine data difference, not a hashing issue: + +``` +seqrepo: 1,214 bp (loaded 2020-04-12, from an older Ensembl release) +refgetstore: 1,120 bp (Ensembl r113) +First base: seqrepo='T' vs refget='A' — completely different content +``` + +Both stores are self-consistent (their stored digests match +`sha512t24u` of their own bytes). Ensembl re-annotated this transcript +between releases under the same version number — a rare but documented +occurrence for complex genomic loci. + +## 6. Which Side Is Correct? + +The GA4GH VRS specification defines the sequence identifier as +`sha512t24u` computed over the raw sequence bytes. The specification +does not mandate stripping `*` or any other normalization before +hashing. Therefore: + +- **RefgetStore is correct** per GA4GH VRS: it computes `sha512t24u` + directly from the stored bytes. +- **SeqRepo is non-conformant** for protein sequences containing `*`: + its `seq_id` (and thus its `ga4gh:SQ.*` alias) is computed from + `*`-stripped bytes, producing a different identifier. + +However, this non-conformance is embedded in the `bioutils` library +(`bioutils.digests.seq_seqhash` → `bioutils.sequences.normalize_sequence`) +which is upstream of both seqrepo and vrs-python. Changing it would be a +breaking change affecting all existing seqrepo identifiers for +`*`-containing sequences. + +## 7. Practical Impact + +### 7.1 DNA workloads: zero impact + +DNA sequences never contain `*`. All genomic operations (VCF annotation, +HGVS translation, allele normalization) are completely unaffected. +This was confirmed by: +- 1,000,000 VCF records: zero allele ID mismatches + (`misc/refgetstore/bulk_equivalence.py`) +- 74/74 canonical identifier checks + (`misc/refgetstore/compare_against_seqrepo.py`) + +### 7.2 Ensembl transcript workloads: zero impact (for transcripts without `*`) + +SPDI expressions using Ensembl transcript accessions (ENST*) produce +identical VRS Allele IDs on both backends. Confirmed by 25/25 checks +across BRCA1, EGFR, KRAS, BRAF, and RB1 transcripts +(`misc/refgetstore/ensembl_translator_equivalence.py`). + +### 7.3 Protein sequences with `*`: identifier divergence + +If a caller: +1. Resolves an ENSP accession containing `*` to a `ga4gh:SQ.*` digest +2. Uses that digest as a VRS identifier or passes it to another system + +...they will get different identifiers depending on which backend was used. +This affects 115 out of 123,845 Ensembl protein entries (0.09%). + +In practice, vrs-python's translator primarily works with genomic and +transcript-level variants, not protein sequences directly. The SPDI and +gnomAD translation paths use chromosome/transcript accessions, not ENSP. +HGVS protein expressions (`p.Val600Glu`) are resolved through UTA at +the transcript level, not via direct protein sequence lookup. + +## 8. Artifacts Produced + +| File | Purpose | +| --- | --- | +| `misc/refgetstore/diagnose_ensembl_mismatches.py` | 4-phase diagnostic script (Phase A-D) | +| `misc/refgetstore/diagnosis_report.json` | Full JSON report with all 116 mismatch records | +| `misc/refgetstore/ensembl_known_divergent.txt` | TSV of all 116 accessions with cause classification | +| `misc/refgetstore/ensembl_translator_equivalence.py` | SPDI-based Ensembl translator equivalence test | +| `misc/refgetstore/verify_store.py` section `[I]` | Regression test for known-divergent digests | +| `misc/refgetstore/README.md` | Version-drift policy and `*` divergence documentation | + +## 9. Verification Summary + +| Check | Result | +| --- | --- | +| `verify_store.py` (67 checks incl. new `[I]` section) | 67 pass, 0 fail | +| `compare_against_seqrepo.py` (74 canonical checks) | 74 pass, 0 fail | +| `test_dataproxy_refget.py` (29 unit tests) | 29 pass | +| `ensembl_translator_equivalence.py` (25 checks) | 25 pass, 0 fail | + +## 10. Conclusion + +The 116 "digest mismatches" break down as: + +- **115/116 (99.1%)**: Not a data issue. Caused by + `bioutils.sequences.normalize_sequence()` stripping `*` characters + before `seq_seqhash()` computes the SHA-512 digest. The actual + sequence bytes in both stores are identical. RefgetStore's digest is + GA4GH VRS-conformant; seqrepo's is a legacy artifact of bioutils + normalization. No fix needed — documented and regression-tested. + +- **1/116 (0.9%)**: Genuine sequence content difference for + ENST00000668831.1 across Ensembl releases (seqrepo loaded from ~2020, + refgetstore from r113 2024). Both stores are self-consistent. The + refgetstore version is authoritative for r113. + +The refgetstore is a safe drop-in replacement for seqrepo across all +DNA/transcript workloads. The `*` normalization divergence is limited to +protein sequences and does not affect any known vrs-python code path. diff --git a/2026-04-16T143407Z-star-normalization-digest-discrepancy.md b/2026-04-16T143407Z-star-normalization-digest-discrepancy.md new file mode 100644 index 00000000..a2d694e2 --- /dev/null +++ b/2026-04-16T143407Z-star-normalization-digest-discrepancy.md @@ -0,0 +1,378 @@ +# Stop Codon (`*`) Normalization Discrepancy in Protein Sequence Digests + +**Date**: 2026-04-16 +**Branch**: `kf/dataproxy-backend-refactor` + +## 1. Summary + +When loading Ensembl release 113 protein sequences into a gtars +RefgetStore, 115 ENSP accessions produce different `ga4gh:SQ.*` +identifiers than the same accessions in seqrepo 2024-12-20 — despite +the underlying sequence bytes being identical in both stores. + +The root cause is that these 115 proteins contain internal `*` (stop +codon / Ter) characters in their FASTA sequences, and the two systems +handle `*` differently when computing the sequence digest: + +- **SeqRepo** (via bioutils) strips `*` before hashing — conformant + with the RefGet spec's allowed alphabet (A-Z only) +- **gtars RefgetStore** hashes the raw FASTA bytes including `*` — + non-conformant with the RefGet spec for these sequences + +This discrepancy was invisible until we loaded Ensembl protein data +because NCBI RefSeq protein FASTAs never contain `*` characters. Zero +of the 197,939 RefSeq protein sequences have `*`, so the normalization +difference was never triggered. + +## 2. Which proteins are affected? + +All 115 are Ensembl-only protein annotations — none has a RefSeq (NP_/ +XP_) counterpart with the same sequence, even after stripping `*`. +These fall into a few gene families: + +| Category | Count | Description | +|---|---|---| +| IG D-segments (IGHD*) | 26 | Immunoglobulin diversity segments — short (5-12 aa) peptides from D-region CDS that inherently hit stop codons in translation | +| KIR receptors (KIR2DS4) | 25 | Killer cell immunoglobulin receptors — highly polymorphic immune locus with frameshift/stop alleles | +| Olfactory receptors (OR*) | 22 | Many OR genes are pseudogenes with in-frame stops | +| CASP12 | 9 | Caspase-12 — pseudogene in most human populations (premature stop) | +| CYP2D7 | 8 | Cytochrome P450 pseudogene | +| Other | 25 | MUC6, PKD1L2, POTEA, IFNL4, GBA3, GSTT2, AKR7L, MROH5, etc. | + +Ensembl annotates CDS regions for these genes even when translation +crosses stop codons. RefSeq's curation is more conservative: if a CDS +hits a stop, RefSeq either omits the protein record entirely, or picks +a different allele/isoform that avoids the stop. This is why the issue +surfaced only when we started loading Ensembl protein data. + +### Stop codon positions + +153 of the 158 `*`-containing Ensembl proteins have `*` at **internal** +positions (not at the end). Only 4 have terminal-only `*`, and 1 has +both. Examples from the FASTA: + +``` +>ENSP00000488168.1 gene_symbol:IGHD4-23 gene_biotype:IG_D_gene (non-functional) +*LRW*L ← 6 aa, * at positions 1 and 5 + +>ENSP00000488711.1 gene_symbol:IGHD3-22 gene_biotype:IG_D_gene +VLL***WLLL ← 10 aa, 3 consecutive stops at positions 4-6 + +>ENSP00000340011.8 gene_symbol:KIR2DS4 gene_biotype:protein_coding +...TEQ*TARILMNKTIRRCHTH ← 297 aa, * at positions 239, 269, 280 +``` + +The 115 mismatched proteins contain between 1 and 23 internal stop +codons: + +| `*` count | Proteins | +|---|---| +| 1 | 54 | +| 2 | 19 | +| 3 | 17 | +| 4-6 | 9 | +| 7-12 | 10 | +| 13-23 | 6 | + +## 3. How the digest is computed — seqrepo vs gtars + +### SeqRepo (bioutils) + +SeqRepo's `store` method imports `seq_seqhash` aliased as `sha512t24u`: + +```python +# biocommons/seqrepo/seqrepo.py, line 11 +from bioutils.digests import seq_seqhash as sha512t24u +``` + +When a sequence is stored (line ~148): + +```python +seqhash = sha512t24u(seq) # actually calls seq_seqhash(seq) +seq_id = seqhash +``` + +`seq_seqhash` normalizes before hashing: + +```python +# bioutils/digests.py — seq_seqhash +def seq_seqhash(seq, normalize=True): + seq = normalize_sequence(seq) if normalize else seq + return str(vmc_digest(seq, digest_size=24)) +``` + +And `normalize_sequence` strips `*`: + +```python +# bioutils/sequences.py — normalize_sequence +def normalize_sequence(seq): + """Removes whitespace and asterisks, and uppercases the string.""" + nseq = re.sub(r"[\s\*]", "", seq).upper() + m = re.search("[^A-Z]", nseq) + if m: + raise RuntimeError("Normalized sequence contains non-alphabetic characters") + return nseq +``` + +So the pipeline is: **strip `\s` and `*`** → **uppercase** → **reject +non-A-Z** → **SHA-512** → **truncate 24 bytes** → **base64url**. + +### gtars RefgetStore + +gtars is a Rust library with Python bindings. The `digest_sequence` +function (from `gtars/refget/__init__.pyi`, line 1663) documents: + +> "The input data is automatically uppercased to ensure consistent digest +> computation (matching FASTA processing behavior)." + +No mention of stripping `*` or any other characters. The Rust +implementation computes SHA-512 directly over the uppercased FASTA bytes, +including any `*` characters. + +When `add_sequence_collection_from_fasta` ingests a FASTA file, each +sequence record's `sha512t24u` field is computed from the raw +(uppercased) FASTA content. + +### Result + +For a protein like ENSP00000340011.8 (297 aa, `*` at positions +239, 269, 280): + +``` +seqrepo seq_id: fv4uQW4AfrwGj3dx9D4xfpccU7Qnqtia ← hash of 294 chars (3 * stripped) +gtars sha512t24u: 8Co4fICavUFj9qH_GGXd5cbWbzh8oSZ0 ← hash of 297 chars (raw) +``` + +For any DNA sequence or any protein without `*`, the two algorithms +produce identical output. + +## 4. What the RefGet specification says + +The RefGet spec (v1.0.1 / v2.0) defines sequence normalization for +checksum computation: + +> "When calculating the checksum for a sequence, all non-base symbols +> (\n, spaces, etc) must be removed and then uppercase the rest. The +> allowed alphabet for checksum calculation is uppercase ASCII +> (`0x41`-`0x5A` or `A-Z`)." + +The `*` character (ASCII `0x2A`) is not in the range `0x41`–`0x5A`. +By the allowed-alphabet clause, `*` is a non-base symbol that should +be removed before hashing. + +The VRS specification's type prefix table +([computed_identifiers.rst](https://github.com/ga4gh/vrs/blob/main/docs/source/conventions/computed_identifiers.rst)) +delegates `SQ.*` identifier computation to RefGet: + +``` +SQ, Sequence (RefGet) +``` + +By the letter of the RefGet spec, **bioutils/seqrepo is conformant** +(strips `*` before hashing) and **gtars is non-conformant** (hashes +`*` with the sequence). + +### Ambiguity + +The spec says "non-base symbols (\n, spaces, **etc**)" — the "etc" is +vague, and `*` is a legitimate IUPAC amino acid code (Ter, adopted in +HGVS nomenclature v2.0). The spec was designed for nucleotide sequences +and says of proteins only: "other sequences could be provided via the +same mechanism e.g. cDNA, CDS, mRNA or proteins." There is no +protein-specific normalization guidance. + +A reasonable case can be made either way: +- **Strip**: `*` is not in A-Z and the spec says A-Z only +- **Keep**: `*` is a legitimate IUPAC code encoding real positional + information, and the "non-base symbols" examples suggest formatting + characters (whitespace, line breaks), not sequence content + +### Collision risk from stripping + +Stripping `*` creates deliberate collisions: `"AB*CD"` and `"ABCD"` +produce the same digest. In practice this risk is negligible — these +proteins don't have `*`-free counterparts in any real database. None of +the 115 affected ENSPs shares a seqrepo `seq_id` with any NP_ or XP_ +accession (verified: 0/82 unique seq_ids overlap with NCBI namespace). + +## 5. Why this was invisible before Ensembl + +NCBI RefSeq protein FASTAs (`human.*.protein.faa.gz`) contain zero `*` +characters across all 197,939 sequences in all 16 shards: + +``` +RefSeq protein shards (16 files): 197,939 sequences, 0 lines with * +Ensembl pep FASTA (1 file): 123,845 sequences, 158 with * (115 in seqrepo) +``` + +NCBI's convention: if translation hits a stop codon, no protein FASTA is +generated. Ensembl's convention: annotate the CDS anyway and emit `*` in +the protein FASTA. Since the `*`-stripping code path in bioutils only +changes the digest when `*` is actually present, the entire RefSeq +protein namespace (and all DNA sequences) produce identical digests in +both systems. + +## 6. Ensembl source files for reproduction + +The Ensembl protein FASTA that triggers this discrepancy is: + +``` +Homo_sapiens.GRCh38.pep.all.fa.gz +``` + +Downloaded from: `ftp://ftp.ensembl.org/pub/release-113/fasta/homo_sapiens/pep/Homo_sapiens.GRCh38.pep.all.fa.gz` + +This is the only file needed to reproduce the issue in a minimal test. +It is configured as a `[[seqset]]` entry in `misc/refgetstore/assemblies.toml`. + +To reproduce, load this single FASTA into both a seqrepo instance and a +gtars RefgetStore, then compare the digests for the accessions listed +below. + +## 7. Full list of affected accessions + +116 total: 115 `star_normalization` (identical bytes, different digests) ++ 1 `sequence_divergence` (different bytes across Ensembl releases). + +| Accession | Length | Stars | Gene | Biotype | Cause | SeqRepo seq_id | gtars sha512t24u | +|---|---|---|---|---|---|---|---| +| ENSP00000340011.8 | 297 | 3 | KIR2DS4 | protein_coding | star_normalization | fv4uQW4AfrwGj3dx9D4xfpccU7Qnqtia | 8Co4fICavUFj9qH_GGXd5cbWbzh8oSZ0 | +| ENSP00000375609.1 | 225 | 1 | KIR2DS4 | protein_coding | star_normalization | 0wFdmvBugnimf4tOFjCVdckpw3P6GmOx | Qg7FS7qyO-pfHp7HYd4wvr26DEx1oov4 | +| ENSP00000385765.6 | 230 | 1 | GSTT2 | protein_coding | star_normalization | lV7Y-VqQL6dNPJ2ZKWTqw46p_da8Eodm | YL07fWvAjY43qO_0ZnVw5BxCvo_M_0x- | +| ENSP00000402491.1 | 205 | 1 | AKR7L | protein_coding | star_normalization | e6sQaY-e_N_HsVyuY3ymiFysYBMBFk9W | dBxeQN4GInX8eFSsicpYD8N5Nn_0My2g | +| ENSP00000411913.1 | 331 | 1 | AKR7L | protein_coding | star_normalization | _RbUr7Oq9qE22xU9fnMC2uLA8RIp2Hri | TQ93TCrFm5e-mb-tkSHz3d_DYaAlo4i3 | +| ENSP00000421815.1 | 226 | 1 | CASP12 | protein_coding | star_normalization | tM9WHP2MFnlP7TaVWy-zF3pASsoLzDdU | PuFuh2fJaeBhwf_CIXb9c4n-5HHCkIA8 | +| ENSP00000422220.1 | 162 | 1 | GBA3 | protein_coding | star_normalization | o6IopMWmnKAE7FUNdEFy2TnsyBDa-swp | 2ludFB9wsyqzj2rD-ba2o2sFeXQIa277 | +| ENSP00000423754.1 | 298 | 1 | GBA3 | protein_coding | star_normalization | K56fM_BMAirMsSULhhLEc2yBkgvEE_aZ | kw3yUvbNK6RIftnKndZqroj4_OZxeNl9 | +| ENSP00000423899.1 | 224 | 1 | CASP12 | protein_coding | star_normalization | RDSLXXiECCksD3fd4HEjkDRDUojB3zm1 | D9gmmANCDpy0XX2JYSF7ZeHkXVaaDAm6 | +| ENSP00000423970.1 | 295 | 1 | CASP12 | protein_coding | star_normalization | fOlJtYkBTNNq6H0kRnKsGbSqtO5PcFEr | 96AGBZz3OkRXlz54m0a5vf04XVfn8MG0 | +| ENSP00000424038.1 | 341 | 1 | CASP12 | protein_coding | star_normalization | lOkFBmBx4LkR4GtP4ljxF4ssKcMpU9S2 | sNGrVi9jJokpvjhGaAkw9-5mCjrkxR5s | +| ENSP00000425652.1 | 277 | 1 | CASP12 | protein_coding | star_normalization | 0UyEhk9QGqb2gKNUMpHZ93nXVAXK9yTO | JQU8ZqYvYRuVO19ZDaHQhHM89PyidybZ | +| ENSP00000426427.1 | 140 | 1 | CASP12 | protein_coding | star_normalization | 03nXBcwbbJQPYE_5LurPSoCpnsikUdEn | pO53MQMJFDxQT6bOb7ey9est-55eH5NH | +| ENSP00000426566.1 | 257 | 1 | CASP12 | protein_coding | star_normalization | _nkNH8COTskYyNkCoTzIRNHzENVylj1y | egHuRL5C7VrWvOx3FvqbEayoBNIeC6ZM | +| ENSP00000426627.1 | 323 | 1 | FCGR2C | protein_coding | star_normalization | OpwKkqOENTdevZ_CrLEOVumim2QTctaK | 2KlIfbXxNKZoNImdwrmKBdRYXPkKL_2G | +| ENSP00000427437.1 | 142 | 1 | CASP12 | protein_coding | star_normalization | wWUGUKB2gnrfgmMm0qg34Iy9mUEhx4ju | 7KGhHMvva3lM9749_TxBRCl5Oz8YcdKo | +| ENSP00000427458.1 | 469 | 1 | GBA3 | protein_coding | star_normalization | rFjG0Bt4bfEH09x5PaR9jZGd03wmgE6O | -x9lIom6XRVzSbgQDyNb0hrandBbttoV | +| ENSP00000427969.1 | 10 | 2 | IGHD2-15 | IG_D_gene | star_normalization | y9mM4wkXZleVNSyYRPm83MhQd47vpV0A | Ah8IodVoXqfHk4DDt8jsFGSic0fRZYU0 | +| ENSP00000428366.1 | 12 | 1 | IGHD3-16 | IG_D_gene | star_normalization | SHyk_qyvZUk1fYzNdSTs6Drv8jS87r_C | CN6dOnzsnNZiifsD2mNJ941p8KVRVPJu | +| ENSP00000428393.1 | 5 | 2 | IGHD4-4 | IG_D_gene | star_normalization | qRfoOZ40q93mvxVq3MDnu2rMZt2mB_ob | yAZ-3aLns9fo2SxCNtvZJGjw_9-p67nz | +| ENSP00000428616.1 | 10 | 1 | IGHD2-8 | IG_D_gene | star_normalization | KRLK6ZBMvRpTakgKIIWxTzvHu6nZtEKQ | 5ZDMzPF5f4u5xiuBDA2ByfCYoG4534KM | +| ENSP00000429324.1 | 9 | 1 | IGHD2-21 | IG_D_gene | star_normalization | m19OoHXspP1j74FYLLR0Fon_Tca6IwAP | fn4oRtmkRm5ikN4QPXXg0RdDOzaLpvUH | +| ENSP00000429952.1 | 10 | 3 | IGHD3-22 | IG_D_gene | star_normalization | 5Ek4xsz2ZS7GfXgv_ZBjOH5FkAP3XMX6 | JCCHtapvgLCCSu0eIJCK34DXB_mOSDrc | +| ENSP00000430034.1 | 5 | 2 | IGHD4-11 | IG_D_gene | star_normalization | qRfoOZ40q93mvxVq3MDnu2rMZt2mB_ob | yAZ-3aLns9fo2SxCNtvZJGjw_9-p67nz | +| ENSP00000430248.1 | 6 | 2 | IGHD4-23 | IG_D_gene | star_normalization | AAWZIb3sgZG4DGzYKPTEdsDRPppA20nl | ZyuIuWLGyac0hNqMfiw0XIJIjTx1UF2Y | +| ENSP00000430788.1 | 10 | 2 | IGHD2-2 | IG_D_gene | star_normalization | 4nJC7z8ekuHU6kAl4yMiV6qZlrpomr_6 | vQGG6MQqX_mw1KxS82QU0Kuk5KtUItWA | +| ENSP00000431031.1 | 1318 | 1 | MROH5 | protein_coding | star_normalization | l4SXGqAbFUh7r1jc-ImcNEdp067Fa0MY | zkX9kDvYwL768RXctEHW7-3lneBHc71d | +| ENSP00000431089.1 | 5 | 2 | IGHD4-17 | IG_D_gene | star_normalization | BM1Hbmp96w4OSkWZ5u_ROCq4tWMf_V69 | hzlOSDnnFBtjxI-qUTYSXVvP8K76MfHk | +| ENSP00000434417.1 | 2459 | 17 | PKD1L2 | protein_coding | star_normalization | zKGzcqTwIi1cpsh_25jTR-g7cycsIEpf | Ih9FQHBcIpxLvdvSmQwf1B_uW07yCbjE | +| ENSP00000434644.1 | 1774 | 17 | PKD1L2 | protein_coding | star_normalization | DZRDVc8gAxzoOkjyLBLK3IcVRKKK46Pw | v8czIuEX2XTC0M9Dw7Ax3fzb4H8qdgOL | +| ENSP00000438013.1 | 314 | 4 | OR4Q2 | protein_coding | star_normalization | nDmCYyT1EKuf9Z8mrnEgCxgF2Hjn0T_P | _nPal56b66hTKeKlOajKFSKjKY8DcuU- | +| ENSP00000439604.1 | 516 | 11 | CYP2D7 | protein_coding | star_normalization | oaVc-acao8fOMIyVp_H98dcz0BSDIUWH | aYF_Y_ycp3u3t9Gqeu96GLzc304fQ0zp | +| ENSP00000445124.1 | 497 | 11 | CYP2D7 | protein_coding | star_normalization | 6VmdTL7nqa7JQpolg-wuBLGwIKBnrP9F | MIwe_aAViS3zDJuv3mQTnIDVQMSLST8L | +| ENSP00000451021.2 | 314 | 1 | OR11H7 | protein_coding | star_normalization | 38LKEEE-120Kwsih-vNlwnx8tb2A4ZSx | _rwlFqSN0jljdGdyuRQXsthR-SSMBJ2G | +| ENSP00000452150.1 | 21 | 1 | TRAJ58 | TR_J_gene | star_normalization | ynPhtGE3u23nkLJNMxVv51pviMYUrBBX | Ede4Q4UXqG9Dzi4MrnjeH9jp9k0ggu7T | +| ENSP00000461363.1 | 517 | 11 | CYP2D7 | protein_coding | star_normalization | O0rR6w_qtUdlVU360MFqtPEbi7ZFraqF | KInA0nFf_sQ_yQXjq92_k4rkC72kfgDG | +| ENSP00000461768.1 | 498 | 11 | CYP2D7 | protein_coding | star_normalization | SBAWxYVz_UK9DZodtRFfbK1-2l5-uvL2 | 6UVE7AZ23iHnnV0dDaNAFu_wGKHPYYoI | +| ENSP00000463502.4 | 469 | 1 | PNLIPRP2 | protein_coding | star_normalization | syXOnAwOvxwpfLO67Z7c84KArOxWUGX- | zWPI0igbczAoDoC3RNroPzMKmMmNOe6I | +| ENSP00000474017.2 | 10 | 2 | IGHD2OR15-2B | IG_D_gene | star_normalization | PkCr3cRIsERXzmDHBNwJKKU2AhW1pcjS | a8CMClpZG-W05aaW-ZTFsdZ8WUQnX6t4 | +| ENSP00000474065.2 | 10 | 2 | IGHD2OR15-2A | IG_D_gene | star_normalization | PkCr3cRIsERXzmDHBNwJKKU2AhW1pcjS | a8CMClpZG-W05aaW-ZTFsdZ8WUQnX6t4 | +| ENSP00000474133.2 | 10 | 1 | IGHD3OR15-3A | IG_D_gene | star_normalization | U7neqpeNT4iNk-l3yhxlfzeh7imlPahR | Qflup9oKLA9_ebzhyw1VwhdIcInZGnb_ | +| ENSP00000474573.2 | 10 | 1 | IGHD3OR15-3B | IG_D_gene | star_normalization | U7neqpeNT4iNk-l3yhxlfzeh7imlPahR | Qflup9oKLA9_ebzhyw1VwhdIcInZGnb_ | +| ENSP00000474693.2 | 6 | 2 | IGHD4OR15-4A | IG_D_gene | star_normalization | FfJUWv8y8zqeEkxc8WC0CBxAVBpHKby_ | SMi7IfQFm46hxfH46aUlNOJd_mTj6CYC | +| ENSP00000475053.2 | 6 | 2 | IGHD4OR15-4B | IG_D_gene | star_normalization | FfJUWv8y8zqeEkxc8WC0CBxAVBpHKby_ | SMi7IfQFm46hxfH46aUlNOJd_mTj6CYC | +| ENSP00000475160.1 | 308 | 9 | OR5AC1 | protein_coding | star_normalization | FOZR2eNkiqv0vQKW28sYDoFUk-pHLf7q | GO28PhtqjbkgqI_ZXen46QVdAUtQsTy0 | +| ENSP00000475351.1 | 315 | 8 | OR12D1 | protein_coding | star_normalization | NaBv12DVCEzFhBS1k7fnmH1RvEOocOQZ | JcuCvOWNKvugDHiBlc5QNxiD9c1xGoHK | +| ENSP00000475611.1 | 311 | 6 | OR10J4 | protein_coding | star_normalization | CafOAbOr7B4OwynxW2mGBs93mmWzlLpj | so4CHY-xDA2Kza_d9Sq8Cz9E96yZMrL8 | +| ENSP00000476186.1 | 310 | 2 | OR5H8 | protein_coding | star_normalization | GQpNKfcojbpwHYXfeaYzPUelZhAsVouD | Mg_qtBntkSHm5xPGBWSLyzHBXF1Ua1Oi | +| ENSP00000476380.1 | 326 | 1 | OR10AC1 | protein_coding | star_normalization | 546P9zkeSm9jcCq9v5rNou3-U-9wOXa0 | 9gXvvtU-EONZkDaTQZTRkqqAeX9YbCjD | +| ENSP00000476467.1 | 315 | 1 | OR4A8 | protein_coding | star_normalization | HV7AX9iHZz2qgacGsUkeLl9JY1wQSy7b | 91F6s9En0f8Tp5H18TUHjtFIhnFA79lA | +| ENSP00000476576.1 | 308 | 3 | OR52E1 | protein_coding | star_normalization | MHCBdS-C2GvAKyaWy5g7YpM5Tl6S7r2P | exEjrMOOllerXl47soSNdLchtvUnq-n4 | +| ENSP00000477333.1 | 319 | 12 | OR13C7 | protein_coding | star_normalization | 81G0gd-_pNefpEX9qCl6Xo3SfpL_GtM_ | jBGqsI0x-vXuNDpnaGRFodo1JLeIrtqk | +| ENSP00000477337.2 | 314 | 16 | OR5G3 | protein_coding | star_normalization | Hgx6zf3aYxje2c12BUxwnW3Wfwcz72-E | sfkGeZ9IhJISVjv1K1tsfeX8obIqTnf0 | +| ENSP00000477393.1 | 328 | 10 | OR5AL1 | protein_coding | star_normalization | -a7aCjSuO0y-Noz6SjHENVXwfOXx2lgl | C8md75ie9b7ZYfCXTAlyTnIUF4YkeQkn | +| ENSP00000478383.1 | 297 | 3 | KIR2DS4 | protein_coding | star_normalization | se_guVGDbGUwQ_VwGptcBPiTSUEMOTDA | IlMIcqDnTyx9LM1oe18o-6-Q6QWvov_6 | +| ENSP00000478656.2 | 517 | 11 | CYP2D7 | protein_coding | star_normalization | 8Jwk52z5pfAcprdAXS7iQN1kTNxxio40 | EbKxx4ZLK27lrDbLCoPaQMFIuY1_aMQs | +| ENSP00000478998.1 | 297 | 3 | KIR2DS4 | protein_coding | star_normalization | Ot3JN_E9VBadQ46vgolO3JMynJDqSGHa | f01vgyglFX1ju7emFuoB3OQn6mBWjCR6 | +| ENSP00000479129.1 | 225 | 1 | KIR2DS4 | protein_coding | star_normalization | PVMp-pPMZHSdjxIHbf9BPZFTVliQPwt8 | TiIdLvCGnMqH2aeTyjlo9IlWOwojBX4g | +| ENSP00000479356.1 | 297 | 3 | KIR2DS4 | protein_coding | star_normalization | se_guVGDbGUwQ_VwGptcBPiTSUEMOTDA | IlMIcqDnTyx9LM1oe18o-6-Q6QWvov_6 | +| ENSP00000479369.1 | 297 | 3 | KIR2DS4 | protein_coding | star_normalization | fv4uQW4AfrwGj3dx9D4xfpccU7Qnqtia | 8Co4fICavUFj9qH_GGXd5cbWbzh8oSZ0 | +| ENSP00000479404.1 | 374 | 4 | KIR2DS4 | protein_coding | star_normalization | t3g70NPf5GGSyB223lAwFe6zw1ChtmEy | F_y21mXr_NuyB93ooYdRBy0kd-PXP6tz | +| ENSP00000481139.1 | 297 | 3 | KIR2DS4 | protein_coding | star_normalization | 6y4nN8mld_LcvD-EbegjezG4J0ipyLq1 | GIq_KqgXV64sQO3gTLidkdIOtqkmWH7A | +| ENSP00000481279.1 | 374 | 4 | KIR2DS4 | protein_coding | star_normalization | XwjGxQ5MlPAw2PL52E6hJdBXJE3Z-Q32 | cUaEhxJnZ5QAuY3JG0Bn6bcW9MwKq73n | +| ENSP00000481428.1 | 297 | 3 | KIR2DS4 | protein_coding | star_normalization | fv4uQW4AfrwGj3dx9D4xfpccU7Qnqtia | 8Co4fICavUFj9qH_GGXd5cbWbzh8oSZ0 | +| ENSP00000482055.2 | 297 | 3 | KIR2DS4 | protein_coding | star_normalization | se_guVGDbGUwQ_VwGptcBPiTSUEMOTDA | IlMIcqDnTyx9LM1oe18o-6-Q6QWvov_6 | +| ENSP00000482745.1 | 341 | 1 | CASP12 | protein_coding | star_normalization | lOkFBmBx4LkR4GtP4ljxF4ssKcMpU9S2 | sNGrVi9jJokpvjhGaAkw9-5mCjrkxR5s | +| ENSP00000483571.1 | 225 | 1 | KIR2DS4 | protein_coding | star_normalization | lv5JmDOztbvJgKGluw0Fh6jFTb0LyBFT | mNBJqEx1npRp6Qy7ofSZAYW4iX81-b3m | +| ENSP00000483620.1 | 297 | 3 | KIR2DS4 | protein_coding | star_normalization | fv4uQW4AfrwGj3dx9D4xfpccU7Qnqtia | 8Co4fICavUFj9qH_GGXd5cbWbzh8oSZ0 | +| ENSP00000485881.1 | 311 | 15 | OR5G3 | protein_coding | star_normalization | 5CcawTtWZa2Bc5otcdnSdzOQV1aMT_cq | N3f7prhPB-ztQHfhGD48lojOdU42AWwG | +| ENSP00000485975.1 | 315 | 1 | OR8J2 | protein_coding | star_normalization | AlKSb7ywKLef2uY_oqJ97OqTeZ1Cdb1D | SzZa7T9y81S_V8iQp9RnpbnKshXX9IHp | +| ENSP00000486295.1 | 4040 | 11 | MUC6 | protein_coding | star_normalization | iyPvjohGDgVYeFEVYQ_n6DFS5sOkwUTI | VJMwcSEKzmAksrvE276Okup2ueBujczb | +| ENSP00000487044.1 | 498 | 11 | CYP2D7 | protein_coding | star_normalization | HeRZbi709aJqVjELYnnZZJrRUW17WjWH | mdGiXxUvar3WuR1h-zsoryFcpLHQgi0H | +| ENSP00000487059.1 | 4714 | 23 | MUC6 | protein_coding | star_normalization | NpQlnZ0jUAHvbmGF37prMDgMPv3THHYN | 2Vgy0wYjqGFzUpqGWPDJipU0XNdrKU8a | +| ENSP00000487596.1 | 493 | 1 | MROH5 | protein_coding | star_normalization | _IPtIdLGP1_WEK4al_dKDId2NigEHXZn | bZJHWz7sSJQUC00fzSqS0wgZMcQ4X1i4 | +| ENSP00000487599.1 | 9 | 1 | IGHD2-21 | IG_D_gene | star_normalization | m19OoHXspP1j74FYLLR0Fon_Tca6IwAP | fn4oRtmkRm5ikN4QPXXg0RdDOzaLpvUH | +| ENSP00000487604.1 | 10 | 2 | IGHD2-2 | IG_D_gene | star_normalization | 4nJC7z8ekuHU6kAl4yMiV6qZlrpomr_6 | vQGG6MQqX_mw1KxS82QU0Kuk5KtUItWA | +| ENSP00000487787.1 | 12 | 1 | IGHD3-16 | IG_D_gene | star_normalization | SHyk_qyvZUk1fYzNdSTs6Drv8jS87r_C | CN6dOnzsnNZiifsD2mNJ941p8KVRVPJu | +| ENSP00000487993.1 | 10 | 2 | IGHD2-15 | IG_D_gene | star_normalization | y9mM4wkXZleVNSyYRPm83MhQd47vpV0A | Ah8IodVoXqfHk4DDt8jsFGSic0fRZYU0 | +| ENSP00000488083.1 | 10 | 1 | IGHD2-8 | IG_D_gene | star_normalization | KRLK6ZBMvRpTakgKIIWxTzvHu6nZtEKQ | 5ZDMzPF5f4u5xiuBDA2ByfCYoG4534KM | +| ENSP00000488168.1 | 6 | 2 | IGHD4-23 | IG_D_gene | star_normalization | AAWZIb3sgZG4DGzYKPTEdsDRPppA20nl | ZyuIuWLGyac0hNqMfiw0XIJIjTx1UF2Y | +| ENSP00000488261.1 | 5 | 2 | IGHD4-17 | IG_D_gene | star_normalization | BM1Hbmp96w4OSkWZ5u_ROCq4tWMf_V69 | hzlOSDnnFBtjxI-qUTYSXVvP8K76MfHk | +| ENSP00000488314.1 | 153 | 1 | UBE2NL | protein_coding | star_normalization | Zbnpiff-E255gCpMrQGZejm0OauEumnc | 4yrFdP8maHkqUKDKBSVQ-gIrkehiwHxr | +| ENSP00000488711.1 | 10 | 3 | IGHD3-22 | IG_D_gene | star_normalization | 5Ek4xsz2ZS7GfXgv_ZBjOH5FkAP3XMX6 | JCCHtapvgLCCSu0eIJCK34DXB_mOSDrc | +| ENSP00000488735.1 | 5 | 2 | IGHD4-11 | IG_D_gene | star_normalization | qRfoOZ40q93mvxVq3MDnu2rMZt2mB_ob | yAZ-3aLns9fo2SxCNtvZJGjw_9-p67nz | +| ENSP00000488889.1 | 5 | 2 | IGHD4-4 | IG_D_gene | star_normalization | qRfoOZ40q93mvxVq3MDnu2rMZt2mB_ob | yAZ-3aLns9fo2SxCNtvZJGjw_9-p67nz | +| ENSP00000488911.1 | 497 | 12 | CYP2D7 | protein_coding | star_normalization | h9XYkJfTPZZjGI3uq_VgHAxQfGOnusMx | Lim-8--cZivPCz0LQqISF0tUGv2hrbrX | +| ENSP00000488993.1 | 244 | 1 | GSTT2 | protein_coding | star_normalization | Pu86r-b0I3U_dqkK9zQ2OPGSshhhhHWG | DnlfpzUcYbk15_dKw1jqPVCwqxRENcFT | +| ENSP00000489014.1 | 516 | 12 | CYP2D7 | protein_coding | star_normalization | -fkYMovNew_mTufEcBSEn80kG2rr4OB_ | ZurDzxk4uhTHtf8xZAPlkm2VY4kZwv5E | +| ENSP00000489240.1 | 108 | 1 | IFNL4 | protein_coding | star_normalization | aPj392cUfAWEeDcwOZLZafkPAwlUuscd | IoKaC3TiSvxPlN1u_b9QaSEGIr9uavgu | +| ENSP00000489559.1 | 132 | 1 | IFNL4 | protein_coding | star_normalization | 7Dqau58VfarSEwoAyKydRrGg8qHMU6sN | jdWrXmbGQ39RS_FxE9AxivWfxAjk0GgM | +| ENSP00000490976.1 | 297 | 3 | KIR2DS4 | protein_coding | star_normalization | Ot3JN_E9VBadQ46vgolO3JMynJDqSGHa | f01vgyglFX1ju7emFuoB3OQn6mBWjCR6 | +| ENSP00000491125.1 | 357 | 1 | VN1R5 | protein_coding | star_normalization | tRRBvR9lKL2Pqn0LDyPFWX9QzwN6R9cI | yJ1li88C4ZhzDqpmIMuJhDlayDIu2i15 | +| ENSP00000491133.1 | 306 | 1 | OR4C45 | protein_coding | star_normalization | 84ZbQcviHt30Ks_HfJnC25WT1H3Ll_aG | UPwfn83YcGH13BQeW32xlO0MSD6SS24V | +| ENSP00000491451.1 | 297 | 3 | KIR2DS4 | protein_coding | star_normalization | se_guVGDbGUwQ_VwGptcBPiTSUEMOTDA | IlMIcqDnTyx9LM1oe18o-6-Q6QWvov_6 | +| ENSP00000491543.1 | 225 | 1 | KIR2DS4 | protein_coding | star_normalization | Ipbqtow-myKtzhLMr1FiU01rBRs7Iwv1 | w6SG0sotavd93SmwWs3HxDb31icvb43V | +| ENSP00000491974.1 | 297 | 3 | KIR2DS4 | protein_coding | star_normalization | fv4uQW4AfrwGj3dx9D4xfpccU7Qnqtia | 8Co4fICavUFj9qH_GGXd5cbWbzh8oSZ0 | +| ENSP00000491994.1 | 225 | 1 | KIR2DS4 | protein_coding | star_normalization | 0wFdmvBugnimf4tOFjCVdckpw3P6GmOx | Qg7FS7qyO-pfHp7HYd4wvr26DEx1oov4 | +| ENSP00000492028.2 | 297 | 3 | KIR2DS4 | protein_coding | star_normalization | fv4uQW4AfrwGj3dx9D4xfpccU7Qnqtia | 8Co4fICavUFj9qH_GGXd5cbWbzh8oSZ0 | +| ENSP00000492090.1 | 225 | 1 | KIR2DS4 | protein_coding | star_normalization | Ipbqtow-myKtzhLMr1FiU01rBRs7Iwv1 | w6SG0sotavd93SmwWs3HxDb31icvb43V | +| ENSP00000492193.1 | 688 | 1 | POTEA | protein_coding | star_normalization | 7Pr4ISih0_g-We99919kiPT8Ft1voOjU | QKLfHnF-uMJbWpYlzxPWivnK9FISFaHv | +| ENSP00000492245.1 | 225 | 1 | KIR2DS4 | protein_coding | star_normalization | Ipbqtow-myKtzhLMr1FiU01rBRs7Iwv1 | w6SG0sotavd93SmwWs3HxDb31icvb43V | +| ENSP00000492265.1 | 642 | 1 | POTEA | protein_coding | star_normalization | NdYD2MxngnXGXKck56SC5Y2pwnSRxQeN | gZoUTjOaMrKyFO7XcjzV7e0F6hHw0Y2I | +| ENSP00000492337.2 | 225 | 1 | KIR2DS4 | protein_coding | star_normalization | 0wFdmvBugnimf4tOFjCVdckpw3P6GmOx | Qg7FS7qyO-pfHp7HYd4wvr26DEx1oov4 | +| ENSP00000492458.1 | 225 | 1 | KIR2DS4 | protein_coding | star_normalization | PVMp-pPMZHSdjxIHbf9BPZFTVliQPwt8 | TiIdLvCGnMqH2aeTyjlo9IlWOwojBX4g | +| ENSP00000492627.1 | 297 | 3 | KIR2DS4 | protein_coding | star_normalization | se_guVGDbGUwQ_VwGptcBPiTSUEMOTDA | IlMIcqDnTyx9LM1oe18o-6-Q6QWvov_6 | +| ENSP00000493090.1 | 331 | 2 | OR1P1 | protein_coding | star_normalization | OwOpiUeTZxOiRyJpUVgFPEknheNaytQk | 7gOPz5rjQBMvXPqGeF8kBFb1MRWsv-XP | +| ENSP00000493099.2 | 105 | 1 | SCYGR10 | protein_coding | star_normalization | lcX6lIyYgyvgNIcxL8Nf_9c2te0uWR7J | bX-m0VrO77yaMLzVcz__B14CYNhMhNS8 | +| ENSP00000493221.1 | 317 | 6 | OR2T7 | protein_coding | star_normalization | YhrPozOQOjywrkWpQWPgu49GdiZqFZk8 | vax-AZkewD5SpDAT5szm8meLHgf2beXC | +| ENSP00000493243.1 | 317 | 6 | OR2T7 | protein_coding | star_normalization | YhrPozOQOjywrkWpQWPgu49GdiZqFZk8 | vax-AZkewD5SpDAT5szm8meLHgf2beXC | +| ENSP00000493291.1 | 315 | 7 | OR4K3 | protein_coding | star_normalization | _CbilrTwIJnxQ8UDDZ3ieqBW5Jz6IlVd | 8GVt4hPQBAH0n47yoEBplEk176LMj6fZ | +| ENSP00000493317.1 | 315 | 1 | OR8J2 | protein_coding | star_normalization | AlKSb7ywKLef2uY_oqJ97OqTeZ1Cdb1D | SzZa7T9y81S_V8iQp9RnpbnKshXX9IHp | +| ENSP00000493452.1 | 308 | 1 | OR9H1P | protein_coding | star_normalization | mICKQK_3Sn5lQNys1NRoohE3GrKeB5eh | HO0I-zZFjRZAJMqZ7io8-u7lxa3y9P6_ | +| ENSP00000495205.1 | 954 | 1 | PTCHD3 | protein_coding | star_normalization | Nmuo8TKv6WCrdaDDfamxvpgrA26KLpLS | qR9P0vEzxBhBS34XQHLkcbx9GCG08qoa | +| ENSP00000496497.1 | 326 | 1 | OR10AC1 | protein_coding | star_normalization | 546P9zkeSm9jcCq9v5rNou3-U-9wOXa0 | 9gXvvtU-EONZkDaTQZTRkqqAeX9YbCjD | +| ENSP00000498832.1 | 232 | 2 | FAM246C | protein_coding | star_normalization | PZVNRQLqkqRCjj4klpDJmfxJqLHzCSyO | Gr7F6_lh_SAfec-UJ4tgY0vLz7oag0LV | +| ENST00000668831.1 | 1120 | 0 | | | sequence_divergence | NllKO15Wb60jTlfVHY5YOFfN3WFZyn5M | 2EdYgXlBf5xGHlhCifMYtWt9QZPGOyxA | + +## 8. References + +### Specifications + +- [RefGet Sequences v2.0](https://ga4gh.github.io/refget/sequences/) — "allowed alphabet for checksum calculation is uppercase ASCII (0x41-0x5A or A-Z)" +- [RefGet v1.0.1 (hts-specs)](https://samtools.github.io/hts-specs/refget.html) — "all non-base symbols (\n, spaces, etc) must be removed" +- [RefGet spec source (GitHub)](https://github.com/samtools/hts-specs/blob/master/refget.md) +- [VRS Computed Identifiers](https://github.com/ga4gh/vrs/blob/main/docs/source/conventions/computed_identifiers.rst) — SQ type prefix delegates to RefGet +- [HGVS amino acid nomenclature](https://www.hgvs.org/mutnomen/references.html) — `*` / Ter as IUPAC code for stop + +### Source code + +- `bioutils.sequences.normalize_sequence` — `.venv/.../bioutils/sequences.py` — `re.sub(r"[\s\*]", "", seq).upper()` +- `bioutils.digests.seq_seqhash` — `.venv/.../bioutils/digests.py` — calls `normalize_sequence` then `vmc_digest` +- `biocommons.seqrepo.seqrepo.SeqRepo` — `.venv/.../biocommons/seqrepo/seqrepo.py` — line 11: `from bioutils.digests import seq_seqhash as sha512t24u`; line 148: `seqhash = sha512t24u(seq)` (actually `seq_seqhash`) +- `gtars.refget.digest_sequence` — Rust implementation, Python stub at `.venv/.../gtars/refget/__init__.pyi:1663` — "input data is automatically uppercased" (no `*` stripping) +- `gtars.refget.sha512t24u_digest` — `.venv/.../gtars/refget/__init__.pyi:1544` — raw hash, no normalization + +### Diagnostic artifacts + +- `misc/refgetstore/diagnose_ensembl_mismatches.py` — 4-phase diagnostic script +- `misc/refgetstore/diagnosis_report.json` — full JSON report +- `misc/refgetstore/ensembl_known_divergent.txt` — TSV of affected accessions diff --git a/README.md b/README.md index 4a7af41b..dfc78d83 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ - Translating from and to other variant formats - [Annotate VCFs with VRS](https://github.com/ga4gh/vrs-python/blob/main/docs/extras/vcf_annotator.md) - Convert GA4GH objects between inlined and referenced forms +- [Pluggable sequence/alias data backends (SeqRepo, refget)](https://github.com/ga4gh/vrs-python/blob/main/docs/dataproxy_backends.md) ## Known Issues @@ -231,12 +232,31 @@ Here are some things to try. curl 'http://127.0.0.1:[your_port_number]/seqrepo/1/sequence/refseq:NM_000059.4?end=20' ``` - Set the `GA4GH_VRS_DATAPROXY_URI` environment variable to point to - this UL: + this URL: ```shell - $ export GA4GH_VRS_DATAPROXY_URI=http://localhost:[your_port_number]/seqrepo + $ export GA4GH_VRS_DATAPROXY_URI=seqrepo+http://localhost:[your_port_number]/seqrepo $ export SEQREPO_URI=http://localhost:[your_port_number] ``` +Remote refget stores are also supported when exposed as a static HTTP(S) URL tree, +such as an S3/R2 bucket endpoint or CDN origin. Set the dataproxy URI to the remote +store and optionally override the local cache root: + +```shell +export GA4GH_VRS_DATAPROXY_URI=refget+https://example-bucket.s3.amazonaws.com/refget-store +export GA4GH_VRS_REFGET_CACHE_DIR=/path/to/refget-cache-root +``` + +If `GA4GH_VRS_REFGET_CACHE_DIR` is unset, vrs-python falls back to an XDG-style +cache root under `XDG_CACHE_HOME` or `~/.cache`. Each remote refget store gets +its own deterministic cache subdirectory under that root. + +In-process lookup caches can be sized with `GA4GH_VRS_DATAPROXY_CACHE_SIZE` +(lengths, digests, and aliases; default 8192) and +`GA4GH_VRS_REST_DATAPROXY_SEQUENCE_CACHE_SIZE` (REST sequence payloads; default +128). See [DataProxy backends](docs/dataproxy_backends.md) for the full list of +backends, URI schemes, and caching behavior. + ## VRS-Python and VRS Version Correspondence The ga4gh/vrs-python repo embeds the ga4gh/vrs repo as a git submodule for testing purposes. diff --git a/docs/dataproxy_backends.md b/docs/dataproxy_backends.md new file mode 100644 index 00000000..57e46862 --- /dev/null +++ b/docs/dataproxy_backends.md @@ -0,0 +1,145 @@ +# DataProxy backends + +VRS-Python accesses reference sequences and resolves cross-namespace sequence +identifiers (e.g. `refseq:NM_000551.3` → `ga4gh:SQ.…`) through a **DataProxy**. +A DataProxy combines two responsibilities: + +- **Sequence access** — `get_sequence()`, `get_sequence_length()` +- **Alias / identifier resolution** — `get_aliases()`, + `translate_sequence_identifier()`, `derive_refget_accession()` + +You construct one with `create_dataproxy(uri)`, where the URI scheme selects the +backend: + +```python +from ga4gh.vrs.dataproxy import create_dataproxy + +dp = create_dataproxy("seqrepo+file:///usr/local/share/seqrepo/2024-12-20") +``` + +If `uri` is omitted, `create_dataproxy` falls back to the +`GA4GH_VRS_DATAPROXY_URI` environment variable. + +## Available backends and URI schemes + +| Scheme | Backend | Notes | +| --- | --- | --- | +| `seqrepo+file:///abs/path` | Local [SeqRepo](https://github.com/biocommons/biocommons.seqrepo) | Default backend; reads a SeqRepo directory. | +| `seqrepo+:relative/path` | Local SeqRepo | Same as above with a relative path (note the empty protocol). | +| `seqrepo+http://host:port/seqrepo` | [SeqRepo REST service](https://github.com/biocommons/seqrepo-rest-service) | Remote SeqRepo over HTTP. | +| `seqrepo+https://host/seqrepo` | SeqRepo REST service | TLS variant. | +| `refget+file:///abs/path` | [gtars](https://github.com/databio/gtars) RefgetStore | Local on-disk RefgetStore. Requires `pip install ga4gh.vrs[refget]`. | +| `refget+http://host[:port]/path` | gtars RefgetStore | Remote static HTTP(S) store; working set is cached locally. | +| `refget+https://host/path` | gtars RefgetStore | TLS variant. | + +> **Relative paths:** `urlparse` treats the first segment after `file://` as a +> host, so `refget+file://my/store` resolves to the path `/store`. Use an +> absolute path (`refget+file:///abs/path`) or the empty-protocol form +> (`refget+:relative/path`). + +The `refget` backend composes a `RefgetSequenceDataProxy` (sequence retrieval by +GA4GH digest) and a `RefgetAliasDataProxy` (alias resolution via the store's +alias index) behind a `CompositeDataProxy`: identifiers are tried against the +sequence store first, and on a miss are resolved to a `ga4gh:SQ.` digest via the +alias index and retried. + +### Remote refget caching + +Remote `refget+http(s)://` stores cache their index, alias sidecars, and fetched +sequence payloads on the local filesystem. The cache root resolves to, in order: + +1. `GA4GH_VRS_REFGET_CACHE_DIR`, if set +2. `$XDG_CACHE_HOME/gtars/refget`, if `XDG_CACHE_HOME` is set +3. `~/.cache/gtars/refget` + +## In-memory caching + +Separately from the on-disk refget cache above, all backends memoize lookups +in-process with `functools.lru_cache`. Two environment variables tune the sizes: + +| Variable | Default | Covers | +| --- | --- | --- | +| `GA4GH_VRS_DATAPROXY_CACHE_SIZE` | `8192` | Sequence lengths, digests, and alias lists, across all backends. | +| `GA4GH_VRS_REST_DATAPROXY_SEQUENCE_CACHE_SIZE` | `128` | Sequence payloads fetched by the SeqRepo REST backend. | + +These are separate knobs because the cached values differ in size by orders of +magnitude. The first holds small fixed-size values — an integer length, a +digest, a list of a dozen short alias strings — so 8192 entries costs a few MB. +The second holds sequence text, which is bounded only by the caller's requested +span: the normalization path stores 1-2 base strings per entry, but +`get_sequence(identifier)` with no bounds stores an entire contig in one entry. + +`GA4GH_VRS_REST_DATAPROXY_SEQUENCE_CACHE_SIZE` applies only to +`SeqRepoRESTDataProxy` — the only backend that caches sequence payloads at all. +Local SeqRepo and refget `get_sequence()` calls hit the store every time. + +**This cache memoizes fetched regions, not sequences.** The key is the exact +`(identifier, start, end)` triple that was requested, so it is only useful when +the identical region is fetched repeatedly. In particular: + +- An overlapping or adjacent span is a miss. Having `(chr1, 100, 200)` cached + does nothing for a request for `(chr1, 150, 250)` or `(chr1, 200, 300)`. +- A cached full-sequence entry cannot serve a sub-range request. `get_sequence(id)` + is stored under `(id, None, None)` and is never consulted for `(id, 0, 10)`. +- A linear scan over distinct positions gets no benefit at all — every request is + a fresh key. Raising this value does not speed up streaming workloads such as + annotating a VCF of distinct variants; it only helps genuinely repeated + identical lookups (e.g. re-normalizing the same allele, or several alleles + resolving the same span). +- Positional and keyword arguments hash to different keys, a `functools.lru_cache` + property. `get_sequence(ac, 5, 9)` and `get_sequence(ac, start=5, end=9)` are + two separate entries and two separate HTTP requests, even though they request + the same bases. + +So treat it as a de-duplication cache for repeated identical fetches, not as a +read-ahead or a local mirror of the sequence. + +Accepted values for both: + +| Value | Meaning | +| --- | --- | +| positive integer | Maximum number of cached entries | +| `0` | Disable caching (`functools` semantics) | +| `-1` or `none` | Unbounded cache | +| anything else | Raises `ValueError` at import | + +> **Both are read once, at import time.** The `lru_cache` decorators are applied +> when the classes are defined, so setting these after `ga4gh.vrs.dataproxy` has +> been imported has no effect. Export them before starting the process. + +## Known divergences and caveats (refget vs SeqRepo) + +The refget and SeqRepo backends are functionally equivalent for the sequence and +alias data VRS-Python consumes, with two differences worth knowing about. + +### Stop-codon (`*`) digest divergence on protein sequences + +For protein sequences that contain embedded stop-codon `*` characters, a +RefgetStore can compute a **different `ga4gh:SQ.` digest than SeqRepo for the +same underlying sequence bytes**. + +- **Cause:** SeqRepo runs sequences through + `bioutils.sequences.normalize_sequence()`, which strips `*` before computing + the `sha512t24u` digest. gtars computes the digest directly from the raw FASTA + bytes, including any `*`. +- **Which is correct:** The RefgetStore digest. The GA4GH VRS specification does + not prescribe `*` stripping, so hashing the raw bytes is spec-correct. +- **Scope:** Small. In a full Ensembl r113 + RefSeq load, 116 of 322,402 + comparable Ensembl aliases diverged (~0.04%); 115 were pure `*`-normalization + artifacts (identical bytes, different digest) and 1 was genuine cross-release + sequence drift. **NCBI RefSeq protein FASTAs contain no `*` and showed zero + digest mismatches.** DNA and RNA sequences are never affected. + +The practical implication: a VRS Allele on an affected Ensembl protein will have +a different sequence identifier depending on which backend produced it. If you +need identifiers that match an existing SeqRepo-derived corpus on these specific +accessions, stay on SeqRepo for protein annotation. + +### Version-drift policy + +SeqRepo is **accumulative**: it retains every version of every accession it has +ever loaded across historical imports. A RefgetStore typically pins **one +version** of each source (e.g. a single Ensembl release; the current NCBI RefSeq +transcript/protein shards, which publish only the latest version of each +accession). As a result, the refget backend may raise `KeyError` on older +transcript/protein versions that a long-lived SeqRepo instance still holds. diff --git a/docs/extras/vcf_annotator.md b/docs/extras/vcf_annotator.md index eced28b0..79045b5b 100644 --- a/docs/extras/vcf_annotator.md +++ b/docs/extras/vcf_annotator.md @@ -15,7 +15,7 @@ vrs-annotate vcf --help ### Configuring the sequence data proxy -Like other VRS-Python tools, the VCF annotator requires access to [sequence and identifier data services](https://vrs.ga4gh.org/en/stable/impl-guide/required_data.html#data-services), as implemented in libraries like [SeqRepo](https://github.com/biocommons/biocommons.seqrepo). By default, the CLI will attempt to connect to a [SeqRepo REST instance](https://github.com/biocommons/seqrepo-rest-service) at `http://localhost:5000/seqrepo`, but a URI can be passed with the `--dataproxy-uri` option or set with the `GA4GH_VRS_DATAPROXY_URI` environment variable (the former takes priority over the latter). +Like other VRS-Python tools, the VCF annotator requires access to [sequence and identifier data services](https://vrs.ga4gh.org/en/stable/impl-guide/required_data.html#data-services), as implemented in libraries like [SeqRepo](https://github.com/biocommons/biocommons.seqrepo). By default, the CLI will attempt to connect to a [SeqRepo REST instance](https://github.com/biocommons/seqrepo-rest-service) at `http://localhost:5000/seqrepo`, but a URI can be passed with the `--dataproxy-uri` option or set with the `GA4GH_VRS_DATAPROXY_URI` environment variable (the former takes priority over the latter). Remote refget stores use `GA4GH_VRS_REFGET_CACHE_DIR` as the local cache root when set. For example, to use a local set of SeqRepo data, you can use an absolute file path: @@ -35,6 +35,13 @@ Or an alternate REST path: vrs-annotate vcf --dataproxy-uri="seqrepo+http://mylabwebsite.org/seqrepo" --vcf-out=out.vcf.gz input.vcf.gz ``` +Or a remote refget store exposed as a static HTTP(S) directory tree: + +```commandline +GA4GH_VRS_REFGET_CACHE_DIR=/tmp/vrs-refget-cache \ +vrs-annotate vcf --dataproxy-uri="refget+https://example-bucket.s3.amazonaws.com/refget-store" --vcf-out=out.vcf.gz input.vcf.gz +``` + ### Other Options `--vrs-attributes` >Will include VRS_Start, VRS_End, VRS_State fields in the INFO field. diff --git a/misc/refgetstore/README.md b/misc/refgetstore/README.md new file mode 100644 index 00000000..3144f915 --- /dev/null +++ b/misc/refgetstore/README.md @@ -0,0 +1,173 @@ +# misc/refgetstore + +Build a custom `gtars.refget.RefgetStore` archive augmented with the +per-sequence and per-collection aliases vrs-python needs to use refget as +its alias backend (replacing seqrepo for alias mapping). This covers: + +- Assembly aliases: `GRCh38:chr1`, `GRCh38.p14:chrM`, etc. +- Cross-assembly genomic accessions: `refseq:NC_000001.11`, `insdc:CM000663.2`. +- Collection-level: `refseq:GCF_*`, `insdc:GCA_*`. +- **NCBI RefSeq transcripts + proteins**: `refseq:NM_000551.3`, + `refseq:NP_000542.1`, etc., loaded from NCBI RefSeq mRNA/Prot shards. +- **Ensembl transcripts + proteins**: `ensembl:ENST00000256474.3`, + `ensembl:ENSP00000256474.3`, etc., loaded from Ensembl release cdna / + ncrna / pep FASTAs (release is pinned in `assemblies.toml`). + +No dependency on `bioutils` or `biocommons.seqrepo` — the loader reads NCBI +assembly reports directly and derives transcript/protein aliases from FASTA +header names. + +## Prerequisites + +- `gtars.refget` (already a vrs-python extras dependency). +- Network access to `ftp.ncbi.nlm.nih.gov` and `ftp.ensembl.org` (or + pre-populated cache dirs). +- Disk: + - ~5 GB for one GRCh38 major + GRCh38.p14 assemblies (the `.fna.gz` files + are ~900 MB each; the Encoded store is ~2–3 GB). + - ~3-5 GB additional for the RefSeq mRNA+protein shards (64 files, + ~50-100 MB each). + - ~500 MB additional for the Ensembl cdna + ncrna + pep FASTAs. + +## Layout + + misc/refgetstore/ + assemblies.toml # declarative assembly list (edit to add patches) + build_store.py # loader entry point + README.md # this file + downloads/ # cached FASTA + assembly_report.txt (gitignored) + store/ # output RefgetStore (gitignored) + +## Usage + +Build everything in `assemblies.toml`: + + uv run python misc/refgetstore/build_store.py + +Build only one assembly: + + uv run python misc/refgetstore/build_store.py --assembly GRCh38 + +Build only one seqset: + + uv run python misc/refgetstore/build_store.py --seqset refseq_human_rna + +Skip categories entirely: + + uv run python misc/refgetstore/build_store.py --skip-seqsets + uv run python misc/refgetstore/build_store.py --skip-assemblies + +All CLI flags: + + --config PATH assemblies.toml (default: misc/refgetstore/assemblies.toml) + --store-dir PATH output RefgetStore (default: misc/refgetstore/store) + --download-dir PATH download cache (default: misc/refgetstore/downloads) + --assembly NAME only process this assembly namespace + --seqset NAME only process this seqset name + --skip-assemblies skip all [[assembly]] entries + --skip-seqsets skip all [[seqset]] entries + --force-download re-fetch even if cached files exist + +## Config reference + +Each `[[assembly]]` block: + +| field | required | description | +| --- | --- | --- | +| `namespace` | yes | Sequence-alias namespace to populate (e.g. `GRCh38`, `GRCh38.p14`). Becomes the namespace of `GRCh38:chr1`-style aliases. | +| `fasta_url` | yes | URL of NCBI `*_genomic.fna.gz`. gtars ingests `.gz` directly. | +| `report_url` | yes | URL of the corresponding `*_assembly_report.txt`. | +| `load_fasta` | no, default `true` | If false, aliases are added only for digests already present in the store (cheap patch fanout). | +| `fasta_path` | no | Local path overriding the downloaded FASTA (relative to repo root). Used for the already-downloaded GRCh38 major. | + +Each `[[seqset]]` block (flat FASTA where the header name is the accession): + +| field | required | description | +| --- | --- | --- | +| `name` | yes | Human id for logs + CLI selection. | +| `namespace` | yes | Alias namespace for every ingested sequence (e.g. `refseq`, `ensembl`). | +| `url_template` | yes | URL with `{shard}` placeholder, or plain URL if unsharded. | +| `shard_range` | no | `[min, max]` inclusive substituted into `{shard}`. Omit for a single-file seqset. | + +The same schema handles both the NCBI RefSeq mRNA/Prot shards (sharded +with a `{shard}` placeholder) and the Ensembl cdna / ncrna / pep +releases (single-file, `shard_range` omitted). + +## What gets written + +Per assembly run, the loader adds: + +- One sequence collection per `load_fasta=true` assembly, named by its + collection sha512t24u digest. +- Per-sequence aliases under `` (the assembly namespace): + - UCSC-style-name (`chr1`, `chrM`, …) + - GenBank-Accn (`CM000663.2`, …) +- Per-sequence aliases under the cross-assembly `refseq` namespace + (`NC_000001.11` → digest). Emitted once per digest. +- Per-sequence aliases under the cross-assembly `insdc` namespace + (`CM000663.2` → digest). Emitted once per digest. +- Per-collection aliases under `refseq` (`GCF_000001405.40` → collection + digest) and `insdc` (`GCA_000001405.29` → collection digest), parsed from + the assembly report header. + +Per seqset run, the loader adds: + +- One sequence collection per FASTA file ingested (each + `human..rna.fna.gz` and each Ensembl `*.fa.gz` becomes its own + collection). +- Per-sequence aliases under `` using each FASTA header's first + token. Examples: + - NCBI RefSeq shards: `refseq:NM_000551.3`, `refseq:NP_000542.1`, + `refseq:XM_*`, `refseq:XP_*`. + - Ensembl cdna: `ensembl:ENST00000256474.3` (protein-coding transcripts). + - Ensembl ncrna: `ensembl:ENST00000429829.6` (non-coding — XIST, miRNAs, + snoRNAs, etc.). + - Ensembl pep: `ensembl:ENSP00000256474.3`. + +Aliases for `sha512t24u:…` and `ga4gh:SQ.…` are intentionally NOT written — +they are the raw digest with a prefix and are synthesized at query time by +`RefgetAliasDataProxy`. + +## Version-drift policy + +The refgetstore pins **one version** of each source: + +- **NCBI assemblies**: pinned by GCF accession (e.g. `GCF_000001405.40` for + GRCh38.p14). Each patch is a superset of the prior, so only the latest + patch of each major assembly is loaded. +- **NCBI RefSeq transcripts/proteins**: the FTP shards + (`human.*.rna.fna.gz` / `human.*.protein.faa.gz`) publish only the + current version of each accession. Older versions are not available. +- **Ensembl**: pinned to a single release (currently **r113**, set in + `assemblies.toml`). Only the latest version of each ENST/ENSP in that + release is included. + +This differs from seqrepo's accumulative model, which retains every +version it has ever loaded across multiple historical imports. Callers that +depend on older transcript/protein versions will encounter `KeyError` on +accessions the refgetstore has not loaded. + +### Known digest divergence on Ensembl proteins with `*` + +For ~0.1% of Ensembl protein sequences (115 ENSPs containing embedded +stop codon `*` characters), the refgetstore and seqrepo produce different +`ga4gh:SQ.*` identifiers **despite storing identical sequence bytes**. +This is caused by `bioutils.sequences.normalize_sequence()` stripping `*` +before computing the digest in seqrepo, while gtars computes `sha512t24u` +directly from the raw FASTA bytes. The refgetstore digest is correct per +the GA4GH VRS specification. See `ensembl_known_divergent.txt` and +`diagnosis_report.json` for the full list and investigation details. + +DNA sequences are not affected — the divergence only applies to protein +sequences containing `*`. + +## Verification scripts + +| Script | Purpose | +| --- | --- | +| `verify_store.py` | 67+ post-build checks: inventory, ground-truth digests, coverage, Ensembl, known-divergent regression | +| `compare_against_seqrepo.py` | 3-phase functional equivalence against seqrepo 2024-12-20 | +| `vcf_equivalence.py` | VCF annotator backend comparison + timing | +| `bulk_equivalence.py` | Translator backend comparison on 1M VCF records | +| `ensembl_translator_equivalence.py` | SPDI-based translator comparison using Ensembl transcript accessions | +| `diagnose_ensembl_mismatches.py` | Root-cause analysis of the 116 Ensembl digest mismatches | diff --git a/misc/refgetstore/assemblies.toml b/misc/refgetstore/assemblies.toml new file mode 100644 index 00000000..e70332fe --- /dev/null +++ b/misc/refgetstore/assemblies.toml @@ -0,0 +1,93 @@ +# Sequences to ingest into the custom vrs-python refgetstore archive. +# +# Two entry kinds: +# +# [[assembly]] An assembled genome FASTA + matching NCBI assembly_report.txt. +# Produces one collection and derives per-sequence aliases +# (UCSC chr names, GenBank accns, RefSeq accns) from the +# report. The cross-assembly `refseq` and `insdc` namespaces +# are populated regardless of the entry's own `namespace`. +# +# [[seqset]] A flat FASTA (or sharded set of FASTAs) where the accession +# is the FASTA header's first token. Used for NCBI RefSeq +# transcript/protein shards. Each shard becomes its own +# collection; aliases map header-name -> digest under the +# configured namespace. +# +# Assembly fields +# namespace Per-assembly alias namespace (e.g. "GRCh38.p14"). +# fasta_url URL of the genomic FASTA (.fna.gz supported directly by gtars). +# report_url URL of the NCBI assembly_report.txt. +# load_fasta If true, ingest the FASTA. If false, reuse digests already +# present in the store (cheap patch fanout). +# fasta_path Optional local path override, relative to repo root. +# +# Seqset fields +# name Human-readable id for logs + summary. +# namespace Alias namespace for every ingested sequence (e.g. "refseq"). +# url_template URL with "{shard}" placeholder, or a plain URL if unsharded. +# shard_range [min, max] inclusive integer range substituted into +# url_template. Omit for a single-file seqset. + +[[assembly]] +namespace = "GRCh38" +fasta_url = "https://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/000/001/405/GCF_000001405.26_GRCh38/GCF_000001405.26_GRCh38_genomic.fna.gz" +report_url = "https://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/000/001/405/GCF_000001405.26_GRCh38/GCF_000001405.26_GRCh38_assembly_report.txt" +load_fasta = true +fasta_path = "GCF_000001405.26_GRCh38_genomic.fna.gz" + +[[assembly]] +namespace = "GRCh38.p14" +fasta_url = "https://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/000/001/405/GCF_000001405.40_GRCh38.p14/GCF_000001405.40_GRCh38.p14_genomic.fna.gz" +report_url = "https://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/000/001/405/GCF_000001405.40_GRCh38.p14/GCF_000001405.40_GRCh38.p14_assembly_report.txt" +load_fasta = true + +[[assembly]] +namespace = "GRCh37" +fasta_url = "https://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/000/001/405/GCF_000001405.13_GRCh37/GCF_000001405.13_GRCh37_genomic.fna.gz" +report_url = "https://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/000/001/405/GCF_000001405.13_GRCh37/GCF_000001405.13_GRCh37_assembly_report.txt" +load_fasta = true + +[[assembly]] +namespace = "GRCh37.p13" +fasta_url = "https://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/000/001/405/GCF_000001405.25_GRCh37.p13/GCF_000001405.25_GRCh37.p13_genomic.fna.gz" +report_url = "https://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/000/001/405/GCF_000001405.25_GRCh37.p13/GCF_000001405.25_GRCh37.p13_assembly_report.txt" +load_fasta = true + +[[seqset]] +name = "refseq_human_rna" +namespace = "refseq" +url_template = "https://ftp.ncbi.nlm.nih.gov/refseq/H_sapiens/mRNA_Prot/human.{shard}.rna.fna.gz" +shard_range = [1, 32] + +[[seqset]] +name = "refseq_human_protein" +namespace = "refseq" +url_template = "https://ftp.ncbi.nlm.nih.gov/refseq/H_sapiens/mRNA_Prot/human.{shard}.protein.faa.gz" +shard_range = [1, 32] + +# Ensembl human GRCh38 transcripts + proteins, pinned to release 113 +# (October 2024). Bump the release number in all three URLs to update. +# Archive index: https://www.ensembl.org/info/website/archives/ +# ENST.N / ENSP.N accessions are content-addressable, so the digests are +# stable across releases; only the version suffixes change. +# +# cdna/ ships protein-coding transcripts; ncrna/ ships non-coding +# transcripts (disjoint ENST space). cds/ is intentionally NOT loaded — +# it reuses the same ENST accessions as cdna/ but with UTRs stripped, +# which would collide in the bulk alias merge. + +[[seqset]] +name = "ensembl_human_cdna" +namespace = "ensembl" +url_template = "https://ftp.ensembl.org/pub/release-113/fasta/homo_sapiens/cdna/Homo_sapiens.GRCh38.cdna.all.fa.gz" + +[[seqset]] +name = "ensembl_human_ncrna" +namespace = "ensembl" +url_template = "https://ftp.ensembl.org/pub/release-113/fasta/homo_sapiens/ncrna/Homo_sapiens.GRCh38.ncrna.fa.gz" + +[[seqset]] +name = "ensembl_human_pep" +namespace = "ensembl" +url_template = "https://ftp.ensembl.org/pub/release-113/fasta/homo_sapiens/pep/Homo_sapiens.GRCh38.pep.all.fa.gz" diff --git a/misc/refgetstore/build_store.py b/misc/refgetstore/build_store.py new file mode 100644 index 00000000..edf98cb9 --- /dev/null +++ b/misc/refgetstore/build_store.py @@ -0,0 +1,591 @@ +#!/usr/bin/env python +"""Build a custom gtars RefgetStore with vrs-python alias parity. + +Ingests NCBI genomic FASTA files into a local RefgetStore and augments it with +per-sequence and per-collection aliases parsed from the matching NCBI assembly +reports. See misc/refgetstore/README.md for details. +""" + +from __future__ import annotations + +import argparse +import logging +import sys +import tempfile +import tomllib +import urllib.request +from dataclasses import dataclass, field +from pathlib import Path +from typing import Iterator + +from gtars.refget import RefgetStore + +logger = logging.getLogger("build_store") + +REPO_ROOT = Path(__file__).resolve().parents[2] +DEFAULT_CONFIG = Path(__file__).parent / "assemblies.toml" +DEFAULT_STORE = Path(__file__).parent / "store" +DEFAULT_DOWNLOADS = Path(__file__).parent / "downloads" + +SQ_PREFIX = "SQ." +NA_VALUES = {"na", "", ""} + + +@dataclass +class AssemblyConfig: + namespace: str + fasta_url: str + report_url: str + load_fasta: bool = True + fasta_path: str | None = None + + +@dataclass +class SeqsetConfig: + """Flat FASTA (optionally sharded) where the accession is the header name. + + Used for NCBI RefSeq transcript/protein shards that have no sidecar + assembly_report.txt. Each shard is ingested as its own collection; header + names become ``namespace:`` aliases on the resulting digests. + """ + + name: str + namespace: str + url_template: str + shard_range: list[int] | None = None + + def iter_shard_urls(self) -> Iterator[tuple[str, str]]: + """Yield (shard_label, url) pairs. Shard label is "" for unsharded.""" + if self.shard_range is None: + yield "", self.url_template + return + lo, hi = self.shard_range + for i in range(lo, hi + 1): + yield str(i), self.url_template.replace("{shard}", str(i)) + + +@dataclass +class AssemblyReport: + """Parsed NCBI assembly_report.txt.""" + + refseq_assembly_accession: str | None + genbank_assembly_accession: str | None + rows: list[dict[str, str]] = field(default_factory=list) + + +@dataclass +class AssemblyStats: + namespace: str + collection_digest: str | None = None + rows_total: int = 0 + rows_resolved: int = 0 + rows_skipped_non_refseq: int = 0 + sequence_aliases_added: int = 0 + sequence_aliases_skipped: int = 0 + collection_aliases_added: int = 0 + warnings: int = 0 + + +@dataclass +class SeqsetStats: + name: str + namespace: str + shards_processed: int = 0 + sequences_ingested: int = 0 + sequence_aliases_added: int = 0 + sequence_aliases_skipped: int = 0 + warnings: int = 0 + + +def load_config( + path: Path, +) -> tuple[list[AssemblyConfig], list[SeqsetConfig]]: + with path.open("rb") as fh: + data = tomllib.load(fh) + assemblies = [AssemblyConfig(**entry) for entry in data.get("assembly", [])] + seqsets = [SeqsetConfig(**entry) for entry in data.get("seqset", [])] + return assemblies, seqsets + + +def ensure_download(url: str, target: Path, force: bool) -> Path: + target.parent.mkdir(parents=True, exist_ok=True) + if target.exists() and not force: + logger.info("using cached %s", target) + return target + logger.info("downloading %s -> %s", url, target) + tmp = target.with_suffix(target.suffix + ".part") + urllib.request.urlretrieve(url, tmp) # noqa: S310 + tmp.replace(target) + return target + + +def resolve_fasta_source( + entry: AssemblyConfig, download_dir: Path, force: bool +) -> Path | None: + if not entry.load_fasta: + return None + if entry.fasta_path: + local = (REPO_ROOT / entry.fasta_path).resolve() + if local.exists(): + logger.info("using local fasta override %s", local) + return local + logger.warning("fasta_path %s not found; falling back to download", local) + return ensure_download( + entry.fasta_url, + download_dir / Path(entry.fasta_url).name, + force, + ) + + +def resolve_report_source( + entry: AssemblyConfig, download_dir: Path, force: bool +) -> Path: + return ensure_download( + entry.report_url, + download_dir / f"{entry.namespace}.assembly_report.txt", + force, + ) + + +def parse_assembly_report(path: Path) -> AssemblyReport: + """Parse an NCBI assembly_report.txt into a structured object. + + Header lines begin with ``#`` and include ``GenBank assembly accession`` / + ``RefSeq assembly accession`` entries. The final ``#``-prefixed line is the + column header for the data rows (a tab-separated table). + """ + refseq_accn: str | None = None + genbank_accn: str | None = None + columns: list[str] | None = None + rows: list[dict[str, str]] = [] + with path.open("r", encoding="utf-8") as fh: + for raw in fh: + line = raw.rstrip("\n\r") + if not line: + continue + if line.startswith("#"): + stripped = line.lstrip("#").strip() + if stripped.startswith("GenBank assembly accession:"): + genbank_accn = stripped.split(":", 1)[1].strip() or None + elif stripped.startswith("RefSeq assembly accession:"): + refseq_accn = stripped.split(":", 1)[1].strip() or None + elif "\t" in stripped and "Sequence-Name" in stripped: + columns = stripped.split("\t") + continue + if columns is None: + continue + parts = line.split("\t") + if len(parts) != len(columns): + continue + rows.append(dict(zip(columns, parts))) + return AssemblyReport( + refseq_assembly_accession=refseq_accn, + genbank_assembly_accession=genbank_accn, + rows=rows, + ) + + +def build_name_to_digest_map( + store: RefgetStore, collection_digest: str +) -> dict[str, str]: + store.load_collection(collection_digest) + level2 = store.get_collection_level2(collection_digest) + names = level2["names"] + sequences = level2["sequences"] + out: dict[str, str] = {} + for name, seq_id in zip(names, sequences): + digest = seq_id[len(SQ_PREFIX) :] if seq_id.startswith(SQ_PREFIX) else seq_id + out[name] = digest + return out + + +def build_global_name_to_digest_map(store: RefgetStore) -> dict[str, str]: + """Fallback map built by scanning every sequence in the store. + + Used when `load_fasta=false` so we can still resolve refseq accessions to + a digest without having just ingested this assembly's collection. Names + are globally unique-ish (seqrepo jungle store: 1224 collisions in 580k); + we keep the first digest we see for each name. + """ + out: dict[str, str] = {} + for record in store.iter_sequences(): + md = record.metadata + out.setdefault(md.name, md.sha512t24u) + return out + + +def add_unique_alias( + store: RefgetStore, + namespace: str, + alias: str, + digest: str, + stats: AssemblyStats, +) -> None: + existing = store.get_sequence_by_alias(namespace, alias) + if existing is not None: + stats.sequence_aliases_skipped += 1 + return + store.add_sequence_alias(namespace, alias, digest) + stats.sequence_aliases_added += 1 + + +def add_aliases_for_row( + store: RefgetStore, + row: dict[str, str], + namespace: str, + name_to_digest: dict[str, str], + stats: AssemblyStats, +) -> None: + refseq_ac = row.get("RefSeq-Accn", "").strip() + ucsc_name = row.get("UCSC-style-name", "").strip() + genbank_ac = row.get("GenBank-Accn", "").strip() + + # Scope: we only alias sequences that are actually in the RefSeq dataset. + # Rows with RefSeq-Accn=na are GenBank-only contigs that the NCBI + # _genomic.fna.gz (RefSeq view) deliberately omits, so there is no + # sequence data to alias to. Skip cleanly without a warning. + if refseq_ac.lower() in NA_VALUES: + stats.rows_skipped_non_refseq += 1 + return + + digest = name_to_digest.get(refseq_ac) + if digest is None: + stats.warnings += 1 + logger.warning( + "no digest for %s in namespace %s (FASTA does not contain it)", + refseq_ac, + namespace, + ) + return + + stats.rows_resolved += 1 + if ucsc_name and ucsc_name.lower() not in NA_VALUES: + add_unique_alias(store, namespace, ucsc_name, digest, stats) + if genbank_ac and genbank_ac.lower() not in NA_VALUES: + add_unique_alias(store, namespace, genbank_ac, digest, stats) + add_unique_alias(store, "insdc", genbank_ac, digest, stats) + add_unique_alias(store, "refseq", refseq_ac, digest, stats) + + +def add_collection_aliases( + store: RefgetStore, + report: AssemblyReport, + collection_digest: str, + stats: AssemblyStats, +) -> None: + if report.refseq_assembly_accession: + store.add_collection_alias( + "refseq", report.refseq_assembly_accession, collection_digest + ) + stats.collection_aliases_added += 1 + if report.genbank_assembly_accession: + store.add_collection_alias( + "insdc", report.genbank_assembly_accession, collection_digest + ) + stats.collection_aliases_added += 1 + + +def process_assembly( + store: RefgetStore, + entry: AssemblyConfig, + download_dir: Path, + force_download: bool, + global_name_map_cache: dict[str, dict[str, str]], +) -> AssemblyStats: + stats = AssemblyStats(namespace=entry.namespace) + logger.info("=== %s ===", entry.namespace) + fasta_path = resolve_fasta_source(entry, download_dir, force_download) + report_path = resolve_report_source(entry, download_dir, force_download) + + name_to_digest: dict[str, str] + coll_digest: str | None = None + if fasta_path is not None: + logger.info("ingesting %s", fasta_path) + coll_meta, was_new = store.add_sequence_collection_from_fasta(str(fasta_path)) + coll_digest = coll_meta.digest + stats.collection_digest = coll_digest + logger.info( + "collection %s (%s, %d sequences)", + coll_digest, + "new" if was_new else "existing", + coll_meta.n_sequences, + ) + name_to_digest = build_name_to_digest_map(store, coll_digest) + else: + logger.info("load_fasta=false; using global name map") + cached = global_name_map_cache.get("_global") + if cached is None: + cached = build_global_name_to_digest_map(store) + global_name_map_cache["_global"] = cached + name_to_digest = cached + + logger.info("parsing %s", report_path) + report = parse_assembly_report(report_path) + stats.rows_total = len(report.rows) + logger.info( + "report: %d rows, refseq=%s genbank=%s", + stats.rows_total, + report.refseq_assembly_accession, + report.genbank_assembly_accession, + ) + + for row in report.rows: + add_aliases_for_row(store, row, entry.namespace, name_to_digest, stats) + + if coll_digest is not None: + add_collection_aliases(store, report, coll_digest, stats) + + return stats + + +def process_seqset( + store: RefgetStore, + entry: SeqsetConfig, + download_dir: Path, + force_download: bool, +) -> SeqsetStats: + """Ingest a flat/sharded seqset and alias every header name. + + For each shard FASTA: + 1. Download (cached). + 2. ``add_sequence_collection_from_fasta`` -> collection digest. + 3. Walk collection level-2 contents; collect ``(name, digest)`` pairs. + + Aliases are written in bulk via ``store.load_sequence_aliases`` after all + shards have been ingested. ``add_sequence_alias`` rewrites the full + namespace TSV on every call (gtars AliasManager), so inserting N entries + one at a time is O(N^2) — 21k RNA aliases per shard turns into hours. + ``load_sequence_aliases`` merges a whole TSV into the in-memory alias + map and triggers exactly one persist, dropping the per-alias cost from + ~7.5 ms to ~5 µs. + """ + stats = SeqsetStats(name=entry.name, namespace=entry.namespace) + logger.info("=== seqset %s ===", entry.name) + + # alias -> digest; later entries win on collision. setdefault() preserves + # the first digest seen (matches the old add_unique_alias behavior). + pending: dict[str, str] = {} + + for shard_label, url in entry.iter_shard_urls(): + target = download_dir / Path(url).name + logger.info( + "shard %s%s", + shard_label or "-", + f" ({Path(url).name})" if shard_label else "", + ) + try: + fasta_path = ensure_download(url, target, force_download) + except Exception as exc: # noqa: BLE001 + logger.warning("download failed for %s: %s", url, exc) + stats.warnings += 1 + continue + + try: + coll_meta, was_new = store.add_sequence_collection_from_fasta( + str(fasta_path) + ) + except Exception as exc: # noqa: BLE001 + logger.warning("ingest failed for %s: %s", fasta_path, exc) + stats.warnings += 1 + continue + + logger.info( + " collection %s (%s, %d sequences)", + coll_meta.digest, + "new" if was_new else "existing", + coll_meta.n_sequences, + ) + stats.shards_processed += 1 + stats.sequences_ingested += coll_meta.n_sequences + + name_to_digest = build_name_to_digest_map(store, coll_meta.digest) + for name, digest in name_to_digest.items(): + pending.setdefault(name, digest) + + if pending: + # Drop anything already aliased in this namespace (may be left over + # from prior runs or overlap with other seqsets pointing at the same + # namespace). list_sequence_aliases is an O(n) in-memory scan. + existing = set(store.list_sequence_aliases(entry.namespace) or []) + new_aliases = {a: d for a, d in pending.items() if a not in existing} + stats.sequence_aliases_skipped += len(pending) - len(new_aliases) + + if new_aliases: + with tempfile.NamedTemporaryFile( + mode="w", suffix=".tsv", delete=False + ) as tmp: + for alias, digest in new_aliases.items(): + tmp.write(f"{alias}\t{digest}\n") + tmp_path = Path(tmp.name) + try: + count = store.load_sequence_aliases(entry.namespace, str(tmp_path)) + stats.sequence_aliases_added += count + logger.info( + " %s: bulk-loaded %d new aliases into ns=%s", + entry.name, + count, + entry.namespace, + ) + finally: + tmp_path.unlink(missing_ok=True) + + return stats + + +def print_summary( + all_stats: list[AssemblyStats], + seqset_stats: list[SeqsetStats], + store: RefgetStore, +) -> None: + print() + print("Summary") + print("-------") + for s in all_stats: + print( + f" {s.namespace:14s} rows={s.rows_total:4d} " + f"resolved={s.rows_resolved:4d} " + f"non_refseq={s.rows_skipped_non_refseq:3d} " + f"seq_aliases_added={s.sequence_aliases_added:4d} " + f"skipped={s.sequence_aliases_skipped:4d} " + f"coll_aliases_added={s.collection_aliases_added:2d} " + f"warnings={s.warnings:3d}" + ) + for s in seqset_stats: + print( + f" {s.name:24s} ns={s.namespace:8s} " + f"shards={s.shards_processed:3d} " + f"seqs={s.sequences_ingested:7d} " + f"seq_aliases_added={s.sequence_aliases_added:7d} " + f"skipped={s.sequence_aliases_skipped:6d} " + f"warnings={s.warnings:3d}" + ) + print() + print("Store stats:", store.stats()) + print( + "Sequence alias namespaces:", + sorted(store.list_sequence_alias_namespaces()), + ) + print( + "Collection alias namespaces:", + sorted(store.list_collection_alias_namespaces()), + ) + + +def iter_selected_assemblies( + entries: list[AssemblyConfig], selected: str | None +) -> Iterator[AssemblyConfig]: + if selected is None: + yield from entries + return + found = False + for entry in entries: + if entry.namespace == selected: + found = True + yield entry + if not found: + raise SystemExit(f"no assembly with namespace {selected!r} in config") + + +def iter_selected_seqsets( + entries: list[SeqsetConfig], selected: str | None +) -> Iterator[SeqsetConfig]: + if selected is None: + yield from entries + return + found = False + for entry in entries: + if entry.name == selected: + found = True + yield entry + if not found: + raise SystemExit(f"no seqset with name {selected!r} in config") + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", type=Path, default=DEFAULT_CONFIG) + parser.add_argument("--store-dir", type=Path, default=DEFAULT_STORE) + parser.add_argument("--download-dir", type=Path, default=DEFAULT_DOWNLOADS) + parser.add_argument( + "--assembly", + type=str, + default=None, + help="Process only this assembly namespace (skips seqsets).", + ) + parser.add_argument( + "--seqset", + type=str, + default=None, + help="Process only this seqset name (skips assemblies).", + ) + parser.add_argument( + "--skip-assemblies", + action="store_true", + help="Skip all [[assembly]] entries.", + ) + parser.add_argument( + "--skip-seqsets", + action="store_true", + help="Skip all [[seqset]] entries.", + ) + parser.add_argument("--force-download", action="store_true") + parser.add_argument("--verbose", "-v", action="store_true") + args = parser.parse_args() + + logging.basicConfig( + level=logging.DEBUG if args.verbose else logging.INFO, + format="%(levelname)s %(name)s: %(message)s", + ) + + assemblies, seqsets = load_config(args.config) + logger.info( + "loaded %d assembly + %d seqset entries from %s", + len(assemblies), + len(seqsets), + args.config, + ) + + args.store_dir.mkdir(parents=True, exist_ok=True) + args.download_dir.mkdir(parents=True, exist_ok=True) + + store = RefgetStore.on_disk(str(args.store_dir)) + logger.info("opened store at %s (mode=%s)", args.store_dir, store.storage_mode) + + # If the user passes --assembly or --seqset, they implicitly only want + # that kind — otherwise we run both. + run_assemblies = not args.skip_assemblies and args.seqset is None + run_seqsets = not args.skip_seqsets and args.assembly is None + + all_stats: list[AssemblyStats] = [] + seqset_stats: list[SeqsetStats] = [] + global_name_map_cache: dict[str, dict[str, str]] = {} + + if run_assemblies: + for entry in iter_selected_assemblies(assemblies, args.assembly): + stats = process_assembly( + store, + entry, + args.download_dir, + args.force_download, + global_name_map_cache, + ) + all_stats.append(stats) + + if run_seqsets: + for entry in iter_selected_seqsets(seqsets, args.seqset): + seqset_stats.append( + process_seqset( + store, + entry, + args.download_dir, + args.force_download, + ) + ) + + logger.info("persisting store to %s", args.store_dir) + store.write() + print_summary(all_stats, seqset_stats, store) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/misc/refgetstore/bulk_equivalence.py b/misc/refgetstore/bulk_equivalence.py new file mode 100644 index 00000000..69006f8b --- /dev/null +++ b/misc/refgetstore/bulk_equivalence.py @@ -0,0 +1,384 @@ +"""Bulk translator-path equivalence test: seqrepo vs refgetstore. + +Drives ga4gh.vrs.extras.translator.AlleleTranslator end-to-end over every +record of a large VCF with each dataproxy backend, then compares the +resulting VRS Allele IDs for exact agreement. + +This complements ``vcf_equivalence.py`` (which drives the VCF *annotator* +and compares VCF output). ``bulk_equivalence.py`` drives the *translator* +directly via gnomad-style expressions so the test exercises the same +data-proxy call sites as HGVS/SPDI import paths do. + +Seqrepo is always run with ``fd_cache_size=None`` (unbounded file-descriptor +cache) so the comparison reflects steady-state performance rather than +per-record file-open cost. Export a larger ``SEQREPO_LRU_CACHE_MAXSIZE`` +(default 1_000_000) if your workload does more distinct sequence fetches +than that. + +Everything runs single-threaded (no pysam thread pool, no VRS parallelism). + +Usage: + SEQREPO_ROOT_DIR=/path/to/seqrepo/2024-12-20 \\ + uv run python misc/refgetstore/bulk_equivalence.py \\ + --input ~/dev/vrs-rust/gnomad_chr1.1M.vcf.bgz \\ + --refget-store misc/refgetstore/store + +The refget store must already be built by build_store.py. + +Exits non-zero if any Allele ID mismatch is found. Writes a JSON summary +to ``--report`` if provided. +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +import time +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import TYPE_CHECKING + +from biocommons.seqrepo import SeqRepo + +from ga4gh.vrs.dataproxy import CompositeDataProxy +from ga4gh.vrs.dataproxy_refget import ( + RefgetAliasDataProxy, + RefgetSequenceDataProxy, +) +from ga4gh.vrs.dataproxy_seqrepo import SeqRepoDataProxy +from ga4gh.vrs.extras.translator import AlleleTranslator + +if TYPE_CHECKING: + from gtars.refget import RefgetStore + + +# --------------------------------------------------------------------------- +# Dataproxy construction (kept in sync with vcf_equivalence.py) +# --------------------------------------------------------------------------- + + +def open_refget_store(store_path: Path) -> RefgetStore: + from gtars.refget import RefgetStore + + if not (store_path / "rgstore.json").exists(): + raise SystemExit( + f"refget store not found at {store_path}; " + f"run `uv run python misc/refgetstore/build_store.py` first" + ) + return RefgetStore.open_local(str(store_path)) + + +def make_seqrepo_proxy(seqrepo_root: Path) -> SeqRepoDataProxy: + """Construct a seqrepo-backed proxy with an unbounded fd cache. + + ``SEQREPO_FD_CACHE_MAXSIZE`` env var overrides this if set; otherwise + ``fd_cache_size=None`` means `functools.lru_cache(maxsize=None)` — no + eviction at any point, so the seqrepo baseline measures steady state. + """ + sr = SeqRepo(root_dir=str(seqrepo_root), fd_cache_size=None) + return SeqRepoDataProxy(sr) + + +def make_refget_native_proxy(store: RefgetStore) -> CompositeDataProxy: + return CompositeDataProxy( + RefgetSequenceDataProxy(store), + RefgetAliasDataProxy(store), + ) + + +# --------------------------------------------------------------------------- +# Translation + comparison +# --------------------------------------------------------------------------- + + +@dataclass +class BackendResult: + name: str + allele_ids: list[str | None] = field(default_factory=list) + errors: list[str | None] = field(default_factory=list) + wall_seconds: float = 0.0 + + @property + def n_success(self) -> int: + return sum(1 for e in self.errors if e is None) + + @property + def n_errors(self) -> int: + return len(self.errors) - self.n_success + + +def iter_gnomad_expressions(input_vcf: Path, limit: int | None): + """Yield (line_no, gnomad_expr, coord_str) for each record/alt-allele. + + One input VCF record with N alt alleles produces N gnomad expressions — + matching how annotators split multi-allelic sites. + """ + import pysam + + vcf = pysam.VariantFile(str(input_vcf)) + i = 0 + for rec in vcf: + if not rec.alts: + continue + for alt in rec.alts: + i += 1 + expr = f"{rec.chrom}-{rec.pos}-{rec.ref}-{alt}" + coord = expr + yield i, expr, coord + if limit is not None and i >= limit: + vcf.close() + return + vcf.close() + + +def run_backend( + name: str, + tlr: AlleleTranslator, + input_vcf: Path, + limit: int | None, + progress_every: int, +) -> BackendResult: + result = BackendResult(name=name) + t0 = time.perf_counter() + for i, expr, _coord in iter_gnomad_expressions(input_vcf, limit): + try: + allele = tlr.translate_from(expr, fmt="gnomad") + result.allele_ids.append(allele.id if allele is not None else None) + result.errors.append(None if allele is not None else "returned None") + except Exception as exc: + result.allele_ids.append(None) + result.errors.append(f"{type(exc).__name__}: {exc}") + if progress_every and i % progress_every == 0: + elapsed = time.perf_counter() - t0 + rate = i / elapsed if elapsed > 0 else 0.0 + print( + f" [{name}] {i} records in {elapsed:.1f}s ({rate:.0f}/s)", + file=sys.stderr, + ) + result.wall_seconds = time.perf_counter() - t0 + return result + + +@dataclass +class Mismatch: + index: int + expr: str + baseline_id: str | None + other_id: str | None + baseline_error: str | None + other_error: str | None + + +def compare_results( + baseline: BackendResult, + other: BackendResult, + exprs: list[str], + max_mismatches: int, +) -> tuple[int, int, list[Mismatch]]: + total = min(len(baseline.allele_ids), len(other.allele_ids)) + id_matches = 0 + mismatches: list[Mismatch] = [] + for i in range(total): + b_id = baseline.allele_ids[i] + o_id = other.allele_ids[i] + if b_id == o_id and baseline.errors[i] == other.errors[i]: + id_matches += 1 + elif len(mismatches) < max_mismatches: + mismatches.append( + Mismatch( + index=i, + expr=exprs[i], + baseline_id=b_id, + other_id=o_id, + baseline_error=baseline.errors[i], + other_error=other.errors[i], + ) + ) + return total, id_matches, mismatches + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +def parse_args() -> argparse.Namespace: + home = Path.home() + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--input", + type=Path, + default=home / "dev/vrs-rust/gnomad_chr1.1M.vcf.bgz", + help="Input VCF (bgz/plain). Default: %(default)s", + ) + parser.add_argument( + "--limit", + type=int, + default=None, + help="Stop after N alt-allele records (default: process entire file).", + ) + parser.add_argument( + "--seqrepo-root", + type=Path, + default=Path(os.environ.get("SEQREPO_ROOT_DIR", "")), + ) + parser.add_argument( + "--refget-store", + type=Path, + default=Path("misc/refgetstore/store"), + ) + parser.add_argument( + "--assembly", + default="GRCh38", + help="Default assembly name for the translator.", + ) + parser.add_argument( + "--report", + type=Path, + default=None, + help="Optional JSON report path.", + ) + parser.add_argument( + "--max-mismatches", + type=int, + default=20, + help="Max mismatches to collect for reporting.", + ) + parser.add_argument( + "--progress-every", + type=int, + default=5000, + help="Emit a progress line every N records (0 to disable).", + ) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + + if not args.input.exists(): + print(f"Error: input not found: {args.input}", file=sys.stderr) + return 2 + if not args.seqrepo_root or not args.seqrepo_root.exists(): + print( + f"Error: seqrepo root not found: {args.seqrepo_root}", + file=sys.stderr, + ) + return 2 + + print("=" * 70) + print("Bulk equivalence: translator path") + print("=" * 70) + print(f" Input: {args.input}") + print(f" SeqRepo root: {args.seqrepo_root}") + print(f" Refget store: {args.refget_store}") + print(f" Assembly: {args.assembly}") + print(f" Limit: {args.limit if args.limit else 'all records'}") + print( + f" SEQREPO_FD_CACHE_MAXSIZE={os.environ.get('SEQREPO_FD_CACHE_MAXSIZE', '(unset, constructor=None=unbounded)')}" + ) + print( + f" SEQREPO_LRU_CACHE_MAXSIZE={os.environ.get('SEQREPO_LRU_CACHE_MAXSIZE', '(unset, default=1_000_000)')}" + ) + print() + + print("[setup] Opening refget store...") + store = open_refget_store(args.refget_store) + + print("[setup] Building dataproxies...") + seqrepo_proxy = make_seqrepo_proxy(args.seqrepo_root) + refget_native_proxy = make_refget_native_proxy(store) + + seqrepo_tlr = AlleleTranslator(seqrepo_proxy, default_assembly_name=args.assembly) + refget_native_tlr = AlleleTranslator( + refget_native_proxy, default_assembly_name=args.assembly + ) + + # Collect expressions once for mismatch reporting + print("[setup] Indexing expressions from input VCF...") + exprs = [expr for _, expr, _ in iter_gnomad_expressions(args.input, args.limit)] + print(f" {len(exprs)} expressions to translate") + print() + + backends: dict[str, BackendResult] = {} + + print("[run] seqrepo...") + backends["seqrepo"] = run_backend( + "seqrepo", seqrepo_tlr, args.input, args.limit, args.progress_every + ) + print(f" {backends['seqrepo'].wall_seconds:.2f}s") + + print("[run] refget_native...") + backends["refget_native"] = run_backend( + "refget_native", + refget_native_tlr, + args.input, + args.limit, + args.progress_every, + ) + print(f" {backends['refget_native'].wall_seconds:.2f}s") + + # --- Compare against seqrepo baseline --- + baseline = backends["seqrepo"] + print("\n" + "=" * 70) + print("Results") + print("=" * 70) + print( + f" baseline (seqrepo): successes={baseline.n_success} " + f"errors={baseline.n_errors} wall={baseline.wall_seconds:.2f}s" + ) + + report_payload: dict = { + "input": str(args.input), + "limit": args.limit, + "total_expressions": len(exprs), + "backends": {}, + } + + all_match = True + for name, result in backends.items(): + report_payload["backends"][name] = { + "wall_seconds": result.wall_seconds, + "n_success": result.n_success, + "n_errors": result.n_errors, + } + if name == "seqrepo": + continue + total, id_matches, mismatches = compare_results( + baseline, result, exprs, args.max_mismatches + ) + print( + f" {name:28s}: id_matches={id_matches}/{total} " + f"wall={result.wall_seconds:.2f}s " + f"(ratio={baseline.wall_seconds / result.wall_seconds:.2f}x)" + ) + report_payload["backends"][name]["id_matches"] = id_matches + report_payload["backends"][name]["total_compared"] = total + report_payload["backends"][name]["mismatches"] = [asdict(m) for m in mismatches] + if id_matches != total: + all_match = False + n_show = min(5, len(mismatches)) + print(f" First {n_show} mismatches:") + for m in mismatches[:n_show]: + print( + f" [{m.index}] {m.expr}: " + f"{m.baseline_id} (err={m.baseline_error}) vs " + f"{m.other_id} (err={m.other_error})" + ) + + if args.report: + args.report.parent.mkdir(parents=True, exist_ok=True) + args.report.write_text(json.dumps(report_payload, indent=2)) + print(f"\n report written to {args.report}") + + if all_match: + print("\nPASS: all backends agree on Allele IDs") + return 0 + print("\nFAIL: at least one backend diverged from seqrepo baseline") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/misc/refgetstore/compare_against_seqrepo.py b/misc/refgetstore/compare_against_seqrepo.py new file mode 100644 index 00000000..a4b20882 --- /dev/null +++ b/misc/refgetstore/compare_against_seqrepo.py @@ -0,0 +1,259 @@ +#!/usr/bin/env python +"""Compare the refget-backed DataProxy against the seqrepo-backed DataProxy. + +Verifies that the RefgetAliasDataProxy (backed by the custom refgetstore +archive under misc/refgetstore/store) behaves equivalently to the existing +SeqRepoDataProxy for the operations vrs-python callers actually use. + +Three phases: + + Phase 1 — Functional equivalence on identifiers both proxies support. + For every canonical identifier, seqrepo and refget must agree on: + derive_refget_accession(x) + translate_sequence_identifier(x, namespace="ga4gh") + translate_sequence_identifier(x, namespace="refseq") + get_sequence_length(x) + + Phase 2 — Capability extension. Identifiers where seqrepo raises + KeyError but refget resolves cleanly (new namespaces / forms we + added that seqrepo never had). Assert refget returns the expected + digest. + + Phase 3 — Minimal get_aliases containment. get_aliases() returns full + alias graphs that differ by design between the two backends + (short-name vs UCSC, insdc, patch versions, legacy namespaces). + For each canonical identifier, we check only that the + refseq: and ga4gh:SQ. entries BOTH proxies emit + are present on both sides — the only alias shapes vrs-python + actually consumes via translate_sequence_identifier. + +Usage: + + SEQREPO_ROOT_DIR=/path/to/seqrepo/2024-12-20 \\ + uv run python misc/refgetstore/compare_against_seqrepo.py + +Exits 0 on full success, 1 on any mismatch. +""" + +from __future__ import annotations + +import logging +import os +import sys +from dataclasses import dataclass +from pathlib import Path + +from ga4gh.vrs.dataproxy import DataProxy, create_dataproxy + +HERE = Path(__file__).resolve().parent +REFGET_STORE_PATH = HERE / "store" + + +@dataclass +class CanonicalInput: + identifier: str + expected_refget_accession: str + expected_refseq: str + expected_length: int + + +# Inputs that BOTH proxies should resolve identically. Expected values come +# from earlier ground-truth spot-checks against seqrepo 2024-12-20. +CANONICAL_INPUTS: list[CanonicalInput] = [ + CanonicalInput("refseq:NC_000001.11", "SQ.Ya6Rs7DHhDeg7YaOSg1EoNi3U_nQ9SvO", "NC_000001.11", 248956422), + CanonicalInput("NC_000001.11", "SQ.Ya6Rs7DHhDeg7YaOSg1EoNi3U_nQ9SvO", "NC_000001.11", 248956422), + CanonicalInput("refseq:NC_000013.11", "SQ._0wi-qoDrvram155UmcSC-zA5ZK4fpLT", "NC_000013.11", 114364328), + CanonicalInput("refseq:NC_000023.11", "SQ.w0WZEvgJF0zf_P4yyTzjjv9oW1z61HHP", "NC_000023.11", 156040895), + CanonicalInput("refseq:NC_012920.1", "SQ.k3grVkjY-hoWcCUojHw6VU6GE3MZ8Sct", "NC_012920.1", 16569), + CanonicalInput("GRCh38:chr1", "SQ.Ya6Rs7DHhDeg7YaOSg1EoNi3U_nQ9SvO", "NC_000001.11", 248956422), + CanonicalInput("GRCh38:chrM", "SQ.k3grVkjY-hoWcCUojHw6VU6GE3MZ8Sct", "NC_012920.1", 16569), + CanonicalInput("GRCh38:chr22", "SQ.7B7SHsmchAR0dFcDCuSFjJAo7tX87krQ", "NC_000022.11", 50818468), + CanonicalInput("GRCh38:chrX", "SQ.w0WZEvgJF0zf_P4yyTzjjv9oW1z61HHP", "NC_000023.11", 156040895), + CanonicalInput("GRCh37:chr1", "SQ.S_KjnFVz-FE7M0W6yoaUDgYxLPc1jyWU", "NC_000001.10", 249250621), + CanonicalInput("ga4gh:SQ.Ya6Rs7DHhDeg7YaOSg1EoNi3U_nQ9SvO", + "SQ.Ya6Rs7DHhDeg7YaOSg1EoNi3U_nQ9SvO", "NC_000001.11", 248956422), +] + + +@dataclass +class RefgetOnlyInput: + identifier: str + expected_refget_accession: str + expected_length: int + + +# Inputs where seqrepo raises KeyError today but refget should resolve +# cleanly because we deliberately populated the relevant namespace. +# +# Bare ``SQ.`` is deliberately NOT tested here. VRS models store +# refgetAccession as ``SQ.`` internally, but every vrs-python code +# path that reads a refgetAccession and passes it to the dataproxy +# prepends ``ga4gh:`` first (see normalize.py:123, translator.py:453, +# hgvs_tools.py:223). Seqrepo has always been broken on bare ``SQ.`` +# inputs (``coerce_namespace`` raises ``ValueError``) and no caller +# constructs that form, so handling it is out of scope. +REFGET_ONLY_INPUTS: list[RefgetOnlyInput] = [ + # seqrepo 2024-12-20 does not carry a GRCh38.p14 namespace + RefgetOnlyInput("GRCh38.p14:chr1", "SQ.Ya6Rs7DHhDeg7YaOSg1EoNi3U_nQ9SvO", 248956422), + # GRCh37.p13 IS in seqrepo (check GRCh37.p13:chr1) — this one is a sanity + # check that our store also covers it. + RefgetOnlyInput("GRCh37.p13:chr1", "SQ.S_KjnFVz-FE7M0W6yoaUDgYxLPc1jyWU", 249250621), + # GenBank cross-assembly namespace — seqrepo has no `insdc` namespace + RefgetOnlyInput("insdc:CM000663.2", "SQ.Ya6Rs7DHhDeg7YaOSg1EoNi3U_nQ9SvO", 248956422), + # sha512t24u: prefix form — seqrepo also stores this as a real alias + # row, but running it through refget alone keeps the phase test + # simple and still exercises the _extract_digest prefix branch. + RefgetOnlyInput("sha512t24u:Ya6Rs7DHhDeg7YaOSg1EoNi3U_nQ9SvO", + "SQ.Ya6Rs7DHhDeg7YaOSg1EoNi3U_nQ9SvO", 248956422), +] + + +class Report: + def __init__(self) -> None: + self.passed = 0 + self.failed = 0 + + def check(self, name: str, ok: bool, details: str = "") -> bool: + mark = "PASS" if ok else "FAIL" + line = f" [{mark}] {name}" + if details: + line += f" — {details}" + print(line) + if ok: + self.passed += 1 + else: + self.failed += 1 + return ok + + +def try_call(fn, *args, **kwargs): + try: + return fn(*args, **kwargs), None + except Exception as exc: # noqa: BLE001 + return None, f"{type(exc).__name__}: {exc}" + + +def phase1_equivalence( + seq_dp: DataProxy, ref_dp: DataProxy, r: Report +) -> None: + print("\n=== Phase 1: functional equivalence on shared inputs ===") + for case in CANONICAL_INPUTS: + ident = case.identifier + print(f"\n-- {ident!r}") + + # derive_refget_accession: both must return the known digest. + s_acc, _ = try_call(seq_dp.derive_refget_accession, ident) + r_acc, _ = try_call(ref_dp.derive_refget_accession, ident) + r.check( + "derive_refget_accession agrees with expected", + s_acc == case.expected_refget_accession + and r_acc == case.expected_refget_accession, + f"expected={case.expected_refget_accession!r} seqrepo={s_acc!r} refget={r_acc!r}", + ) + + # translate ns=ga4gh + s_ga, _ = try_call(seq_dp.translate_sequence_identifier, ident, "ga4gh") + r_ga, _ = try_call(ref_dp.translate_sequence_identifier, ident, "ga4gh") + expected_ga = {f"ga4gh:{case.expected_refget_accession}"} + r.check( + "translate ns=ga4gh agrees with expected", + set(s_ga or []) == expected_ga and set(r_ga or []) == expected_ga, + f"seqrepo={sorted(s_ga or [])} refget={sorted(r_ga or [])}", + ) + + # translate ns=refseq + s_rs, _ = try_call(seq_dp.translate_sequence_identifier, ident, "refseq") + r_rs, _ = try_call(ref_dp.translate_sequence_identifier, ident, "refseq") + expected_rs = {f"refseq:{case.expected_refseq}"} + r.check( + "translate ns=refseq agrees with expected", + set(s_rs or []) == expected_rs and set(r_rs or []) == expected_rs, + f"seqrepo={sorted(s_rs or [])} refget={sorted(r_rs or [])}", + ) + + # get_sequence_length + s_len, _ = try_call(seq_dp.get_sequence_length, ident) + r_len, _ = try_call(ref_dp.get_sequence_length, ident) + r.check( + "get_sequence_length agrees with expected", + s_len == case.expected_length and r_len == case.expected_length, + f"expected={case.expected_length} seqrepo={s_len} refget={r_len}", + ) + + +def phase2_refget_extensions(ref_dp: DataProxy, r: Report) -> None: + print("\n=== Phase 2: refget-only capability extensions ===") + for case in REFGET_ONLY_INPUTS: + ident = case.identifier + print(f"\n-- {ident!r}") + r_acc, r_err = try_call(ref_dp.derive_refget_accession, ident) + r.check( + "derive_refget_accession", + r_acc == case.expected_refget_accession, + f"expected={case.expected_refget_accession!r} got={r_acc!r} err={r_err}", + ) + r_len, r_err = try_call(ref_dp.get_sequence_length, ident) + r.check( + "get_sequence_length", + r_len == case.expected_length, + f"expected={case.expected_length} got={r_len} err={r_err}", + ) + + +def phase3_alias_containment( + seq_dp: DataProxy, ref_dp: DataProxy, r: Report +) -> None: + print("\n=== Phase 3: get_aliases containment for refseq: and ga4gh: ===") + for case in CANONICAL_INPUTS: + ident = case.identifier + print(f"\n-- {ident!r}") + s_all, _ = try_call(seq_dp.get_aliases, ident) + r_all, _ = try_call(ref_dp.get_aliases, ident) + s_set = set(s_all or []) + r_set = set(r_all or []) + expected_ga = f"ga4gh:{case.expected_refget_accession}" + expected_rs = f"refseq:{case.expected_refseq}" + r.check( + "ga4gh: present on both sides", + expected_ga in s_set and expected_ga in r_set, + f"seqrepo={expected_ga in s_set} refget={expected_ga in r_set}", + ) + r.check( + "refseq: present on both sides", + expected_rs in s_set and expected_rs in r_set, + f"seqrepo={expected_rs in s_set} refget={expected_rs in r_set}", + ) + + +def main() -> int: + # Suppress noisy DataProxy stack traces from seqrepo KeyErrors — they're + # expected for phase 2 inputs and the comparison script handles them. + logging.getLogger("ga4gh.vrs.dataproxy").setLevel(logging.CRITICAL) + + seqrepo_root = os.environ.get("SEQREPO_ROOT_DIR") + if not seqrepo_root: + print("ERROR: SEQREPO_ROOT_DIR not set", file=sys.stderr) + return 2 + if not REFGET_STORE_PATH.exists(): + print(f"ERROR: refget store not found at {REFGET_STORE_PATH}", file=sys.stderr) + return 2 + + print(f"seqrepo: {seqrepo_root}") + print(f"refget: {REFGET_STORE_PATH}") + + seq_dp = create_dataproxy(f"seqrepo+file://{seqrepo_root}") + ref_dp = create_dataproxy(f"refget+file://{REFGET_STORE_PATH}") + + r = Report() + phase1_equivalence(seq_dp, ref_dp, r) + phase2_refget_extensions(ref_dp, r) + phase3_alias_containment(seq_dp, ref_dp, r) + + print() + print(f"{r.passed} passed, {r.failed} failed") + print("OVERALL:", "EQUIVALENT" if r.failed == 0 else "DIVERGENT") + return 0 if r.failed == 0 else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/misc/refgetstore/diagnose_ensembl_mismatches.py b/misc/refgetstore/diagnose_ensembl_mismatches.py new file mode 100644 index 00000000..97f4026f --- /dev/null +++ b/misc/refgetstore/diagnose_ensembl_mismatches.py @@ -0,0 +1,386 @@ +#!/usr/bin/env python +"""Diagnose Ensembl digest mismatches between seqrepo and refgetstore. + +A cross-check of the ``ensembl`` namespace found 116 aliases where seqrepo's +stored ``seq_id`` differs from refgetstore's ``sha512t24u`` digest. This +script investigates the root cause by: + + Phase A — Enumerate all mismatches between the two alias tables. + Phase B — For each mismatch, fetch the full sequence from both stores + and compare byte-by-byte. + Phase C — Classify the root cause (``*`` normalization, casing, etc.) + and determine which digest is correct per GA4GH VRS spec. + Phase D — Summarize findings and write a JSON report. + +Usage: + + SEQREPO_ROOT_DIR=/path/to/seqrepo/2024-12-20 \\ + uv run python misc/refgetstore/diagnose_ensembl_mismatches.py \\ + [--refget-store misc/refgetstore/store] \\ + [--report misc/refgetstore/diagnosis_report.json] + +Exits 0 on success (report written), 1 on setup error. +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +import time +from dataclasses import asdict, dataclass, field +from pathlib import Path + +from biocommons.seqrepo import SeqRepo +from bioutils.digests import seq_seqhash +from ga4gh.core.digests import sha512t24u +from gtars.refget import RefgetStore + +HERE = Path(__file__).resolve().parent +DEFAULT_STORE_PATH = HERE / "store" + + +@dataclass +class MismatchRecord: + """A single ensembl alias where seqrepo and refgetstore digests differ.""" + + accession: str + seqrepo_seq_id: str + refget_sha512t24u: str + refget_length: int + # Populated during Phase B + sequences_identical: bool | None = None + seqrepo_seq_length: int | None = None + refget_seq_length: int | None = None + has_stop_codon: bool | None = None + stop_codon_positions: list[int] = field(default_factory=list) + # Digest computed from seqrepo's fetched bytes (no normalization) + seqrepo_raw_digest: str | None = None + # Digest computed from refgetstore's fetched bytes (no normalization) + refget_raw_digest: str | None = None + # Whether seqrepo's stored seq_id matches seq_seqhash (normalized, * stripped) + seqrepo_digest_matches_normalized: bool | None = None + # Whether refgetstore's stored digest matches sha512t24u of its own bytes + refget_digest_self_consistent: bool | None = None + # Classification + cause: str = "" # "star_normalization" or "sequence_divergence" + # Error during fetch + fetch_error: str | None = None + + +@dataclass +class DiagnosisReport: + """Overall report of the investigation.""" + + total_ensembl_aliases_checked: int = 0 + aliases_both_resolve: int = 0 + aliases_digests_agree: int = 0 + aliases_digests_mismatch: int = 0 + aliases_only_in_refget: int = 0 + aliases_only_in_seqrepo: int = 0 + mismatches: list[MismatchRecord] = field(default_factory=list) + # Phase C classification + all_mismatches_are_star_normalization: bool | None = None + root_cause: str = "" + recommendation: str = "" + + +def get_seqrepo_seq_ids(sr: SeqRepo) -> dict[str, str]: + """Map ensembl aliases to seqrepo seq_ids. + + Returns {alias: seq_id} for all current Ensembl aliases. + """ + conn = sr.aliases._db # noqa: SLF001 + cursor = conn.execute( + "SELECT alias, seq_id FROM seqalias " + "WHERE namespace = 'Ensembl' AND is_current = 1" + ) + result: dict[str, str] = {} + for alias, seq_id in cursor: + result[alias] = seq_id + return result + + +def get_refget_ensembl_digests(store_path: Path) -> dict[str, str]: + """Map ensembl aliases to refgetstore sha512t24u digests. + + Reads the ensembl.tsv alias sidecar directly for bulk access. + Format is ``alias\\tdigest`` per line. + + Returns {alias: sha512t24u} for all ensembl namespace aliases. + """ + tsv_path = store_path / "aliases" / "sequences" / "ensembl.tsv" + result: dict[str, str] = {} + with tsv_path.open("r") as f: + for line in f: + line = line.rstrip("\n\r") + if not line: + continue + parts = line.split("\t", 1) + if len(parts) == 2: + result[parts[0]] = parts[1] + return result + + +def phase_a( + sr: SeqRepo, store: RefgetStore, store_path: Path, report: DiagnosisReport +) -> list[MismatchRecord]: + """Phase A: Enumerate all mismatches between seqrepo and refgetstore.""" + print("\n=== Phase A: Enumerating ensembl alias mismatches ===") + + sr_aliases = get_seqrepo_seq_ids(sr) + rg_aliases = get_refget_ensembl_digests(store_path) + + sr_keys = set(sr_aliases) + rg_keys = set(rg_aliases) + common = sr_keys & rg_keys + + report.total_ensembl_aliases_checked = len(sr_keys | rg_keys) + report.aliases_only_in_seqrepo = len(sr_keys - rg_keys) + report.aliases_only_in_refget = len(rg_keys - sr_keys) + report.aliases_both_resolve = len(common) + + mismatches: list[MismatchRecord] = [] + agree_count = 0 + for alias in sorted(common): + sr_id = sr_aliases[alias] + rg_id = rg_aliases[alias] + if sr_id == rg_id: + agree_count += 1 + else: + # Resolve length from refgetstore metadata + rec = store.get_sequence_by_alias("ensembl", alias) + length = rec.metadata.length if rec else -1 + mismatches.append( + MismatchRecord( + accession=alias, + seqrepo_seq_id=sr_id, + refget_sha512t24u=rg_id, + refget_length=length, + ) + ) + + report.aliases_digests_agree = agree_count + report.aliases_digests_mismatch = len(mismatches) + report.mismatches = mismatches + + print(f" Total aliases checked: {report.total_ensembl_aliases_checked}") + print(f" Both resolve, agree: {agree_count}") + print(f" Both resolve, MISMATCH: {len(mismatches)}") + print(f" Only in seqrepo: {report.aliases_only_in_seqrepo}") + print(f" Only in refgetstore: {report.aliases_only_in_refget}") + + # Breakdown by prefix + enst_mm = sum(1 for m in mismatches if m.accession.startswith("ENST")) + ensp_mm = sum(1 for m in mismatches if m.accession.startswith("ENSP")) + print(f" ENST mismatches: {enst_mm}") + print(f" ENSP mismatches: {ensp_mm}") + + return mismatches + + +def phase_b( + sr: SeqRepo, store: RefgetStore, mismatches: list[MismatchRecord] +) -> None: + """Phase B: Fetch full sequences from both stores and compare byte-by-byte.""" + print(f"\n=== Phase B: Byte-level comparison of {len(mismatches)} mismatches ===") + + for i, mm in enumerate(mismatches): + # Fetch from seqrepo + try: + sr_seq: str = sr.fetch_uri(f"Ensembl:{mm.accession}") + except Exception as exc: # noqa: BLE001 + mm.fetch_error = f"seqrepo fetch failed: {exc}" + print(f" [{i+1}/{len(mismatches)}] {mm.accession}: {mm.fetch_error}") + continue + + # Fetch from refgetstore + try: + store.load_sequence(mm.refget_sha512t24u) + rg_seq: str = store.get_substring( + mm.refget_sha512t24u, 0, mm.refget_length + ) + except Exception as exc: # noqa: BLE001 + mm.fetch_error = f"refgetstore fetch failed: {exc}" + print(f" [{i+1}/{len(mismatches)}] {mm.accession}: {mm.fetch_error}") + continue + + mm.seqrepo_seq_length = len(sr_seq) + mm.refget_seq_length = len(rg_seq) + mm.sequences_identical = sr_seq == rg_seq + + # Check for stop codon * + mm.has_stop_codon = "*" in sr_seq or "*" in rg_seq + mm.stop_codon_positions = [j for j, c in enumerate(sr_seq) if c == "*"] + + # Compute sha512t24u from each side's raw bytes (no normalization) + mm.seqrepo_raw_digest = sha512t24u(sr_seq.encode("ascii")) + mm.refget_raw_digest = sha512t24u(rg_seq.encode("ascii")) + + # Check if seqrepo's stored seq_id matches seq_seqhash (which strips *) + normalized_digest: str = seq_seqhash(sr_seq, normalize=True) + mm.seqrepo_digest_matches_normalized = mm.seqrepo_seq_id == normalized_digest + + # Check if refgetstore's stored digest is self-consistent with its own bytes + mm.refget_digest_self_consistent = ( + mm.refget_sha512t24u == mm.refget_raw_digest + ) + + # Classify the cause + if mm.sequences_identical and mm.has_stop_codon: + mm.cause = "star_normalization" + elif not mm.sequences_identical: + mm.cause = "sequence_divergence" + else: + mm.cause = "unknown" + + if (i + 1) % 20 == 0 or (i + 1) == len(mismatches): + print(f" Processed {i+1}/{len(mismatches)}") + + +def phase_c(report: DiagnosisReport) -> None: + """Phase C: Classify root cause across all mismatches.""" + print("\n=== Phase C: Root cause classification ===") + + mismatches = report.mismatches + if not mismatches: + print(" No mismatches to classify.") + return + + # Count by cause + by_cause: dict[str, list[MismatchRecord]] = {} + for m in mismatches: + by_cause.setdefault(m.cause, []).append(m) + n_fetch_errors = sum(1 for m in mismatches if m.fetch_error is not None) + + total = len(mismatches) + n_star = len(by_cause.get("star_normalization", [])) + n_diverge = len(by_cause.get("sequence_divergence", [])) + n_unknown = len(by_cause.get("unknown", [])) + n_sr_self = sum(1 for m in mismatches if m.seqrepo_digest_matches_normalized) + n_rg_self = sum(1 for m in mismatches if m.refget_digest_self_consistent) + + print(f" Total mismatches: {total}") + print(f" Cause: * normalization (same bytes): {n_star}") + print(f" Cause: sequence divergence (diff bytes):{n_diverge}") + print(f" Cause: unknown: {n_unknown}") + print(f" Fetch errors: {n_fetch_errors}") + print(f" seqrepo id matches normalized digest: {n_sr_self}/{total}") + print(f" refget id self-consistent: {n_rg_self}/{total}") + + # Show divergence details + if n_diverge > 0: + print(f"\n Sequence divergence details:") + for m in by_cause["sequence_divergence"]: + print( + f" {m.accession}: " + f"seqrepo={m.seqrepo_seq_length}bp refget={m.refget_seq_length}bp " + f"(different content across Ensembl releases)" + ) + + report.all_mismatches_are_star_normalization = n_star == total + + report.root_cause = ( + f"{n_star}/{total} mismatches are caused by " + "bioutils.sequences.normalize_sequence() stripping '*' (stop codon) " + "characters before computing the sha512t24u digest. SeqRepo uses this " + "normalized digest as its seq_id (and thus its ga4gh:SQ.* alias), while " + "gtars RefgetStore computes sha512t24u directly from the raw FASTA bytes " + "including '*'. The underlying sequence bytes are identical in both stores " + "for these cases." + ) + if n_diverge > 0: + diverged = [m.accession for m in by_cause["sequence_divergence"]] + report.root_cause += ( + f" The remaining {n_diverge} mismatch(es) ({', '.join(diverged)}) " + "are genuine sequence content differences between seqrepo's Ensembl " + "load (from an older release) and refgetstore's Ensembl r113." + ) + + report.recommendation = ( + "The refgetstore digest is correct per the GA4GH VRS specification, " + "which defines the sequence digest as sha512t24u over the raw sequence " + "bytes without normalization. SeqRepo's digest for *-containing proteins " + "is technically non-conformant. Document the divergence and list the " + "affected accessions. For the sequence-divergent transcript(s), the " + "refgetstore carries the r113 version which is authoritative for that release." + ) + + print(f"\n Root cause: * normalization ({n_star}) + sequence divergence ({n_diverge})") + + +def phase_d(report: DiagnosisReport, report_path: Path) -> None: + """Phase D: Write summary and JSON report.""" + print(f"\n=== Phase D: Summary ===") + print(f" Root cause: {report.root_cause[:120]}...") + print(f" Recommendation: {report.recommendation[:120]}...") + + # Show a few examples + print("\n Sample mismatches:") + for mm in report.mismatches[:8]: + stars = len(mm.stop_codon_positions) + print( + f" {mm.accession:30s} " + f"len={mm.refget_length:>5d} " + f"stars={stars:>2d} " + f"cause={mm.cause}" + ) + + # Write JSON report + output = asdict(report) + report_path.parent.mkdir(parents=True, exist_ok=True) + with report_path.open("w") as f: + json.dump(output, f, indent=2, default=str) + print(f"\n Report written to {report_path}") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--refget-store", + type=Path, + default=DEFAULT_STORE_PATH, + help="Path to refgetstore directory (default: %(default)s)", + ) + parser.add_argument( + "--report", + type=Path, + default=HERE / "diagnosis_report.json", + help="Path for JSON report output (default: %(default)s)", + ) + args = parser.parse_args() + + seqrepo_root = os.environ.get("SEQREPO_ROOT_DIR") + if not seqrepo_root: + print("ERROR: SEQREPO_ROOT_DIR not set", file=sys.stderr) + return 1 + if not args.refget_store.exists(): + print( + f"ERROR: refget store not found at {args.refget_store}", + file=sys.stderr, + ) + return 1 + + print(f"seqrepo: {seqrepo_root}") + print(f"refgetstore: {args.refget_store}") + + sr = SeqRepo(seqrepo_root) + store = RefgetStore.open_local(str(args.refget_store)) + store.pull_aliases() + + report = DiagnosisReport() + + t0 = time.monotonic() + mismatches = phase_a(sr, store, args.refget_store, report) + phase_b(sr, store, mismatches) + phase_c(report) + phase_d(report, args.report) + elapsed = time.monotonic() - t0 + + print(f"\nCompleted in {elapsed:.1f}s") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/misc/refgetstore/diagnosis_report.json b/misc/refgetstore/diagnosis_report.json new file mode 100644 index 00000000..414a60cd --- /dev/null +++ b/misc/refgetstore/diagnosis_report.json @@ -0,0 +1,2508 @@ +{ + "total_ensembl_aliases_checked": 600587, + "aliases_both_resolve": 322402, + "aliases_digests_agree": 322286, + "aliases_digests_mismatch": 116, + "aliases_only_in_refget": 212352, + "aliases_only_in_seqrepo": 65833, + "mismatches": [ + { + "accession": "ENSP00000340011.8", + "seqrepo_seq_id": "fv4uQW4AfrwGj3dx9D4xfpccU7Qnqtia", + "refget_sha512t24u": "8Co4fICavUFj9qH_GGXd5cbWbzh8oSZ0", + "refget_length": 297, + "sequences_identical": true, + "seqrepo_seq_length": 297, + "refget_seq_length": 297, + "has_stop_codon": true, + "stop_codon_positions": [ + 239, + 269, + 280 + ], + "seqrepo_raw_digest": "8Co4fICavUFj9qH_GGXd5cbWbzh8oSZ0", + "refget_raw_digest": "8Co4fICavUFj9qH_GGXd5cbWbzh8oSZ0", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000375609.1", + "seqrepo_seq_id": "0wFdmvBugnimf4tOFjCVdckpw3P6GmOx", + "refget_sha512t24u": "Qg7FS7qyO-pfHp7HYd4wvr26DEx1oov4", + "refget_length": 225, + "sequences_identical": true, + "seqrepo_seq_length": 225, + "refget_seq_length": 225, + "has_stop_codon": true, + "stop_codon_positions": [ + 222 + ], + "seqrepo_raw_digest": "Qg7FS7qyO-pfHp7HYd4wvr26DEx1oov4", + "refget_raw_digest": "Qg7FS7qyO-pfHp7HYd4wvr26DEx1oov4", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000385765.6", + "seqrepo_seq_id": "lV7Y-VqQL6dNPJ2ZKWTqw46p_da8Eodm", + "refget_sha512t24u": "YL07fWvAjY43qO_0ZnVw5BxCvo_M_0x-", + "refget_length": 230, + "sequences_identical": true, + "seqrepo_seq_length": 230, + "refget_seq_length": 230, + "has_stop_codon": true, + "stop_codon_positions": [ + 181 + ], + "seqrepo_raw_digest": "YL07fWvAjY43qO_0ZnVw5BxCvo_M_0x-", + "refget_raw_digest": "YL07fWvAjY43qO_0ZnVw5BxCvo_M_0x-", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000402491.1", + "seqrepo_seq_id": "e6sQaY-e_N_HsVyuY3ymiFysYBMBFk9W", + "refget_sha512t24u": "dBxeQN4GInX8eFSsicpYD8N5Nn_0My2g", + "refget_length": 205, + "sequences_identical": true, + "seqrepo_seq_length": 205, + "refget_seq_length": 205, + "has_stop_codon": true, + "stop_codon_positions": [ + 61 + ], + "seqrepo_raw_digest": "dBxeQN4GInX8eFSsicpYD8N5Nn_0My2g", + "refget_raw_digest": "dBxeQN4GInX8eFSsicpYD8N5Nn_0My2g", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000411913.1", + "seqrepo_seq_id": "_RbUr7Oq9qE22xU9fnMC2uLA8RIp2Hri", + "refget_sha512t24u": "TQ93TCrFm5e-mb-tkSHz3d_DYaAlo4i3", + "refget_length": 331, + "sequences_identical": true, + "seqrepo_seq_length": 331, + "refget_seq_length": 331, + "has_stop_codon": true, + "stop_codon_positions": [ + 105 + ], + "seqrepo_raw_digest": "TQ93TCrFm5e-mb-tkSHz3d_DYaAlo4i3", + "refget_raw_digest": "TQ93TCrFm5e-mb-tkSHz3d_DYaAlo4i3", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000421815.1", + "seqrepo_seq_id": "tM9WHP2MFnlP7TaVWy-zF3pASsoLzDdU", + "refget_sha512t24u": "PuFuh2fJaeBhwf_CIXb9c4n-5HHCkIA8", + "refget_length": 226, + "sequences_identical": true, + "seqrepo_seq_length": 226, + "refget_seq_length": 226, + "has_stop_codon": true, + "stop_codon_positions": [ + 124 + ], + "seqrepo_raw_digest": "PuFuh2fJaeBhwf_CIXb9c4n-5HHCkIA8", + "refget_raw_digest": "PuFuh2fJaeBhwf_CIXb9c4n-5HHCkIA8", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000422220.1", + "seqrepo_seq_id": "o6IopMWmnKAE7FUNdEFy2TnsyBDa-swp", + "refget_sha512t24u": "2ludFB9wsyqzj2rD-ba2o2sFeXQIa277", + "refget_length": 162, + "sequences_identical": true, + "seqrepo_seq_length": 162, + "refget_seq_length": 162, + "has_stop_codon": true, + "stop_codon_positions": [ + 148 + ], + "seqrepo_raw_digest": "2ludFB9wsyqzj2rD-ba2o2sFeXQIa277", + "refget_raw_digest": "2ludFB9wsyqzj2rD-ba2o2sFeXQIa277", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000423754.1", + "seqrepo_seq_id": "K56fM_BMAirMsSULhhLEc2yBkgvEE_aZ", + "refget_sha512t24u": "kw3yUvbNK6RIftnKndZqroj4_OZxeNl9", + "refget_length": 298, + "sequences_identical": true, + "seqrepo_seq_length": 298, + "refget_seq_length": 298, + "has_stop_codon": true, + "stop_codon_positions": [ + 284 + ], + "seqrepo_raw_digest": "kw3yUvbNK6RIftnKndZqroj4_OZxeNl9", + "refget_raw_digest": "kw3yUvbNK6RIftnKndZqroj4_OZxeNl9", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000423899.1", + "seqrepo_seq_id": "RDSLXXiECCksD3fd4HEjkDRDUojB3zm1", + "refget_sha512t24u": "D9gmmANCDpy0XX2JYSF7ZeHkXVaaDAm6", + "refget_length": 224, + "sequences_identical": true, + "seqrepo_seq_length": 224, + "refget_seq_length": 224, + "has_stop_codon": true, + "stop_codon_positions": [ + 124 + ], + "seqrepo_raw_digest": "D9gmmANCDpy0XX2JYSF7ZeHkXVaaDAm6", + "refget_raw_digest": "D9gmmANCDpy0XX2JYSF7ZeHkXVaaDAm6", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000423970.1", + "seqrepo_seq_id": "fOlJtYkBTNNq6H0kRnKsGbSqtO5PcFEr", + "refget_sha512t24u": "96AGBZz3OkRXlz54m0a5vf04XVfn8MG0", + "refget_length": 295, + "sequences_identical": true, + "seqrepo_seq_length": 295, + "refget_seq_length": 295, + "has_stop_codon": true, + "stop_codon_positions": [ + 124 + ], + "seqrepo_raw_digest": "96AGBZz3OkRXlz54m0a5vf04XVfn8MG0", + "refget_raw_digest": "96AGBZz3OkRXlz54m0a5vf04XVfn8MG0", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000424038.1", + "seqrepo_seq_id": "lOkFBmBx4LkR4GtP4ljxF4ssKcMpU9S2", + "refget_sha512t24u": "sNGrVi9jJokpvjhGaAkw9-5mCjrkxR5s", + "refget_length": 341, + "sequences_identical": true, + "seqrepo_seq_length": 341, + "refget_seq_length": 341, + "has_stop_codon": true, + "stop_codon_positions": [ + 124 + ], + "seqrepo_raw_digest": "sNGrVi9jJokpvjhGaAkw9-5mCjrkxR5s", + "refget_raw_digest": "sNGrVi9jJokpvjhGaAkw9-5mCjrkxR5s", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000425652.1", + "seqrepo_seq_id": "0UyEhk9QGqb2gKNUMpHZ93nXVAXK9yTO", + "refget_sha512t24u": "JQU8ZqYvYRuVO19ZDaHQhHM89PyidybZ", + "refget_length": 277, + "sequences_identical": true, + "seqrepo_seq_length": 277, + "refget_seq_length": 277, + "has_stop_codon": true, + "stop_codon_positions": [ + 124 + ], + "seqrepo_raw_digest": "JQU8ZqYvYRuVO19ZDaHQhHM89PyidybZ", + "refget_raw_digest": "JQU8ZqYvYRuVO19ZDaHQhHM89PyidybZ", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000426427.1", + "seqrepo_seq_id": "03nXBcwbbJQPYE_5LurPSoCpnsikUdEn", + "refget_sha512t24u": "pO53MQMJFDxQT6bOb7ey9est-55eH5NH", + "refget_length": 140, + "sequences_identical": true, + "seqrepo_seq_length": 140, + "refget_seq_length": 140, + "has_stop_codon": true, + "stop_codon_positions": [ + 40 + ], + "seqrepo_raw_digest": "pO53MQMJFDxQT6bOb7ey9est-55eH5NH", + "refget_raw_digest": "pO53MQMJFDxQT6bOb7ey9est-55eH5NH", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000426566.1", + "seqrepo_seq_id": "_nkNH8COTskYyNkCoTzIRNHzENVylj1y", + "refget_sha512t24u": "egHuRL5C7VrWvOx3FvqbEayoBNIeC6ZM", + "refget_length": 257, + "sequences_identical": true, + "seqrepo_seq_length": 257, + "refget_seq_length": 257, + "has_stop_codon": true, + "stop_codon_positions": [ + 40 + ], + "seqrepo_raw_digest": "egHuRL5C7VrWvOx3FvqbEayoBNIeC6ZM", + "refget_raw_digest": "egHuRL5C7VrWvOx3FvqbEayoBNIeC6ZM", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000426627.1", + "seqrepo_seq_id": "OpwKkqOENTdevZ_CrLEOVumim2QTctaK", + "refget_sha512t24u": "2KlIfbXxNKZoNImdwrmKBdRYXPkKL_2G", + "refget_length": 323, + "sequences_identical": true, + "seqrepo_seq_length": 323, + "refget_seq_length": 323, + "has_stop_codon": true, + "stop_codon_positions": [ + 56 + ], + "seqrepo_raw_digest": "2KlIfbXxNKZoNImdwrmKBdRYXPkKL_2G", + "refget_raw_digest": "2KlIfbXxNKZoNImdwrmKBdRYXPkKL_2G", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000427437.1", + "seqrepo_seq_id": "wWUGUKB2gnrfgmMm0qg34Iy9mUEhx4ju", + "refget_sha512t24u": "7KGhHMvva3lM9749_TxBRCl5Oz8YcdKo", + "refget_length": 142, + "sequences_identical": true, + "seqrepo_seq_length": 142, + "refget_seq_length": 142, + "has_stop_codon": true, + "stop_codon_positions": [ + 40 + ], + "seqrepo_raw_digest": "7KGhHMvva3lM9749_TxBRCl5Oz8YcdKo", + "refget_raw_digest": "7KGhHMvva3lM9749_TxBRCl5Oz8YcdKo", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000427458.1", + "seqrepo_seq_id": "rFjG0Bt4bfEH09x5PaR9jZGd03wmgE6O", + "refget_sha512t24u": "-x9lIom6XRVzSbgQDyNb0hrandBbttoV", + "refget_length": 469, + "sequences_identical": true, + "seqrepo_seq_length": 469, + "refget_seq_length": 469, + "has_stop_codon": true, + "stop_codon_positions": [ + 455 + ], + "seqrepo_raw_digest": "-x9lIom6XRVzSbgQDyNb0hrandBbttoV", + "refget_raw_digest": "-x9lIom6XRVzSbgQDyNb0hrandBbttoV", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000427969.1", + "seqrepo_seq_id": "y9mM4wkXZleVNSyYRPm83MhQd47vpV0A", + "refget_sha512t24u": "Ah8IodVoXqfHk4DDt8jsFGSic0fRZYU0", + "refget_length": 10, + "sequences_identical": true, + "seqrepo_seq_length": 10, + "refget_seq_length": 10, + "has_stop_codon": true, + "stop_codon_positions": [ + 3, + 6 + ], + "seqrepo_raw_digest": "Ah8IodVoXqfHk4DDt8jsFGSic0fRZYU0", + "refget_raw_digest": "Ah8IodVoXqfHk4DDt8jsFGSic0fRZYU0", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000428366.1", + "seqrepo_seq_id": "SHyk_qyvZUk1fYzNdSTs6Drv8jS87r_C", + "refget_sha512t24u": "CN6dOnzsnNZiifsD2mNJ941p8KVRVPJu", + "refget_length": 12, + "sequences_identical": true, + "seqrepo_seq_length": 12, + "refget_seq_length": 12, + "has_stop_codon": true, + "stop_codon_positions": [ + 2 + ], + "seqrepo_raw_digest": "CN6dOnzsnNZiifsD2mNJ941p8KVRVPJu", + "refget_raw_digest": "CN6dOnzsnNZiifsD2mNJ941p8KVRVPJu", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000428393.1", + "seqrepo_seq_id": "qRfoOZ40q93mvxVq3MDnu2rMZt2mB_ob", + "refget_sha512t24u": "yAZ-3aLns9fo2SxCNtvZJGjw_9-p67nz", + "refget_length": 5, + "sequences_identical": true, + "seqrepo_seq_length": 5, + "refget_seq_length": 5, + "has_stop_codon": true, + "stop_codon_positions": [ + 0, + 3 + ], + "seqrepo_raw_digest": "yAZ-3aLns9fo2SxCNtvZJGjw_9-p67nz", + "refget_raw_digest": "yAZ-3aLns9fo2SxCNtvZJGjw_9-p67nz", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000428616.1", + "seqrepo_seq_id": "KRLK6ZBMvRpTakgKIIWxTzvHu6nZtEKQ", + "refget_sha512t24u": "5ZDMzPF5f4u5xiuBDA2ByfCYoG4534KM", + "refget_length": 10, + "sequences_identical": true, + "seqrepo_seq_length": 10, + "refget_seq_length": 10, + "has_stop_codon": true, + "stop_codon_positions": [ + 4 + ], + "seqrepo_raw_digest": "5ZDMzPF5f4u5xiuBDA2ByfCYoG4534KM", + "refget_raw_digest": "5ZDMzPF5f4u5xiuBDA2ByfCYoG4534KM", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000429324.1", + "seqrepo_seq_id": "m19OoHXspP1j74FYLLR0Fon_Tca6IwAP", + "refget_sha512t24u": "fn4oRtmkRm5ikN4QPXXg0RdDOzaLpvUH", + "refget_length": 9, + "sequences_identical": true, + "seqrepo_seq_length": 9, + "refget_seq_length": 9, + "has_stop_codon": true, + "stop_codon_positions": [ + 5 + ], + "seqrepo_raw_digest": "fn4oRtmkRm5ikN4QPXXg0RdDOzaLpvUH", + "refget_raw_digest": "fn4oRtmkRm5ikN4QPXXg0RdDOzaLpvUH", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000429952.1", + "seqrepo_seq_id": "5Ek4xsz2ZS7GfXgv_ZBjOH5FkAP3XMX6", + "refget_sha512t24u": "JCCHtapvgLCCSu0eIJCK34DXB_mOSDrc", + "refget_length": 10, + "sequences_identical": true, + "seqrepo_seq_length": 10, + "refget_seq_length": 10, + "has_stop_codon": true, + "stop_codon_positions": [ + 3, + 4, + 5 + ], + "seqrepo_raw_digest": "JCCHtapvgLCCSu0eIJCK34DXB_mOSDrc", + "refget_raw_digest": "JCCHtapvgLCCSu0eIJCK34DXB_mOSDrc", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000430034.1", + "seqrepo_seq_id": "qRfoOZ40q93mvxVq3MDnu2rMZt2mB_ob", + "refget_sha512t24u": "yAZ-3aLns9fo2SxCNtvZJGjw_9-p67nz", + "refget_length": 5, + "sequences_identical": true, + "seqrepo_seq_length": 5, + "refget_seq_length": 5, + "has_stop_codon": true, + "stop_codon_positions": [ + 0, + 3 + ], + "seqrepo_raw_digest": "yAZ-3aLns9fo2SxCNtvZJGjw_9-p67nz", + "refget_raw_digest": "yAZ-3aLns9fo2SxCNtvZJGjw_9-p67nz", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000430248.1", + "seqrepo_seq_id": "AAWZIb3sgZG4DGzYKPTEdsDRPppA20nl", + "refget_sha512t24u": "ZyuIuWLGyac0hNqMfiw0XIJIjTx1UF2Y", + "refget_length": 6, + "sequences_identical": true, + "seqrepo_seq_length": 6, + "refget_seq_length": 6, + "has_stop_codon": true, + "stop_codon_positions": [ + 0, + 4 + ], + "seqrepo_raw_digest": "ZyuIuWLGyac0hNqMfiw0XIJIjTx1UF2Y", + "refget_raw_digest": "ZyuIuWLGyac0hNqMfiw0XIJIjTx1UF2Y", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000430788.1", + "seqrepo_seq_id": "4nJC7z8ekuHU6kAl4yMiV6qZlrpomr_6", + "refget_sha512t24u": "vQGG6MQqX_mw1KxS82QU0Kuk5KtUItWA", + "refget_length": 10, + "sequences_identical": true, + "seqrepo_seq_length": 10, + "refget_seq_length": 10, + "has_stop_codon": true, + "stop_codon_positions": [ + 3, + 4 + ], + "seqrepo_raw_digest": "vQGG6MQqX_mw1KxS82QU0Kuk5KtUItWA", + "refget_raw_digest": "vQGG6MQqX_mw1KxS82QU0Kuk5KtUItWA", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000431031.1", + "seqrepo_seq_id": "l4SXGqAbFUh7r1jc-ImcNEdp067Fa0MY", + "refget_sha512t24u": "zkX9kDvYwL768RXctEHW7-3lneBHc71d", + "refget_length": 1318, + "sequences_identical": true, + "seqrepo_seq_length": 1318, + "refget_seq_length": 1318, + "has_stop_codon": true, + "stop_codon_positions": [ + 924 + ], + "seqrepo_raw_digest": "zkX9kDvYwL768RXctEHW7-3lneBHc71d", + "refget_raw_digest": "zkX9kDvYwL768RXctEHW7-3lneBHc71d", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000431089.1", + "seqrepo_seq_id": "BM1Hbmp96w4OSkWZ5u_ROCq4tWMf_V69", + "refget_sha512t24u": "hzlOSDnnFBtjxI-qUTYSXVvP8K76MfHk", + "refget_length": 5, + "sequences_identical": true, + "seqrepo_seq_length": 5, + "refget_seq_length": 5, + "has_stop_codon": true, + "stop_codon_positions": [ + 0, + 3 + ], + "seqrepo_raw_digest": "hzlOSDnnFBtjxI-qUTYSXVvP8K76MfHk", + "refget_raw_digest": "hzlOSDnnFBtjxI-qUTYSXVvP8K76MfHk", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000434417.1", + "seqrepo_seq_id": "zKGzcqTwIi1cpsh_25jTR-g7cycsIEpf", + "refget_sha512t24u": "Ih9FQHBcIpxLvdvSmQwf1B_uW07yCbjE", + "refget_length": 2459, + "sequences_identical": true, + "seqrepo_seq_length": 2459, + "refget_seq_length": 2459, + "has_stop_codon": true, + "stop_codon_positions": [ + 2130, + 2153, + 2155, + 2186, + 2207, + 2238, + 2284, + 2292, + 2296, + 2298, + 2314, + 2330, + 2381, + 2386, + 2413, + 2417, + 2418 + ], + "seqrepo_raw_digest": "Ih9FQHBcIpxLvdvSmQwf1B_uW07yCbjE", + "refget_raw_digest": "Ih9FQHBcIpxLvdvSmQwf1B_uW07yCbjE", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000434644.1", + "seqrepo_seq_id": "DZRDVc8gAxzoOkjyLBLK3IcVRKKK46Pw", + "refget_sha512t24u": "v8czIuEX2XTC0M9Dw7Ax3fzb4H8qdgOL", + "refget_length": 1774, + "sequences_identical": true, + "seqrepo_seq_length": 1774, + "refget_seq_length": 1774, + "has_stop_codon": true, + "stop_codon_positions": [ + 1445, + 1468, + 1470, + 1501, + 1522, + 1553, + 1599, + 1607, + 1611, + 1613, + 1629, + 1645, + 1696, + 1701, + 1728, + 1732, + 1733 + ], + "seqrepo_raw_digest": "v8czIuEX2XTC0M9Dw7Ax3fzb4H8qdgOL", + "refget_raw_digest": "v8czIuEX2XTC0M9Dw7Ax3fzb4H8qdgOL", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000438013.1", + "seqrepo_seq_id": "nDmCYyT1EKuf9Z8mrnEgCxgF2Hjn0T_P", + "refget_sha512t24u": "_nPal56b66hTKeKlOajKFSKjKY8DcuU-", + "refget_length": 314, + "sequences_identical": true, + "seqrepo_seq_length": 314, + "refget_seq_length": 314, + "has_stop_codon": true, + "stop_codon_positions": [ + 180, + 191, + 196, + 202 + ], + "seqrepo_raw_digest": "_nPal56b66hTKeKlOajKFSKjKY8DcuU-", + "refget_raw_digest": "_nPal56b66hTKeKlOajKFSKjKY8DcuU-", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000439604.1", + "seqrepo_seq_id": "oaVc-acao8fOMIyVp_H98dcz0BSDIUWH", + "refget_sha512t24u": "aYF_Y_ycp3u3t9Gqeu96GLzc304fQ0zp", + "refget_length": 516, + "sequences_identical": true, + "seqrepo_seq_length": 516, + "refget_seq_length": 516, + "has_stop_codon": true, + "stop_codon_positions": [ + 252, + 256, + 272, + 286, + 291, + 292, + 300, + 367, + 399, + 409, + 424 + ], + "seqrepo_raw_digest": "aYF_Y_ycp3u3t9Gqeu96GLzc304fQ0zp", + "refget_raw_digest": "aYF_Y_ycp3u3t9Gqeu96GLzc304fQ0zp", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000445124.1", + "seqrepo_seq_id": "6VmdTL7nqa7JQpolg-wuBLGwIKBnrP9F", + "refget_sha512t24u": "MIwe_aAViS3zDJuv3mQTnIDVQMSLST8L", + "refget_length": 497, + "sequences_identical": true, + "seqrepo_seq_length": 497, + "refget_seq_length": 497, + "has_stop_codon": true, + "stop_codon_positions": [ + 252, + 256, + 272, + 286, + 291, + 292, + 300, + 348, + 380, + 390, + 405 + ], + "seqrepo_raw_digest": "MIwe_aAViS3zDJuv3mQTnIDVQMSLST8L", + "refget_raw_digest": "MIwe_aAViS3zDJuv3mQTnIDVQMSLST8L", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000451021.2", + "seqrepo_seq_id": "38LKEEE-120Kwsih-vNlwnx8tb2A4ZSx", + "refget_sha512t24u": "_rwlFqSN0jljdGdyuRQXsthR-SSMBJ2G", + "refget_length": 314, + "sequences_identical": true, + "seqrepo_seq_length": 314, + "refget_seq_length": 314, + "has_stop_codon": true, + "stop_codon_positions": [ + 226 + ], + "seqrepo_raw_digest": "_rwlFqSN0jljdGdyuRQXsthR-SSMBJ2G", + "refget_raw_digest": "_rwlFqSN0jljdGdyuRQXsthR-SSMBJ2G", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000452150.1", + "seqrepo_seq_id": "ynPhtGE3u23nkLJNMxVv51pviMYUrBBX", + "refget_sha512t24u": "Ede4Q4UXqG9Dzi4MrnjeH9jp9k0ggu7T", + "refget_length": 21, + "sequences_identical": true, + "seqrepo_seq_length": 21, + "refget_seq_length": 21, + "has_stop_codon": true, + "stop_codon_positions": [ + 1 + ], + "seqrepo_raw_digest": "Ede4Q4UXqG9Dzi4MrnjeH9jp9k0ggu7T", + "refget_raw_digest": "Ede4Q4UXqG9Dzi4MrnjeH9jp9k0ggu7T", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000461363.1", + "seqrepo_seq_id": "O0rR6w_qtUdlVU360MFqtPEbi7ZFraqF", + "refget_sha512t24u": "KInA0nFf_sQ_yQXjq92_k4rkC72kfgDG", + "refget_length": 517, + "sequences_identical": true, + "seqrepo_seq_length": 517, + "refget_seq_length": 517, + "has_stop_codon": true, + "stop_codon_positions": [ + 252, + 256, + 272, + 286, + 291, + 292, + 300, + 367, + 399, + 409, + 424 + ], + "seqrepo_raw_digest": "KInA0nFf_sQ_yQXjq92_k4rkC72kfgDG", + "refget_raw_digest": "KInA0nFf_sQ_yQXjq92_k4rkC72kfgDG", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000461768.1", + "seqrepo_seq_id": "SBAWxYVz_UK9DZodtRFfbK1-2l5-uvL2", + "refget_sha512t24u": "6UVE7AZ23iHnnV0dDaNAFu_wGKHPYYoI", + "refget_length": 498, + "sequences_identical": true, + "seqrepo_seq_length": 498, + "refget_seq_length": 498, + "has_stop_codon": true, + "stop_codon_positions": [ + 252, + 256, + 272, + 286, + 291, + 292, + 300, + 348, + 380, + 390, + 405 + ], + "seqrepo_raw_digest": "6UVE7AZ23iHnnV0dDaNAFu_wGKHPYYoI", + "refget_raw_digest": "6UVE7AZ23iHnnV0dDaNAFu_wGKHPYYoI", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000463502.4", + "seqrepo_seq_id": "syXOnAwOvxwpfLO67Z7c84KArOxWUGX-", + "refget_sha512t24u": "zWPI0igbczAoDoC3RNroPzMKmMmNOe6I", + "refget_length": 469, + "sequences_identical": true, + "seqrepo_seq_length": 469, + "refget_seq_length": 469, + "has_stop_codon": true, + "stop_codon_positions": [ + 356 + ], + "seqrepo_raw_digest": "zWPI0igbczAoDoC3RNroPzMKmMmNOe6I", + "refget_raw_digest": "zWPI0igbczAoDoC3RNroPzMKmMmNOe6I", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000474017.2", + "seqrepo_seq_id": "PkCr3cRIsERXzmDHBNwJKKU2AhW1pcjS", + "refget_sha512t24u": "a8CMClpZG-W05aaW-ZTFsdZ8WUQnX6t4", + "refget_length": 10, + "sequences_identical": true, + "seqrepo_seq_length": 10, + "refget_seq_length": 10, + "has_stop_codon": true, + "stop_codon_positions": [ + 3, + 4 + ], + "seqrepo_raw_digest": "a8CMClpZG-W05aaW-ZTFsdZ8WUQnX6t4", + "refget_raw_digest": "a8CMClpZG-W05aaW-ZTFsdZ8WUQnX6t4", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000474065.2", + "seqrepo_seq_id": "PkCr3cRIsERXzmDHBNwJKKU2AhW1pcjS", + "refget_sha512t24u": "a8CMClpZG-W05aaW-ZTFsdZ8WUQnX6t4", + "refget_length": 10, + "sequences_identical": true, + "seqrepo_seq_length": 10, + "refget_seq_length": 10, + "has_stop_codon": true, + "stop_codon_positions": [ + 3, + 4 + ], + "seqrepo_raw_digest": "a8CMClpZG-W05aaW-ZTFsdZ8WUQnX6t4", + "refget_raw_digest": "a8CMClpZG-W05aaW-ZTFsdZ8WUQnX6t4", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000474133.2", + "seqrepo_seq_id": "U7neqpeNT4iNk-l3yhxlfzeh7imlPahR", + "refget_sha512t24u": "Qflup9oKLA9_ebzhyw1VwhdIcInZGnb_", + "refget_length": 10, + "sequences_identical": true, + "seqrepo_seq_length": 10, + "refget_seq_length": 10, + "has_stop_codon": true, + "stop_codon_positions": [ + 2 + ], + "seqrepo_raw_digest": "Qflup9oKLA9_ebzhyw1VwhdIcInZGnb_", + "refget_raw_digest": "Qflup9oKLA9_ebzhyw1VwhdIcInZGnb_", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000474573.2", + "seqrepo_seq_id": "U7neqpeNT4iNk-l3yhxlfzeh7imlPahR", + "refget_sha512t24u": "Qflup9oKLA9_ebzhyw1VwhdIcInZGnb_", + "refget_length": 10, + "sequences_identical": true, + "seqrepo_seq_length": 10, + "refget_seq_length": 10, + "has_stop_codon": true, + "stop_codon_positions": [ + 2 + ], + "seqrepo_raw_digest": "Qflup9oKLA9_ebzhyw1VwhdIcInZGnb_", + "refget_raw_digest": "Qflup9oKLA9_ebzhyw1VwhdIcInZGnb_", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000474693.2", + "seqrepo_seq_id": "FfJUWv8y8zqeEkxc8WC0CBxAVBpHKby_", + "refget_sha512t24u": "SMi7IfQFm46hxfH46aUlNOJd_mTj6CYC", + "refget_length": 6, + "sequences_identical": true, + "seqrepo_seq_length": 6, + "refget_seq_length": 6, + "has_stop_codon": true, + "stop_codon_positions": [ + 0, + 4 + ], + "seqrepo_raw_digest": "SMi7IfQFm46hxfH46aUlNOJd_mTj6CYC", + "refget_raw_digest": "SMi7IfQFm46hxfH46aUlNOJd_mTj6CYC", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000475053.2", + "seqrepo_seq_id": "FfJUWv8y8zqeEkxc8WC0CBxAVBpHKby_", + "refget_sha512t24u": "SMi7IfQFm46hxfH46aUlNOJd_mTj6CYC", + "refget_length": 6, + "sequences_identical": true, + "seqrepo_seq_length": 6, + "refget_seq_length": 6, + "has_stop_codon": true, + "stop_codon_positions": [ + 0, + 4 + ], + "seqrepo_raw_digest": "SMi7IfQFm46hxfH46aUlNOJd_mTj6CYC", + "refget_raw_digest": "SMi7IfQFm46hxfH46aUlNOJd_mTj6CYC", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000475160.1", + "seqrepo_seq_id": "FOZR2eNkiqv0vQKW28sYDoFUk-pHLf7q", + "refget_sha512t24u": "GO28PhtqjbkgqI_ZXen46QVdAUtQsTy0", + "refget_length": 308, + "sequences_identical": true, + "seqrepo_seq_length": 308, + "refget_seq_length": 308, + "has_stop_codon": true, + "stop_codon_positions": [ + 95, + 120, + 138, + 164, + 179, + 194, + 230, + 233, + 262 + ], + "seqrepo_raw_digest": "GO28PhtqjbkgqI_ZXen46QVdAUtQsTy0", + "refget_raw_digest": "GO28PhtqjbkgqI_ZXen46QVdAUtQsTy0", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000475351.1", + "seqrepo_seq_id": "NaBv12DVCEzFhBS1k7fnmH1RvEOocOQZ", + "refget_sha512t24u": "JcuCvOWNKvugDHiBlc5QNxiD9c1xGoHK", + "refget_length": 315, + "sequences_identical": true, + "seqrepo_seq_length": 315, + "refget_seq_length": 315, + "has_stop_codon": true, + "stop_codon_positions": [ + 238, + 240, + 278, + 281, + 285, + 290, + 298, + 308 + ], + "seqrepo_raw_digest": "JcuCvOWNKvugDHiBlc5QNxiD9c1xGoHK", + "refget_raw_digest": "JcuCvOWNKvugDHiBlc5QNxiD9c1xGoHK", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000475611.1", + "seqrepo_seq_id": "CafOAbOr7B4OwynxW2mGBs93mmWzlLpj", + "refget_sha512t24u": "so4CHY-xDA2Kza_d9Sq8Cz9E96yZMrL8", + "refget_length": 311, + "sequences_identical": true, + "seqrepo_seq_length": 311, + "refget_seq_length": 311, + "has_stop_codon": true, + "stop_codon_positions": [ + 213, + 220, + 260, + 275, + 284, + 291 + ], + "seqrepo_raw_digest": "so4CHY-xDA2Kza_d9Sq8Cz9E96yZMrL8", + "refget_raw_digest": "so4CHY-xDA2Kza_d9Sq8Cz9E96yZMrL8", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000476186.1", + "seqrepo_seq_id": "GQpNKfcojbpwHYXfeaYzPUelZhAsVouD", + "refget_sha512t24u": "Mg_qtBntkSHm5xPGBWSLyzHBXF1Ua1Oi", + "refget_length": 310, + "sequences_identical": true, + "seqrepo_seq_length": 310, + "refget_seq_length": 310, + "has_stop_codon": true, + "stop_codon_positions": [ + 268, + 294 + ], + "seqrepo_raw_digest": "Mg_qtBntkSHm5xPGBWSLyzHBXF1Ua1Oi", + "refget_raw_digest": "Mg_qtBntkSHm5xPGBWSLyzHBXF1Ua1Oi", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000476380.1", + "seqrepo_seq_id": "546P9zkeSm9jcCq9v5rNou3-U-9wOXa0", + "refget_sha512t24u": "9gXvvtU-EONZkDaTQZTRkqqAeX9YbCjD", + "refget_length": 326, + "sequences_identical": true, + "seqrepo_seq_length": 326, + "refget_seq_length": 326, + "has_stop_codon": true, + "stop_codon_positions": [ + 124 + ], + "seqrepo_raw_digest": "9gXvvtU-EONZkDaTQZTRkqqAeX9YbCjD", + "refget_raw_digest": "9gXvvtU-EONZkDaTQZTRkqqAeX9YbCjD", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000476467.1", + "seqrepo_seq_id": "HV7AX9iHZz2qgacGsUkeLl9JY1wQSy7b", + "refget_sha512t24u": "91F6s9En0f8Tp5H18TUHjtFIhnFA79lA", + "refget_length": 315, + "sequences_identical": true, + "seqrepo_seq_length": 315, + "refget_seq_length": 315, + "has_stop_codon": true, + "stop_codon_positions": [ + 135 + ], + "seqrepo_raw_digest": "91F6s9En0f8Tp5H18TUHjtFIhnFA79lA", + "refget_raw_digest": "91F6s9En0f8Tp5H18TUHjtFIhnFA79lA", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000476576.1", + "seqrepo_seq_id": "MHCBdS-C2GvAKyaWy5g7YpM5Tl6S7r2P", + "refget_sha512t24u": "exEjrMOOllerXl47soSNdLchtvUnq-n4", + "refget_length": 308, + "sequences_identical": true, + "seqrepo_seq_length": 308, + "refget_seq_length": 308, + "has_stop_codon": true, + "stop_codon_positions": [ + 278, + 287, + 301 + ], + "seqrepo_raw_digest": "exEjrMOOllerXl47soSNdLchtvUnq-n4", + "refget_raw_digest": "exEjrMOOllerXl47soSNdLchtvUnq-n4", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000477333.1", + "seqrepo_seq_id": "81G0gd-_pNefpEX9qCl6Xo3SfpL_GtM_", + "refget_sha512t24u": "jBGqsI0x-vXuNDpnaGRFodo1JLeIrtqk", + "refget_length": 319, + "sequences_identical": true, + "seqrepo_seq_length": 319, + "refget_seq_length": 319, + "has_stop_codon": true, + "stop_codon_positions": [ + 151, + 165, + 186, + 197, + 202, + 205, + 227, + 288, + 299, + 304, + 308, + 311 + ], + "seqrepo_raw_digest": "jBGqsI0x-vXuNDpnaGRFodo1JLeIrtqk", + "refget_raw_digest": "jBGqsI0x-vXuNDpnaGRFodo1JLeIrtqk", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000477337.2", + "seqrepo_seq_id": "Hgx6zf3aYxje2c12BUxwnW3Wfwcz72-E", + "refget_sha512t24u": "sfkGeZ9IhJISVjv1K1tsfeX8obIqTnf0", + "refget_length": 314, + "sequences_identical": true, + "seqrepo_seq_length": 314, + "refget_seq_length": 314, + "has_stop_codon": true, + "stop_codon_positions": [ + 145, + 153, + 156, + 186, + 193, + 205, + 211, + 221, + 222, + 225, + 256, + 272, + 280, + 291, + 296, + 311 + ], + "seqrepo_raw_digest": "sfkGeZ9IhJISVjv1K1tsfeX8obIqTnf0", + "refget_raw_digest": "sfkGeZ9IhJISVjv1K1tsfeX8obIqTnf0", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000477393.1", + "seqrepo_seq_id": "-a7aCjSuO0y-Noz6SjHENVXwfOXx2lgl", + "refget_sha512t24u": "C8md75ie9b7ZYfCXTAlyTnIUF4YkeQkn", + "refget_length": 328, + "sequences_identical": true, + "seqrepo_seq_length": 328, + "refget_seq_length": 328, + "has_stop_codon": true, + "stop_codon_positions": [ + 157, + 161, + 184, + 194, + 195, + 205, + 246, + 286, + 292, + 305 + ], + "seqrepo_raw_digest": "C8md75ie9b7ZYfCXTAlyTnIUF4YkeQkn", + "refget_raw_digest": "C8md75ie9b7ZYfCXTAlyTnIUF4YkeQkn", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000478383.1", + "seqrepo_seq_id": "se_guVGDbGUwQ_VwGptcBPiTSUEMOTDA", + "refget_sha512t24u": "IlMIcqDnTyx9LM1oe18o-6-Q6QWvov_6", + "refget_length": 297, + "sequences_identical": true, + "seqrepo_seq_length": 297, + "refget_seq_length": 297, + "has_stop_codon": true, + "stop_codon_positions": [ + 239, + 269, + 280 + ], + "seqrepo_raw_digest": "IlMIcqDnTyx9LM1oe18o-6-Q6QWvov_6", + "refget_raw_digest": "IlMIcqDnTyx9LM1oe18o-6-Q6QWvov_6", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000478656.2", + "seqrepo_seq_id": "8Jwk52z5pfAcprdAXS7iQN1kTNxxio40", + "refget_sha512t24u": "EbKxx4ZLK27lrDbLCoPaQMFIuY1_aMQs", + "refget_length": 517, + "sequences_identical": true, + "seqrepo_seq_length": 517, + "refget_seq_length": 517, + "has_stop_codon": true, + "stop_codon_positions": [ + 252, + 256, + 272, + 286, + 291, + 292, + 300, + 367, + 399, + 409, + 424 + ], + "seqrepo_raw_digest": "EbKxx4ZLK27lrDbLCoPaQMFIuY1_aMQs", + "refget_raw_digest": "EbKxx4ZLK27lrDbLCoPaQMFIuY1_aMQs", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000478998.1", + "seqrepo_seq_id": "Ot3JN_E9VBadQ46vgolO3JMynJDqSGHa", + "refget_sha512t24u": "f01vgyglFX1ju7emFuoB3OQn6mBWjCR6", + "refget_length": 297, + "sequences_identical": true, + "seqrepo_seq_length": 297, + "refget_seq_length": 297, + "has_stop_codon": true, + "stop_codon_positions": [ + 239, + 269, + 280 + ], + "seqrepo_raw_digest": "f01vgyglFX1ju7emFuoB3OQn6mBWjCR6", + "refget_raw_digest": "f01vgyglFX1ju7emFuoB3OQn6mBWjCR6", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000479129.1", + "seqrepo_seq_id": "PVMp-pPMZHSdjxIHbf9BPZFTVliQPwt8", + "refget_sha512t24u": "TiIdLvCGnMqH2aeTyjlo9IlWOwojBX4g", + "refget_length": 225, + "sequences_identical": true, + "seqrepo_seq_length": 225, + "refget_seq_length": 225, + "has_stop_codon": true, + "stop_codon_positions": [ + 222 + ], + "seqrepo_raw_digest": "TiIdLvCGnMqH2aeTyjlo9IlWOwojBX4g", + "refget_raw_digest": "TiIdLvCGnMqH2aeTyjlo9IlWOwojBX4g", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000479356.1", + "seqrepo_seq_id": "se_guVGDbGUwQ_VwGptcBPiTSUEMOTDA", + "refget_sha512t24u": "IlMIcqDnTyx9LM1oe18o-6-Q6QWvov_6", + "refget_length": 297, + "sequences_identical": true, + "seqrepo_seq_length": 297, + "refget_seq_length": 297, + "has_stop_codon": true, + "stop_codon_positions": [ + 239, + 269, + 280 + ], + "seqrepo_raw_digest": "IlMIcqDnTyx9LM1oe18o-6-Q6QWvov_6", + "refget_raw_digest": "IlMIcqDnTyx9LM1oe18o-6-Q6QWvov_6", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000479369.1", + "seqrepo_seq_id": "fv4uQW4AfrwGj3dx9D4xfpccU7Qnqtia", + "refget_sha512t24u": "8Co4fICavUFj9qH_GGXd5cbWbzh8oSZ0", + "refget_length": 297, + "sequences_identical": true, + "seqrepo_seq_length": 297, + "refget_seq_length": 297, + "has_stop_codon": true, + "stop_codon_positions": [ + 239, + 269, + 280 + ], + "seqrepo_raw_digest": "8Co4fICavUFj9qH_GGXd5cbWbzh8oSZ0", + "refget_raw_digest": "8Co4fICavUFj9qH_GGXd5cbWbzh8oSZ0", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000479404.1", + "seqrepo_seq_id": "t3g70NPf5GGSyB223lAwFe6zw1ChtmEy", + "refget_sha512t24u": "F_y21mXr_NuyB93ooYdRBy0kd-PXP6tz", + "refget_length": 374, + "sequences_identical": true, + "seqrepo_seq_length": 374, + "refget_seq_length": 374, + "has_stop_codon": true, + "stop_codon_positions": [ + 239, + 269, + 280, + 326 + ], + "seqrepo_raw_digest": "F_y21mXr_NuyB93ooYdRBy0kd-PXP6tz", + "refget_raw_digest": "F_y21mXr_NuyB93ooYdRBy0kd-PXP6tz", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000481139.1", + "seqrepo_seq_id": "6y4nN8mld_LcvD-EbegjezG4J0ipyLq1", + "refget_sha512t24u": "GIq_KqgXV64sQO3gTLidkdIOtqkmWH7A", + "refget_length": 297, + "sequences_identical": true, + "seqrepo_seq_length": 297, + "refget_seq_length": 297, + "has_stop_codon": true, + "stop_codon_positions": [ + 239, + 269, + 280 + ], + "seqrepo_raw_digest": "GIq_KqgXV64sQO3gTLidkdIOtqkmWH7A", + "refget_raw_digest": "GIq_KqgXV64sQO3gTLidkdIOtqkmWH7A", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000481279.1", + "seqrepo_seq_id": "XwjGxQ5MlPAw2PL52E6hJdBXJE3Z-Q32", + "refget_sha512t24u": "cUaEhxJnZ5QAuY3JG0Bn6bcW9MwKq73n", + "refget_length": 374, + "sequences_identical": true, + "seqrepo_seq_length": 374, + "refget_seq_length": 374, + "has_stop_codon": true, + "stop_codon_positions": [ + 239, + 269, + 280, + 326 + ], + "seqrepo_raw_digest": "cUaEhxJnZ5QAuY3JG0Bn6bcW9MwKq73n", + "refget_raw_digest": "cUaEhxJnZ5QAuY3JG0Bn6bcW9MwKq73n", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000481428.1", + "seqrepo_seq_id": "fv4uQW4AfrwGj3dx9D4xfpccU7Qnqtia", + "refget_sha512t24u": "8Co4fICavUFj9qH_GGXd5cbWbzh8oSZ0", + "refget_length": 297, + "sequences_identical": true, + "seqrepo_seq_length": 297, + "refget_seq_length": 297, + "has_stop_codon": true, + "stop_codon_positions": [ + 239, + 269, + 280 + ], + "seqrepo_raw_digest": "8Co4fICavUFj9qH_GGXd5cbWbzh8oSZ0", + "refget_raw_digest": "8Co4fICavUFj9qH_GGXd5cbWbzh8oSZ0", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000482055.2", + "seqrepo_seq_id": "se_guVGDbGUwQ_VwGptcBPiTSUEMOTDA", + "refget_sha512t24u": "IlMIcqDnTyx9LM1oe18o-6-Q6QWvov_6", + "refget_length": 297, + "sequences_identical": true, + "seqrepo_seq_length": 297, + "refget_seq_length": 297, + "has_stop_codon": true, + "stop_codon_positions": [ + 239, + 269, + 280 + ], + "seqrepo_raw_digest": "IlMIcqDnTyx9LM1oe18o-6-Q6QWvov_6", + "refget_raw_digest": "IlMIcqDnTyx9LM1oe18o-6-Q6QWvov_6", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000482745.1", + "seqrepo_seq_id": "lOkFBmBx4LkR4GtP4ljxF4ssKcMpU9S2", + "refget_sha512t24u": "sNGrVi9jJokpvjhGaAkw9-5mCjrkxR5s", + "refget_length": 341, + "sequences_identical": true, + "seqrepo_seq_length": 341, + "refget_seq_length": 341, + "has_stop_codon": true, + "stop_codon_positions": [ + 124 + ], + "seqrepo_raw_digest": "sNGrVi9jJokpvjhGaAkw9-5mCjrkxR5s", + "refget_raw_digest": "sNGrVi9jJokpvjhGaAkw9-5mCjrkxR5s", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000483571.1", + "seqrepo_seq_id": "lv5JmDOztbvJgKGluw0Fh6jFTb0LyBFT", + "refget_sha512t24u": "mNBJqEx1npRp6Qy7ofSZAYW4iX81-b3m", + "refget_length": 225, + "sequences_identical": true, + "seqrepo_seq_length": 225, + "refget_seq_length": 225, + "has_stop_codon": true, + "stop_codon_positions": [ + 222 + ], + "seqrepo_raw_digest": "mNBJqEx1npRp6Qy7ofSZAYW4iX81-b3m", + "refget_raw_digest": "mNBJqEx1npRp6Qy7ofSZAYW4iX81-b3m", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000483620.1", + "seqrepo_seq_id": "fv4uQW4AfrwGj3dx9D4xfpccU7Qnqtia", + "refget_sha512t24u": "8Co4fICavUFj9qH_GGXd5cbWbzh8oSZ0", + "refget_length": 297, + "sequences_identical": true, + "seqrepo_seq_length": 297, + "refget_seq_length": 297, + "has_stop_codon": true, + "stop_codon_positions": [ + 239, + 269, + 280 + ], + "seqrepo_raw_digest": "8Co4fICavUFj9qH_GGXd5cbWbzh8oSZ0", + "refget_raw_digest": "8Co4fICavUFj9qH_GGXd5cbWbzh8oSZ0", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000485881.1", + "seqrepo_seq_id": "5CcawTtWZa2Bc5otcdnSdzOQV1aMT_cq", + "refget_sha512t24u": "N3f7prhPB-ztQHfhGD48lojOdU42AWwG", + "refget_length": 311, + "sequences_identical": true, + "seqrepo_seq_length": 311, + "refget_seq_length": 311, + "has_stop_codon": true, + "stop_codon_positions": [ + 145, + 153, + 156, + 186, + 193, + 205, + 211, + 221, + 222, + 225, + 256, + 272, + 280, + 291, + 296 + ], + "seqrepo_raw_digest": "N3f7prhPB-ztQHfhGD48lojOdU42AWwG", + "refget_raw_digest": "N3f7prhPB-ztQHfhGD48lojOdU42AWwG", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000485975.1", + "seqrepo_seq_id": "AlKSb7ywKLef2uY_oqJ97OqTeZ1Cdb1D", + "refget_sha512t24u": "SzZa7T9y81S_V8iQp9RnpbnKshXX9IHp", + "refget_length": 315, + "sequences_identical": true, + "seqrepo_seq_length": 315, + "refget_seq_length": 315, + "has_stop_codon": true, + "stop_codon_positions": [ + 63 + ], + "seqrepo_raw_digest": "SzZa7T9y81S_V8iQp9RnpbnKshXX9IHp", + "refget_raw_digest": "SzZa7T9y81S_V8iQp9RnpbnKshXX9IHp", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000486295.1", + "seqrepo_seq_id": "iyPvjohGDgVYeFEVYQ_n6DFS5sOkwUTI", + "refget_sha512t24u": "VJMwcSEKzmAksrvE276Okup2ueBujczb", + "refget_length": 4040, + "sequences_identical": true, + "seqrepo_seq_length": 4040, + "refget_seq_length": 4040, + "has_stop_codon": true, + "stop_codon_positions": [ + 1978, + 2075, + 2244, + 2393, + 2412, + 2507, + 2548, + 2637, + 2670, + 2676, + 2717 + ], + "seqrepo_raw_digest": "VJMwcSEKzmAksrvE276Okup2ueBujczb", + "refget_raw_digest": "VJMwcSEKzmAksrvE276Okup2ueBujczb", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000487044.1", + "seqrepo_seq_id": "HeRZbi709aJqVjELYnnZZJrRUW17WjWH", + "refget_sha512t24u": "mdGiXxUvar3WuR1h-zsoryFcpLHQgi0H", + "refget_length": 498, + "sequences_identical": true, + "seqrepo_seq_length": 498, + "refget_seq_length": 498, + "has_stop_codon": true, + "stop_codon_positions": [ + 252, + 256, + 272, + 286, + 291, + 292, + 300, + 348, + 380, + 390, + 405 + ], + "seqrepo_raw_digest": "mdGiXxUvar3WuR1h-zsoryFcpLHQgi0H", + "refget_raw_digest": "mdGiXxUvar3WuR1h-zsoryFcpLHQgi0H", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000487059.1", + "seqrepo_seq_id": "NpQlnZ0jUAHvbmGF37prMDgMPv3THHYN", + "refget_sha512t24u": "2Vgy0wYjqGFzUpqGWPDJipU0XNdrKU8a", + "refget_length": 4714, + "sequences_identical": true, + "seqrepo_seq_length": 4714, + "refget_seq_length": 4714, + "has_stop_codon": true, + "stop_codon_positions": [ + 2382, + 2388, + 2418, + 2437, + 2587, + 2606, + 2775, + 2943, + 3471, + 3621, + 3640, + 3790, + 3809, + 3978, + 4014, + 4033, + 4183, + 4202, + 4372, + 4398, + 4482, + 4646, + 4676 + ], + "seqrepo_raw_digest": "2Vgy0wYjqGFzUpqGWPDJipU0XNdrKU8a", + "refget_raw_digest": "2Vgy0wYjqGFzUpqGWPDJipU0XNdrKU8a", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000487596.1", + "seqrepo_seq_id": "_IPtIdLGP1_WEK4al_dKDId2NigEHXZn", + "refget_sha512t24u": "bZJHWz7sSJQUC00fzSqS0wgZMcQ4X1i4", + "refget_length": 493, + "sequences_identical": true, + "seqrepo_seq_length": 493, + "refget_seq_length": 493, + "has_stop_codon": true, + "stop_codon_positions": [ + 99 + ], + "seqrepo_raw_digest": "bZJHWz7sSJQUC00fzSqS0wgZMcQ4X1i4", + "refget_raw_digest": "bZJHWz7sSJQUC00fzSqS0wgZMcQ4X1i4", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000487599.1", + "seqrepo_seq_id": "m19OoHXspP1j74FYLLR0Fon_Tca6IwAP", + "refget_sha512t24u": "fn4oRtmkRm5ikN4QPXXg0RdDOzaLpvUH", + "refget_length": 9, + "sequences_identical": true, + "seqrepo_seq_length": 9, + "refget_seq_length": 9, + "has_stop_codon": true, + "stop_codon_positions": [ + 5 + ], + "seqrepo_raw_digest": "fn4oRtmkRm5ikN4QPXXg0RdDOzaLpvUH", + "refget_raw_digest": "fn4oRtmkRm5ikN4QPXXg0RdDOzaLpvUH", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000487604.1", + "seqrepo_seq_id": "4nJC7z8ekuHU6kAl4yMiV6qZlrpomr_6", + "refget_sha512t24u": "vQGG6MQqX_mw1KxS82QU0Kuk5KtUItWA", + "refget_length": 10, + "sequences_identical": true, + "seqrepo_seq_length": 10, + "refget_seq_length": 10, + "has_stop_codon": true, + "stop_codon_positions": [ + 3, + 4 + ], + "seqrepo_raw_digest": "vQGG6MQqX_mw1KxS82QU0Kuk5KtUItWA", + "refget_raw_digest": "vQGG6MQqX_mw1KxS82QU0Kuk5KtUItWA", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000487787.1", + "seqrepo_seq_id": "SHyk_qyvZUk1fYzNdSTs6Drv8jS87r_C", + "refget_sha512t24u": "CN6dOnzsnNZiifsD2mNJ941p8KVRVPJu", + "refget_length": 12, + "sequences_identical": true, + "seqrepo_seq_length": 12, + "refget_seq_length": 12, + "has_stop_codon": true, + "stop_codon_positions": [ + 2 + ], + "seqrepo_raw_digest": "CN6dOnzsnNZiifsD2mNJ941p8KVRVPJu", + "refget_raw_digest": "CN6dOnzsnNZiifsD2mNJ941p8KVRVPJu", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000487993.1", + "seqrepo_seq_id": "y9mM4wkXZleVNSyYRPm83MhQd47vpV0A", + "refget_sha512t24u": "Ah8IodVoXqfHk4DDt8jsFGSic0fRZYU0", + "refget_length": 10, + "sequences_identical": true, + "seqrepo_seq_length": 10, + "refget_seq_length": 10, + "has_stop_codon": true, + "stop_codon_positions": [ + 3, + 6 + ], + "seqrepo_raw_digest": "Ah8IodVoXqfHk4DDt8jsFGSic0fRZYU0", + "refget_raw_digest": "Ah8IodVoXqfHk4DDt8jsFGSic0fRZYU0", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000488083.1", + "seqrepo_seq_id": "KRLK6ZBMvRpTakgKIIWxTzvHu6nZtEKQ", + "refget_sha512t24u": "5ZDMzPF5f4u5xiuBDA2ByfCYoG4534KM", + "refget_length": 10, + "sequences_identical": true, + "seqrepo_seq_length": 10, + "refget_seq_length": 10, + "has_stop_codon": true, + "stop_codon_positions": [ + 4 + ], + "seqrepo_raw_digest": "5ZDMzPF5f4u5xiuBDA2ByfCYoG4534KM", + "refget_raw_digest": "5ZDMzPF5f4u5xiuBDA2ByfCYoG4534KM", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000488168.1", + "seqrepo_seq_id": "AAWZIb3sgZG4DGzYKPTEdsDRPppA20nl", + "refget_sha512t24u": "ZyuIuWLGyac0hNqMfiw0XIJIjTx1UF2Y", + "refget_length": 6, + "sequences_identical": true, + "seqrepo_seq_length": 6, + "refget_seq_length": 6, + "has_stop_codon": true, + "stop_codon_positions": [ + 0, + 4 + ], + "seqrepo_raw_digest": "ZyuIuWLGyac0hNqMfiw0XIJIjTx1UF2Y", + "refget_raw_digest": "ZyuIuWLGyac0hNqMfiw0XIJIjTx1UF2Y", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000488261.1", + "seqrepo_seq_id": "BM1Hbmp96w4OSkWZ5u_ROCq4tWMf_V69", + "refget_sha512t24u": "hzlOSDnnFBtjxI-qUTYSXVvP8K76MfHk", + "refget_length": 5, + "sequences_identical": true, + "seqrepo_seq_length": 5, + "refget_seq_length": 5, + "has_stop_codon": true, + "stop_codon_positions": [ + 0, + 3 + ], + "seqrepo_raw_digest": "hzlOSDnnFBtjxI-qUTYSXVvP8K76MfHk", + "refget_raw_digest": "hzlOSDnnFBtjxI-qUTYSXVvP8K76MfHk", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000488314.1", + "seqrepo_seq_id": "Zbnpiff-E255gCpMrQGZejm0OauEumnc", + "refget_sha512t24u": "4yrFdP8maHkqUKDKBSVQ-gIrkehiwHxr", + "refget_length": 153, + "sequences_identical": true, + "seqrepo_seq_length": 153, + "refget_seq_length": 153, + "has_stop_codon": true, + "stop_codon_positions": [ + 88 + ], + "seqrepo_raw_digest": "4yrFdP8maHkqUKDKBSVQ-gIrkehiwHxr", + "refget_raw_digest": "4yrFdP8maHkqUKDKBSVQ-gIrkehiwHxr", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000488711.1", + "seqrepo_seq_id": "5Ek4xsz2ZS7GfXgv_ZBjOH5FkAP3XMX6", + "refget_sha512t24u": "JCCHtapvgLCCSu0eIJCK34DXB_mOSDrc", + "refget_length": 10, + "sequences_identical": true, + "seqrepo_seq_length": 10, + "refget_seq_length": 10, + "has_stop_codon": true, + "stop_codon_positions": [ + 3, + 4, + 5 + ], + "seqrepo_raw_digest": "JCCHtapvgLCCSu0eIJCK34DXB_mOSDrc", + "refget_raw_digest": "JCCHtapvgLCCSu0eIJCK34DXB_mOSDrc", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000488735.1", + "seqrepo_seq_id": "qRfoOZ40q93mvxVq3MDnu2rMZt2mB_ob", + "refget_sha512t24u": "yAZ-3aLns9fo2SxCNtvZJGjw_9-p67nz", + "refget_length": 5, + "sequences_identical": true, + "seqrepo_seq_length": 5, + "refget_seq_length": 5, + "has_stop_codon": true, + "stop_codon_positions": [ + 0, + 3 + ], + "seqrepo_raw_digest": "yAZ-3aLns9fo2SxCNtvZJGjw_9-p67nz", + "refget_raw_digest": "yAZ-3aLns9fo2SxCNtvZJGjw_9-p67nz", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000488889.1", + "seqrepo_seq_id": "qRfoOZ40q93mvxVq3MDnu2rMZt2mB_ob", + "refget_sha512t24u": "yAZ-3aLns9fo2SxCNtvZJGjw_9-p67nz", + "refget_length": 5, + "sequences_identical": true, + "seqrepo_seq_length": 5, + "refget_seq_length": 5, + "has_stop_codon": true, + "stop_codon_positions": [ + 0, + 3 + ], + "seqrepo_raw_digest": "yAZ-3aLns9fo2SxCNtvZJGjw_9-p67nz", + "refget_raw_digest": "yAZ-3aLns9fo2SxCNtvZJGjw_9-p67nz", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000488911.1", + "seqrepo_seq_id": "h9XYkJfTPZZjGI3uq_VgHAxQfGOnusMx", + "refget_sha512t24u": "Lim-8--cZivPCz0LQqISF0tUGv2hrbrX", + "refget_length": 497, + "sequences_identical": true, + "seqrepo_seq_length": 497, + "refget_seq_length": 497, + "has_stop_codon": true, + "stop_codon_positions": [ + 252, + 256, + 272, + 286, + 291, + 292, + 300, + 348, + 361, + 380, + 390, + 405 + ], + "seqrepo_raw_digest": "Lim-8--cZivPCz0LQqISF0tUGv2hrbrX", + "refget_raw_digest": "Lim-8--cZivPCz0LQqISF0tUGv2hrbrX", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000488993.1", + "seqrepo_seq_id": "Pu86r-b0I3U_dqkK9zQ2OPGSshhhhHWG", + "refget_sha512t24u": "DnlfpzUcYbk15_dKw1jqPVCwqxRENcFT", + "refget_length": 244, + "sequences_identical": true, + "seqrepo_seq_length": 244, + "refget_seq_length": 244, + "has_stop_codon": true, + "stop_codon_positions": [ + 195 + ], + "seqrepo_raw_digest": "DnlfpzUcYbk15_dKw1jqPVCwqxRENcFT", + "refget_raw_digest": "DnlfpzUcYbk15_dKw1jqPVCwqxRENcFT", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000489014.1", + "seqrepo_seq_id": "-fkYMovNew_mTufEcBSEn80kG2rr4OB_", + "refget_sha512t24u": "ZurDzxk4uhTHtf8xZAPlkm2VY4kZwv5E", + "refget_length": 516, + "sequences_identical": true, + "seqrepo_seq_length": 516, + "refget_seq_length": 516, + "has_stop_codon": true, + "stop_codon_positions": [ + 252, + 256, + 272, + 286, + 291, + 292, + 300, + 367, + 380, + 399, + 409, + 424 + ], + "seqrepo_raw_digest": "ZurDzxk4uhTHtf8xZAPlkm2VY4kZwv5E", + "refget_raw_digest": "ZurDzxk4uhTHtf8xZAPlkm2VY4kZwv5E", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000489240.1", + "seqrepo_seq_id": "aPj392cUfAWEeDcwOZLZafkPAwlUuscd", + "refget_sha512t24u": "IoKaC3TiSvxPlN1u_b9QaSEGIr9uavgu", + "refget_length": 108, + "sequences_identical": true, + "seqrepo_seq_length": 108, + "refget_seq_length": 108, + "has_stop_codon": true, + "stop_codon_positions": [ + 51 + ], + "seqrepo_raw_digest": "IoKaC3TiSvxPlN1u_b9QaSEGIr9uavgu", + "refget_raw_digest": "IoKaC3TiSvxPlN1u_b9QaSEGIr9uavgu", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000489559.1", + "seqrepo_seq_id": "7Dqau58VfarSEwoAyKydRrGg8qHMU6sN", + "refget_sha512t24u": "jdWrXmbGQ39RS_FxE9AxivWfxAjk0GgM", + "refget_length": 132, + "sequences_identical": true, + "seqrepo_seq_length": 132, + "refget_seq_length": 132, + "has_stop_codon": true, + "stop_codon_positions": [ + 75 + ], + "seqrepo_raw_digest": "jdWrXmbGQ39RS_FxE9AxivWfxAjk0GgM", + "refget_raw_digest": "jdWrXmbGQ39RS_FxE9AxivWfxAjk0GgM", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000490976.1", + "seqrepo_seq_id": "Ot3JN_E9VBadQ46vgolO3JMynJDqSGHa", + "refget_sha512t24u": "f01vgyglFX1ju7emFuoB3OQn6mBWjCR6", + "refget_length": 297, + "sequences_identical": true, + "seqrepo_seq_length": 297, + "refget_seq_length": 297, + "has_stop_codon": true, + "stop_codon_positions": [ + 239, + 269, + 280 + ], + "seqrepo_raw_digest": "f01vgyglFX1ju7emFuoB3OQn6mBWjCR6", + "refget_raw_digest": "f01vgyglFX1ju7emFuoB3OQn6mBWjCR6", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000491125.1", + "seqrepo_seq_id": "tRRBvR9lKL2Pqn0LDyPFWX9QzwN6R9cI", + "refget_sha512t24u": "yJ1li88C4ZhzDqpmIMuJhDlayDIu2i15", + "refget_length": 357, + "sequences_identical": true, + "seqrepo_seq_length": 357, + "refget_seq_length": 357, + "has_stop_codon": true, + "stop_codon_positions": [ + 45 + ], + "seqrepo_raw_digest": "yJ1li88C4ZhzDqpmIMuJhDlayDIu2i15", + "refget_raw_digest": "yJ1li88C4ZhzDqpmIMuJhDlayDIu2i15", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000491133.1", + "seqrepo_seq_id": "84ZbQcviHt30Ks_HfJnC25WT1H3Ll_aG", + "refget_sha512t24u": "UPwfn83YcGH13BQeW32xlO0MSD6SS24V", + "refget_length": 306, + "sequences_identical": true, + "seqrepo_seq_length": 306, + "refget_seq_length": 306, + "has_stop_codon": true, + "stop_codon_positions": [ + 59 + ], + "seqrepo_raw_digest": "UPwfn83YcGH13BQeW32xlO0MSD6SS24V", + "refget_raw_digest": "UPwfn83YcGH13BQeW32xlO0MSD6SS24V", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000491451.1", + "seqrepo_seq_id": "se_guVGDbGUwQ_VwGptcBPiTSUEMOTDA", + "refget_sha512t24u": "IlMIcqDnTyx9LM1oe18o-6-Q6QWvov_6", + "refget_length": 297, + "sequences_identical": true, + "seqrepo_seq_length": 297, + "refget_seq_length": 297, + "has_stop_codon": true, + "stop_codon_positions": [ + 239, + 269, + 280 + ], + "seqrepo_raw_digest": "IlMIcqDnTyx9LM1oe18o-6-Q6QWvov_6", + "refget_raw_digest": "IlMIcqDnTyx9LM1oe18o-6-Q6QWvov_6", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000491543.1", + "seqrepo_seq_id": "Ipbqtow-myKtzhLMr1FiU01rBRs7Iwv1", + "refget_sha512t24u": "w6SG0sotavd93SmwWs3HxDb31icvb43V", + "refget_length": 225, + "sequences_identical": true, + "seqrepo_seq_length": 225, + "refget_seq_length": 225, + "has_stop_codon": true, + "stop_codon_positions": [ + 222 + ], + "seqrepo_raw_digest": "w6SG0sotavd93SmwWs3HxDb31icvb43V", + "refget_raw_digest": "w6SG0sotavd93SmwWs3HxDb31icvb43V", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000491974.1", + "seqrepo_seq_id": "fv4uQW4AfrwGj3dx9D4xfpccU7Qnqtia", + "refget_sha512t24u": "8Co4fICavUFj9qH_GGXd5cbWbzh8oSZ0", + "refget_length": 297, + "sequences_identical": true, + "seqrepo_seq_length": 297, + "refget_seq_length": 297, + "has_stop_codon": true, + "stop_codon_positions": [ + 239, + 269, + 280 + ], + "seqrepo_raw_digest": "8Co4fICavUFj9qH_GGXd5cbWbzh8oSZ0", + "refget_raw_digest": "8Co4fICavUFj9qH_GGXd5cbWbzh8oSZ0", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000491994.1", + "seqrepo_seq_id": "0wFdmvBugnimf4tOFjCVdckpw3P6GmOx", + "refget_sha512t24u": "Qg7FS7qyO-pfHp7HYd4wvr26DEx1oov4", + "refget_length": 225, + "sequences_identical": true, + "seqrepo_seq_length": 225, + "refget_seq_length": 225, + "has_stop_codon": true, + "stop_codon_positions": [ + 222 + ], + "seqrepo_raw_digest": "Qg7FS7qyO-pfHp7HYd4wvr26DEx1oov4", + "refget_raw_digest": "Qg7FS7qyO-pfHp7HYd4wvr26DEx1oov4", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000492028.2", + "seqrepo_seq_id": "fv4uQW4AfrwGj3dx9D4xfpccU7Qnqtia", + "refget_sha512t24u": "8Co4fICavUFj9qH_GGXd5cbWbzh8oSZ0", + "refget_length": 297, + "sequences_identical": true, + "seqrepo_seq_length": 297, + "refget_seq_length": 297, + "has_stop_codon": true, + "stop_codon_positions": [ + 239, + 269, + 280 + ], + "seqrepo_raw_digest": "8Co4fICavUFj9qH_GGXd5cbWbzh8oSZ0", + "refget_raw_digest": "8Co4fICavUFj9qH_GGXd5cbWbzh8oSZ0", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000492090.1", + "seqrepo_seq_id": "Ipbqtow-myKtzhLMr1FiU01rBRs7Iwv1", + "refget_sha512t24u": "w6SG0sotavd93SmwWs3HxDb31icvb43V", + "refget_length": 225, + "sequences_identical": true, + "seqrepo_seq_length": 225, + "refget_seq_length": 225, + "has_stop_codon": true, + "stop_codon_positions": [ + 222 + ], + "seqrepo_raw_digest": "w6SG0sotavd93SmwWs3HxDb31icvb43V", + "refget_raw_digest": "w6SG0sotavd93SmwWs3HxDb31icvb43V", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000492193.1", + "seqrepo_seq_id": "7Pr4ISih0_g-We99919kiPT8Ft1voOjU", + "refget_sha512t24u": "QKLfHnF-uMJbWpYlzxPWivnK9FISFaHv", + "refget_length": 688, + "sequences_identical": true, + "seqrepo_seq_length": 688, + "refget_seq_length": 688, + "has_stop_codon": true, + "stop_codon_positions": [ + 498 + ], + "seqrepo_raw_digest": "QKLfHnF-uMJbWpYlzxPWivnK9FISFaHv", + "refget_raw_digest": "QKLfHnF-uMJbWpYlzxPWivnK9FISFaHv", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000492245.1", + "seqrepo_seq_id": "Ipbqtow-myKtzhLMr1FiU01rBRs7Iwv1", + "refget_sha512t24u": "w6SG0sotavd93SmwWs3HxDb31icvb43V", + "refget_length": 225, + "sequences_identical": true, + "seqrepo_seq_length": 225, + "refget_seq_length": 225, + "has_stop_codon": true, + "stop_codon_positions": [ + 222 + ], + "seqrepo_raw_digest": "w6SG0sotavd93SmwWs3HxDb31icvb43V", + "refget_raw_digest": "w6SG0sotavd93SmwWs3HxDb31icvb43V", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000492265.1", + "seqrepo_seq_id": "NdYD2MxngnXGXKck56SC5Y2pwnSRxQeN", + "refget_sha512t24u": "gZoUTjOaMrKyFO7XcjzV7e0F6hHw0Y2I", + "refget_length": 642, + "sequences_identical": true, + "seqrepo_seq_length": 642, + "refget_seq_length": 642, + "has_stop_codon": true, + "stop_codon_positions": [ + 452 + ], + "seqrepo_raw_digest": "gZoUTjOaMrKyFO7XcjzV7e0F6hHw0Y2I", + "refget_raw_digest": "gZoUTjOaMrKyFO7XcjzV7e0F6hHw0Y2I", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000492337.2", + "seqrepo_seq_id": "0wFdmvBugnimf4tOFjCVdckpw3P6GmOx", + "refget_sha512t24u": "Qg7FS7qyO-pfHp7HYd4wvr26DEx1oov4", + "refget_length": 225, + "sequences_identical": true, + "seqrepo_seq_length": 225, + "refget_seq_length": 225, + "has_stop_codon": true, + "stop_codon_positions": [ + 222 + ], + "seqrepo_raw_digest": "Qg7FS7qyO-pfHp7HYd4wvr26DEx1oov4", + "refget_raw_digest": "Qg7FS7qyO-pfHp7HYd4wvr26DEx1oov4", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000492458.1", + "seqrepo_seq_id": "PVMp-pPMZHSdjxIHbf9BPZFTVliQPwt8", + "refget_sha512t24u": "TiIdLvCGnMqH2aeTyjlo9IlWOwojBX4g", + "refget_length": 225, + "sequences_identical": true, + "seqrepo_seq_length": 225, + "refget_seq_length": 225, + "has_stop_codon": true, + "stop_codon_positions": [ + 222 + ], + "seqrepo_raw_digest": "TiIdLvCGnMqH2aeTyjlo9IlWOwojBX4g", + "refget_raw_digest": "TiIdLvCGnMqH2aeTyjlo9IlWOwojBX4g", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000492627.1", + "seqrepo_seq_id": "se_guVGDbGUwQ_VwGptcBPiTSUEMOTDA", + "refget_sha512t24u": "IlMIcqDnTyx9LM1oe18o-6-Q6QWvov_6", + "refget_length": 297, + "sequences_identical": true, + "seqrepo_seq_length": 297, + "refget_seq_length": 297, + "has_stop_codon": true, + "stop_codon_positions": [ + 239, + 269, + 280 + ], + "seqrepo_raw_digest": "IlMIcqDnTyx9LM1oe18o-6-Q6QWvov_6", + "refget_raw_digest": "IlMIcqDnTyx9LM1oe18o-6-Q6QWvov_6", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000493090.1", + "seqrepo_seq_id": "OwOpiUeTZxOiRyJpUVgFPEknheNaytQk", + "refget_sha512t24u": "7gOPz5rjQBMvXPqGeF8kBFb1MRWsv-XP", + "refget_length": 331, + "sequences_identical": true, + "seqrepo_seq_length": 331, + "refget_seq_length": 331, + "has_stop_codon": true, + "stop_codon_positions": [ + 184, + 330 + ], + "seqrepo_raw_digest": "7gOPz5rjQBMvXPqGeF8kBFb1MRWsv-XP", + "refget_raw_digest": "7gOPz5rjQBMvXPqGeF8kBFb1MRWsv-XP", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000493099.2", + "seqrepo_seq_id": "lcX6lIyYgyvgNIcxL8Nf_9c2te0uWR7J", + "refget_sha512t24u": "bX-m0VrO77yaMLzVcz__B14CYNhMhNS8", + "refget_length": 105, + "sequences_identical": true, + "seqrepo_seq_length": 105, + "refget_seq_length": 105, + "has_stop_codon": true, + "stop_codon_positions": [ + 47 + ], + "seqrepo_raw_digest": "bX-m0VrO77yaMLzVcz__B14CYNhMhNS8", + "refget_raw_digest": "bX-m0VrO77yaMLzVcz__B14CYNhMhNS8", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000493221.1", + "seqrepo_seq_id": "YhrPozOQOjywrkWpQWPgu49GdiZqFZk8", + "refget_sha512t24u": "vax-AZkewD5SpDAT5szm8meLHgf2beXC", + "refget_length": 317, + "sequences_identical": true, + "seqrepo_seq_length": 317, + "refget_seq_length": 317, + "has_stop_codon": true, + "stop_codon_positions": [ + 109, + 119, + 132, + 225, + 267, + 291 + ], + "seqrepo_raw_digest": "vax-AZkewD5SpDAT5szm8meLHgf2beXC", + "refget_raw_digest": "vax-AZkewD5SpDAT5szm8meLHgf2beXC", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000493243.1", + "seqrepo_seq_id": "YhrPozOQOjywrkWpQWPgu49GdiZqFZk8", + "refget_sha512t24u": "vax-AZkewD5SpDAT5szm8meLHgf2beXC", + "refget_length": 317, + "sequences_identical": true, + "seqrepo_seq_length": 317, + "refget_seq_length": 317, + "has_stop_codon": true, + "stop_codon_positions": [ + 109, + 119, + 132, + 225, + 267, + 291 + ], + "seqrepo_raw_digest": "vax-AZkewD5SpDAT5szm8meLHgf2beXC", + "refget_raw_digest": "vax-AZkewD5SpDAT5szm8meLHgf2beXC", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000493291.1", + "seqrepo_seq_id": "_CbilrTwIJnxQ8UDDZ3ieqBW5Jz6IlVd", + "refget_sha512t24u": "8GVt4hPQBAH0n47yoEBplEk176LMj6fZ", + "refget_length": 315, + "sequences_identical": true, + "seqrepo_seq_length": 315, + "refget_seq_length": 315, + "has_stop_codon": true, + "stop_codon_positions": [ + 214, + 246, + 266, + 281, + 284, + 297, + 300 + ], + "seqrepo_raw_digest": "8GVt4hPQBAH0n47yoEBplEk176LMj6fZ", + "refget_raw_digest": "8GVt4hPQBAH0n47yoEBplEk176LMj6fZ", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000493317.1", + "seqrepo_seq_id": "AlKSb7ywKLef2uY_oqJ97OqTeZ1Cdb1D", + "refget_sha512t24u": "SzZa7T9y81S_V8iQp9RnpbnKshXX9IHp", + "refget_length": 315, + "sequences_identical": true, + "seqrepo_seq_length": 315, + "refget_seq_length": 315, + "has_stop_codon": true, + "stop_codon_positions": [ + 63 + ], + "seqrepo_raw_digest": "SzZa7T9y81S_V8iQp9RnpbnKshXX9IHp", + "refget_raw_digest": "SzZa7T9y81S_V8iQp9RnpbnKshXX9IHp", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000493452.1", + "seqrepo_seq_id": "mICKQK_3Sn5lQNys1NRoohE3GrKeB5eh", + "refget_sha512t24u": "HO0I-zZFjRZAJMqZ7io8-u7lxa3y9P6_", + "refget_length": 308, + "sequences_identical": true, + "seqrepo_seq_length": 308, + "refget_seq_length": 308, + "has_stop_codon": true, + "stop_codon_positions": [ + 232 + ], + "seqrepo_raw_digest": "HO0I-zZFjRZAJMqZ7io8-u7lxa3y9P6_", + "refget_raw_digest": "HO0I-zZFjRZAJMqZ7io8-u7lxa3y9P6_", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000495205.1", + "seqrepo_seq_id": "Nmuo8TKv6WCrdaDDfamxvpgrA26KLpLS", + "refget_sha512t24u": "qR9P0vEzxBhBS34XQHLkcbx9GCG08qoa", + "refget_length": 954, + "sequences_identical": true, + "seqrepo_seq_length": 954, + "refget_seq_length": 954, + "has_stop_codon": true, + "stop_codon_positions": [ + 767 + ], + "seqrepo_raw_digest": "qR9P0vEzxBhBS34XQHLkcbx9GCG08qoa", + "refget_raw_digest": "qR9P0vEzxBhBS34XQHLkcbx9GCG08qoa", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000496497.1", + "seqrepo_seq_id": "546P9zkeSm9jcCq9v5rNou3-U-9wOXa0", + "refget_sha512t24u": "9gXvvtU-EONZkDaTQZTRkqqAeX9YbCjD", + "refget_length": 326, + "sequences_identical": true, + "seqrepo_seq_length": 326, + "refget_seq_length": 326, + "has_stop_codon": true, + "stop_codon_positions": [ + 124 + ], + "seqrepo_raw_digest": "9gXvvtU-EONZkDaTQZTRkqqAeX9YbCjD", + "refget_raw_digest": "9gXvvtU-EONZkDaTQZTRkqqAeX9YbCjD", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENSP00000498832.1", + "seqrepo_seq_id": "PZVNRQLqkqRCjj4klpDJmfxJqLHzCSyO", + "refget_sha512t24u": "Gr7F6_lh_SAfec-UJ4tgY0vLz7oag0LV", + "refget_length": 232, + "sequences_identical": true, + "seqrepo_seq_length": 232, + "refget_seq_length": 232, + "has_stop_codon": true, + "stop_codon_positions": [ + 115, + 185 + ], + "seqrepo_raw_digest": "Gr7F6_lh_SAfec-UJ4tgY0vLz7oag0LV", + "refget_raw_digest": "Gr7F6_lh_SAfec-UJ4tgY0vLz7oag0LV", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "star_normalization", + "fetch_error": null + }, + { + "accession": "ENST00000668831.1", + "seqrepo_seq_id": "NllKO15Wb60jTlfVHY5YOFfN3WFZyn5M", + "refget_sha512t24u": "2EdYgXlBf5xGHlhCifMYtWt9QZPGOyxA", + "refget_length": 1120, + "sequences_identical": false, + "seqrepo_seq_length": 1214, + "refget_seq_length": 1120, + "has_stop_codon": false, + "stop_codon_positions": [], + "seqrepo_raw_digest": "NllKO15Wb60jTlfVHY5YOFfN3WFZyn5M", + "refget_raw_digest": "2EdYgXlBf5xGHlhCifMYtWt9QZPGOyxA", + "seqrepo_digest_matches_normalized": true, + "refget_digest_self_consistent": true, + "cause": "sequence_divergence", + "fetch_error": null + } + ], + "all_mismatches_are_star_normalization": false, + "root_cause": "115/116 mismatches are caused by bioutils.sequences.normalize_sequence() stripping '*' (stop codon) characters before computing the sha512t24u digest. SeqRepo uses this normalized digest as its seq_id (and thus its ga4gh:SQ.* alias), while gtars RefgetStore computes sha512t24u directly from the raw FASTA bytes including '*'. The underlying sequence bytes are identical in both stores for these cases. The remaining 1 mismatch(es) (ENST00000668831.1) are genuine sequence content differences between seqrepo's Ensembl load (from an older release) and refgetstore's Ensembl r113.", + "recommendation": "The refgetstore digest is correct per the GA4GH VRS specification, which defines the sequence digest as sha512t24u over the raw sequence bytes without normalization. SeqRepo's digest for *-containing proteins is technically non-conformant. Document the divergence and list the affected accessions. For the sequence-divergent transcript(s), the refgetstore carries the r113 version which is authoritative for that release." +} diff --git a/misc/refgetstore/download.sh b/misc/refgetstore/download.sh new file mode 100755 index 00000000..772e151c --- /dev/null +++ b/misc/refgetstore/download.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Pre-fetch FASTAs and assembly reports into the loader's cache directory. +# Mirrors the filenames build_store.py expects so subsequent runs are cache hits. +# Supports resuming partial downloads via curl -C -. +set -euo pipefail + +HERE="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +DEST="${HERE}/downloads" +mkdir -p "${DEST}" + +fetch() { + local url="$1" out="$2" + if [ -f "${out}" ]; then + echo "[skip] ${out} (already present)" + return + fi + echo "[get ] ${url}" + curl -fL --retry 3 --retry-delay 5 -C - -o "${out}.part" "${url}" + mv "${out}.part" "${out}" +} + +base_url="https://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/000/001/405" + +# GRCh38 major (FASTA is already in the repo root via fasta_path override) +fetch "${base_url}/GCF_000001405.26_GRCh38/GCF_000001405.26_GRCh38_assembly_report.txt" \ + "${DEST}/GRCh38.assembly_report.txt" + +# GRCh38.p14 +fetch "${base_url}/GCF_000001405.40_GRCh38.p14/GCF_000001405.40_GRCh38.p14_genomic.fna.gz" \ + "${DEST}/GCF_000001405.40_GRCh38.p14_genomic.fna.gz" +fetch "${base_url}/GCF_000001405.40_GRCh38.p14/GCF_000001405.40_GRCh38.p14_assembly_report.txt" \ + "${DEST}/GRCh38.p14.assembly_report.txt" + +# GRCh37 major +fetch "${base_url}/GCF_000001405.13_GRCh37/GCF_000001405.13_GRCh37_genomic.fna.gz" \ + "${DEST}/GCF_000001405.13_GRCh37_genomic.fna.gz" +fetch "${base_url}/GCF_000001405.13_GRCh37/GCF_000001405.13_GRCh37_assembly_report.txt" \ + "${DEST}/GRCh37.assembly_report.txt" + +# GRCh37.p13 +fetch "${base_url}/GCF_000001405.25_GRCh37.p13/GCF_000001405.25_GRCh37.p13_genomic.fna.gz" \ + "${DEST}/GCF_000001405.25_GRCh37.p13_genomic.fna.gz" +fetch "${base_url}/GCF_000001405.25_GRCh37.p13/GCF_000001405.25_GRCh37.p13_assembly_report.txt" \ + "${DEST}/GRCh37.p13.assembly_report.txt" + +echo +echo "Downloads ready in ${DEST}" +ls -lh "${DEST}" diff --git a/misc/refgetstore/ensembl_known_divergent.txt b/misc/refgetstore/ensembl_known_divergent.txt new file mode 100644 index 00000000..0f49a1ca --- /dev/null +++ b/misc/refgetstore/ensembl_known_divergent.txt @@ -0,0 +1,123 @@ +# Ensembl aliases with known digest divergence between seqrepo and refgetstore +# +# 115 entries: bioutils seq_seqhash strips * before hashing (star_normalization) +# 1 entry: genuine sequence difference across Ensembl releases (sequence_divergence) +# +# Format: accession seqrepo_seq_id refget_sha512t24u cause +# +ENSP00000340011.8 fv4uQW4AfrwGj3dx9D4xfpccU7Qnqtia 8Co4fICavUFj9qH_GGXd5cbWbzh8oSZ0 star_normalization +ENSP00000375609.1 0wFdmvBugnimf4tOFjCVdckpw3P6GmOx Qg7FS7qyO-pfHp7HYd4wvr26DEx1oov4 star_normalization +ENSP00000385765.6 lV7Y-VqQL6dNPJ2ZKWTqw46p_da8Eodm YL07fWvAjY43qO_0ZnVw5BxCvo_M_0x- star_normalization +ENSP00000402491.1 e6sQaY-e_N_HsVyuY3ymiFysYBMBFk9W dBxeQN4GInX8eFSsicpYD8N5Nn_0My2g star_normalization +ENSP00000411913.1 _RbUr7Oq9qE22xU9fnMC2uLA8RIp2Hri TQ93TCrFm5e-mb-tkSHz3d_DYaAlo4i3 star_normalization +ENSP00000421815.1 tM9WHP2MFnlP7TaVWy-zF3pASsoLzDdU PuFuh2fJaeBhwf_CIXb9c4n-5HHCkIA8 star_normalization +ENSP00000422220.1 o6IopMWmnKAE7FUNdEFy2TnsyBDa-swp 2ludFB9wsyqzj2rD-ba2o2sFeXQIa277 star_normalization +ENSP00000423754.1 K56fM_BMAirMsSULhhLEc2yBkgvEE_aZ kw3yUvbNK6RIftnKndZqroj4_OZxeNl9 star_normalization +ENSP00000423899.1 RDSLXXiECCksD3fd4HEjkDRDUojB3zm1 D9gmmANCDpy0XX2JYSF7ZeHkXVaaDAm6 star_normalization +ENSP00000423970.1 fOlJtYkBTNNq6H0kRnKsGbSqtO5PcFEr 96AGBZz3OkRXlz54m0a5vf04XVfn8MG0 star_normalization +ENSP00000424038.1 lOkFBmBx4LkR4GtP4ljxF4ssKcMpU9S2 sNGrVi9jJokpvjhGaAkw9-5mCjrkxR5s star_normalization +ENSP00000425652.1 0UyEhk9QGqb2gKNUMpHZ93nXVAXK9yTO JQU8ZqYvYRuVO19ZDaHQhHM89PyidybZ star_normalization +ENSP00000426427.1 03nXBcwbbJQPYE_5LurPSoCpnsikUdEn pO53MQMJFDxQT6bOb7ey9est-55eH5NH star_normalization +ENSP00000426566.1 _nkNH8COTskYyNkCoTzIRNHzENVylj1y egHuRL5C7VrWvOx3FvqbEayoBNIeC6ZM star_normalization +ENSP00000426627.1 OpwKkqOENTdevZ_CrLEOVumim2QTctaK 2KlIfbXxNKZoNImdwrmKBdRYXPkKL_2G star_normalization +ENSP00000427437.1 wWUGUKB2gnrfgmMm0qg34Iy9mUEhx4ju 7KGhHMvva3lM9749_TxBRCl5Oz8YcdKo star_normalization +ENSP00000427458.1 rFjG0Bt4bfEH09x5PaR9jZGd03wmgE6O -x9lIom6XRVzSbgQDyNb0hrandBbttoV star_normalization +ENSP00000427969.1 y9mM4wkXZleVNSyYRPm83MhQd47vpV0A Ah8IodVoXqfHk4DDt8jsFGSic0fRZYU0 star_normalization +ENSP00000428366.1 SHyk_qyvZUk1fYzNdSTs6Drv8jS87r_C CN6dOnzsnNZiifsD2mNJ941p8KVRVPJu star_normalization +ENSP00000428393.1 qRfoOZ40q93mvxVq3MDnu2rMZt2mB_ob yAZ-3aLns9fo2SxCNtvZJGjw_9-p67nz star_normalization +ENSP00000428616.1 KRLK6ZBMvRpTakgKIIWxTzvHu6nZtEKQ 5ZDMzPF5f4u5xiuBDA2ByfCYoG4534KM star_normalization +ENSP00000429324.1 m19OoHXspP1j74FYLLR0Fon_Tca6IwAP fn4oRtmkRm5ikN4QPXXg0RdDOzaLpvUH star_normalization +ENSP00000429952.1 5Ek4xsz2ZS7GfXgv_ZBjOH5FkAP3XMX6 JCCHtapvgLCCSu0eIJCK34DXB_mOSDrc star_normalization +ENSP00000430034.1 qRfoOZ40q93mvxVq3MDnu2rMZt2mB_ob yAZ-3aLns9fo2SxCNtvZJGjw_9-p67nz star_normalization +ENSP00000430248.1 AAWZIb3sgZG4DGzYKPTEdsDRPppA20nl ZyuIuWLGyac0hNqMfiw0XIJIjTx1UF2Y star_normalization +ENSP00000430788.1 4nJC7z8ekuHU6kAl4yMiV6qZlrpomr_6 vQGG6MQqX_mw1KxS82QU0Kuk5KtUItWA star_normalization +ENSP00000431031.1 l4SXGqAbFUh7r1jc-ImcNEdp067Fa0MY zkX9kDvYwL768RXctEHW7-3lneBHc71d star_normalization +ENSP00000431089.1 BM1Hbmp96w4OSkWZ5u_ROCq4tWMf_V69 hzlOSDnnFBtjxI-qUTYSXVvP8K76MfHk star_normalization +ENSP00000434417.1 zKGzcqTwIi1cpsh_25jTR-g7cycsIEpf Ih9FQHBcIpxLvdvSmQwf1B_uW07yCbjE star_normalization +ENSP00000434644.1 DZRDVc8gAxzoOkjyLBLK3IcVRKKK46Pw v8czIuEX2XTC0M9Dw7Ax3fzb4H8qdgOL star_normalization +ENSP00000438013.1 nDmCYyT1EKuf9Z8mrnEgCxgF2Hjn0T_P _nPal56b66hTKeKlOajKFSKjKY8DcuU- star_normalization +ENSP00000439604.1 oaVc-acao8fOMIyVp_H98dcz0BSDIUWH aYF_Y_ycp3u3t9Gqeu96GLzc304fQ0zp star_normalization +ENSP00000445124.1 6VmdTL7nqa7JQpolg-wuBLGwIKBnrP9F MIwe_aAViS3zDJuv3mQTnIDVQMSLST8L star_normalization +ENSP00000451021.2 38LKEEE-120Kwsih-vNlwnx8tb2A4ZSx _rwlFqSN0jljdGdyuRQXsthR-SSMBJ2G star_normalization +ENSP00000452150.1 ynPhtGE3u23nkLJNMxVv51pviMYUrBBX Ede4Q4UXqG9Dzi4MrnjeH9jp9k0ggu7T star_normalization +ENSP00000461363.1 O0rR6w_qtUdlVU360MFqtPEbi7ZFraqF KInA0nFf_sQ_yQXjq92_k4rkC72kfgDG star_normalization +ENSP00000461768.1 SBAWxYVz_UK9DZodtRFfbK1-2l5-uvL2 6UVE7AZ23iHnnV0dDaNAFu_wGKHPYYoI star_normalization +ENSP00000463502.4 syXOnAwOvxwpfLO67Z7c84KArOxWUGX- zWPI0igbczAoDoC3RNroPzMKmMmNOe6I star_normalization +ENSP00000474017.2 PkCr3cRIsERXzmDHBNwJKKU2AhW1pcjS a8CMClpZG-W05aaW-ZTFsdZ8WUQnX6t4 star_normalization +ENSP00000474065.2 PkCr3cRIsERXzmDHBNwJKKU2AhW1pcjS a8CMClpZG-W05aaW-ZTFsdZ8WUQnX6t4 star_normalization +ENSP00000474133.2 U7neqpeNT4iNk-l3yhxlfzeh7imlPahR Qflup9oKLA9_ebzhyw1VwhdIcInZGnb_ star_normalization +ENSP00000474573.2 U7neqpeNT4iNk-l3yhxlfzeh7imlPahR Qflup9oKLA9_ebzhyw1VwhdIcInZGnb_ star_normalization +ENSP00000474693.2 FfJUWv8y8zqeEkxc8WC0CBxAVBpHKby_ SMi7IfQFm46hxfH46aUlNOJd_mTj6CYC star_normalization +ENSP00000475053.2 FfJUWv8y8zqeEkxc8WC0CBxAVBpHKby_ SMi7IfQFm46hxfH46aUlNOJd_mTj6CYC star_normalization +ENSP00000475160.1 FOZR2eNkiqv0vQKW28sYDoFUk-pHLf7q GO28PhtqjbkgqI_ZXen46QVdAUtQsTy0 star_normalization +ENSP00000475351.1 NaBv12DVCEzFhBS1k7fnmH1RvEOocOQZ JcuCvOWNKvugDHiBlc5QNxiD9c1xGoHK star_normalization +ENSP00000475611.1 CafOAbOr7B4OwynxW2mGBs93mmWzlLpj so4CHY-xDA2Kza_d9Sq8Cz9E96yZMrL8 star_normalization +ENSP00000476186.1 GQpNKfcojbpwHYXfeaYzPUelZhAsVouD Mg_qtBntkSHm5xPGBWSLyzHBXF1Ua1Oi star_normalization +ENSP00000476380.1 546P9zkeSm9jcCq9v5rNou3-U-9wOXa0 9gXvvtU-EONZkDaTQZTRkqqAeX9YbCjD star_normalization +ENSP00000476467.1 HV7AX9iHZz2qgacGsUkeLl9JY1wQSy7b 91F6s9En0f8Tp5H18TUHjtFIhnFA79lA star_normalization +ENSP00000476576.1 MHCBdS-C2GvAKyaWy5g7YpM5Tl6S7r2P exEjrMOOllerXl47soSNdLchtvUnq-n4 star_normalization +ENSP00000477333.1 81G0gd-_pNefpEX9qCl6Xo3SfpL_GtM_ jBGqsI0x-vXuNDpnaGRFodo1JLeIrtqk star_normalization +ENSP00000477337.2 Hgx6zf3aYxje2c12BUxwnW3Wfwcz72-E sfkGeZ9IhJISVjv1K1tsfeX8obIqTnf0 star_normalization +ENSP00000477393.1 -a7aCjSuO0y-Noz6SjHENVXwfOXx2lgl C8md75ie9b7ZYfCXTAlyTnIUF4YkeQkn star_normalization +ENSP00000478383.1 se_guVGDbGUwQ_VwGptcBPiTSUEMOTDA IlMIcqDnTyx9LM1oe18o-6-Q6QWvov_6 star_normalization +ENSP00000478656.2 8Jwk52z5pfAcprdAXS7iQN1kTNxxio40 EbKxx4ZLK27lrDbLCoPaQMFIuY1_aMQs star_normalization +ENSP00000478998.1 Ot3JN_E9VBadQ46vgolO3JMynJDqSGHa f01vgyglFX1ju7emFuoB3OQn6mBWjCR6 star_normalization +ENSP00000479129.1 PVMp-pPMZHSdjxIHbf9BPZFTVliQPwt8 TiIdLvCGnMqH2aeTyjlo9IlWOwojBX4g star_normalization +ENSP00000479356.1 se_guVGDbGUwQ_VwGptcBPiTSUEMOTDA IlMIcqDnTyx9LM1oe18o-6-Q6QWvov_6 star_normalization +ENSP00000479369.1 fv4uQW4AfrwGj3dx9D4xfpccU7Qnqtia 8Co4fICavUFj9qH_GGXd5cbWbzh8oSZ0 star_normalization +ENSP00000479404.1 t3g70NPf5GGSyB223lAwFe6zw1ChtmEy F_y21mXr_NuyB93ooYdRBy0kd-PXP6tz star_normalization +ENSP00000481139.1 6y4nN8mld_LcvD-EbegjezG4J0ipyLq1 GIq_KqgXV64sQO3gTLidkdIOtqkmWH7A star_normalization +ENSP00000481279.1 XwjGxQ5MlPAw2PL52E6hJdBXJE3Z-Q32 cUaEhxJnZ5QAuY3JG0Bn6bcW9MwKq73n star_normalization +ENSP00000481428.1 fv4uQW4AfrwGj3dx9D4xfpccU7Qnqtia 8Co4fICavUFj9qH_GGXd5cbWbzh8oSZ0 star_normalization +ENSP00000482055.2 se_guVGDbGUwQ_VwGptcBPiTSUEMOTDA IlMIcqDnTyx9LM1oe18o-6-Q6QWvov_6 star_normalization +ENSP00000482745.1 lOkFBmBx4LkR4GtP4ljxF4ssKcMpU9S2 sNGrVi9jJokpvjhGaAkw9-5mCjrkxR5s star_normalization +ENSP00000483571.1 lv5JmDOztbvJgKGluw0Fh6jFTb0LyBFT mNBJqEx1npRp6Qy7ofSZAYW4iX81-b3m star_normalization +ENSP00000483620.1 fv4uQW4AfrwGj3dx9D4xfpccU7Qnqtia 8Co4fICavUFj9qH_GGXd5cbWbzh8oSZ0 star_normalization +ENSP00000485881.1 5CcawTtWZa2Bc5otcdnSdzOQV1aMT_cq N3f7prhPB-ztQHfhGD48lojOdU42AWwG star_normalization +ENSP00000485975.1 AlKSb7ywKLef2uY_oqJ97OqTeZ1Cdb1D SzZa7T9y81S_V8iQp9RnpbnKshXX9IHp star_normalization +ENSP00000486295.1 iyPvjohGDgVYeFEVYQ_n6DFS5sOkwUTI VJMwcSEKzmAksrvE276Okup2ueBujczb star_normalization +ENSP00000487044.1 HeRZbi709aJqVjELYnnZZJrRUW17WjWH mdGiXxUvar3WuR1h-zsoryFcpLHQgi0H star_normalization +ENSP00000487059.1 NpQlnZ0jUAHvbmGF37prMDgMPv3THHYN 2Vgy0wYjqGFzUpqGWPDJipU0XNdrKU8a star_normalization +ENSP00000487596.1 _IPtIdLGP1_WEK4al_dKDId2NigEHXZn bZJHWz7sSJQUC00fzSqS0wgZMcQ4X1i4 star_normalization +ENSP00000487599.1 m19OoHXspP1j74FYLLR0Fon_Tca6IwAP fn4oRtmkRm5ikN4QPXXg0RdDOzaLpvUH star_normalization +ENSP00000487604.1 4nJC7z8ekuHU6kAl4yMiV6qZlrpomr_6 vQGG6MQqX_mw1KxS82QU0Kuk5KtUItWA star_normalization +ENSP00000487787.1 SHyk_qyvZUk1fYzNdSTs6Drv8jS87r_C CN6dOnzsnNZiifsD2mNJ941p8KVRVPJu star_normalization +ENSP00000487993.1 y9mM4wkXZleVNSyYRPm83MhQd47vpV0A Ah8IodVoXqfHk4DDt8jsFGSic0fRZYU0 star_normalization +ENSP00000488083.1 KRLK6ZBMvRpTakgKIIWxTzvHu6nZtEKQ 5ZDMzPF5f4u5xiuBDA2ByfCYoG4534KM star_normalization +ENSP00000488168.1 AAWZIb3sgZG4DGzYKPTEdsDRPppA20nl ZyuIuWLGyac0hNqMfiw0XIJIjTx1UF2Y star_normalization +ENSP00000488261.1 BM1Hbmp96w4OSkWZ5u_ROCq4tWMf_V69 hzlOSDnnFBtjxI-qUTYSXVvP8K76MfHk star_normalization +ENSP00000488314.1 Zbnpiff-E255gCpMrQGZejm0OauEumnc 4yrFdP8maHkqUKDKBSVQ-gIrkehiwHxr star_normalization +ENSP00000488711.1 5Ek4xsz2ZS7GfXgv_ZBjOH5FkAP3XMX6 JCCHtapvgLCCSu0eIJCK34DXB_mOSDrc star_normalization +ENSP00000488735.1 qRfoOZ40q93mvxVq3MDnu2rMZt2mB_ob yAZ-3aLns9fo2SxCNtvZJGjw_9-p67nz star_normalization +ENSP00000488889.1 qRfoOZ40q93mvxVq3MDnu2rMZt2mB_ob yAZ-3aLns9fo2SxCNtvZJGjw_9-p67nz star_normalization +ENSP00000488911.1 h9XYkJfTPZZjGI3uq_VgHAxQfGOnusMx Lim-8--cZivPCz0LQqISF0tUGv2hrbrX star_normalization +ENSP00000488993.1 Pu86r-b0I3U_dqkK9zQ2OPGSshhhhHWG DnlfpzUcYbk15_dKw1jqPVCwqxRENcFT star_normalization +ENSP00000489014.1 -fkYMovNew_mTufEcBSEn80kG2rr4OB_ ZurDzxk4uhTHtf8xZAPlkm2VY4kZwv5E star_normalization +ENSP00000489240.1 aPj392cUfAWEeDcwOZLZafkPAwlUuscd IoKaC3TiSvxPlN1u_b9QaSEGIr9uavgu star_normalization +ENSP00000489559.1 7Dqau58VfarSEwoAyKydRrGg8qHMU6sN jdWrXmbGQ39RS_FxE9AxivWfxAjk0GgM star_normalization +ENSP00000490976.1 Ot3JN_E9VBadQ46vgolO3JMynJDqSGHa f01vgyglFX1ju7emFuoB3OQn6mBWjCR6 star_normalization +ENSP00000491125.1 tRRBvR9lKL2Pqn0LDyPFWX9QzwN6R9cI yJ1li88C4ZhzDqpmIMuJhDlayDIu2i15 star_normalization +ENSP00000491133.1 84ZbQcviHt30Ks_HfJnC25WT1H3Ll_aG UPwfn83YcGH13BQeW32xlO0MSD6SS24V star_normalization +ENSP00000491451.1 se_guVGDbGUwQ_VwGptcBPiTSUEMOTDA IlMIcqDnTyx9LM1oe18o-6-Q6QWvov_6 star_normalization +ENSP00000491543.1 Ipbqtow-myKtzhLMr1FiU01rBRs7Iwv1 w6SG0sotavd93SmwWs3HxDb31icvb43V star_normalization +ENSP00000491974.1 fv4uQW4AfrwGj3dx9D4xfpccU7Qnqtia 8Co4fICavUFj9qH_GGXd5cbWbzh8oSZ0 star_normalization +ENSP00000491994.1 0wFdmvBugnimf4tOFjCVdckpw3P6GmOx Qg7FS7qyO-pfHp7HYd4wvr26DEx1oov4 star_normalization +ENSP00000492028.2 fv4uQW4AfrwGj3dx9D4xfpccU7Qnqtia 8Co4fICavUFj9qH_GGXd5cbWbzh8oSZ0 star_normalization +ENSP00000492090.1 Ipbqtow-myKtzhLMr1FiU01rBRs7Iwv1 w6SG0sotavd93SmwWs3HxDb31icvb43V star_normalization +ENSP00000492193.1 7Pr4ISih0_g-We99919kiPT8Ft1voOjU QKLfHnF-uMJbWpYlzxPWivnK9FISFaHv star_normalization +ENSP00000492245.1 Ipbqtow-myKtzhLMr1FiU01rBRs7Iwv1 w6SG0sotavd93SmwWs3HxDb31icvb43V star_normalization +ENSP00000492265.1 NdYD2MxngnXGXKck56SC5Y2pwnSRxQeN gZoUTjOaMrKyFO7XcjzV7e0F6hHw0Y2I star_normalization +ENSP00000492337.2 0wFdmvBugnimf4tOFjCVdckpw3P6GmOx Qg7FS7qyO-pfHp7HYd4wvr26DEx1oov4 star_normalization +ENSP00000492458.1 PVMp-pPMZHSdjxIHbf9BPZFTVliQPwt8 TiIdLvCGnMqH2aeTyjlo9IlWOwojBX4g star_normalization +ENSP00000492627.1 se_guVGDbGUwQ_VwGptcBPiTSUEMOTDA IlMIcqDnTyx9LM1oe18o-6-Q6QWvov_6 star_normalization +ENSP00000493090.1 OwOpiUeTZxOiRyJpUVgFPEknheNaytQk 7gOPz5rjQBMvXPqGeF8kBFb1MRWsv-XP star_normalization +ENSP00000493099.2 lcX6lIyYgyvgNIcxL8Nf_9c2te0uWR7J bX-m0VrO77yaMLzVcz__B14CYNhMhNS8 star_normalization +ENSP00000493221.1 YhrPozOQOjywrkWpQWPgu49GdiZqFZk8 vax-AZkewD5SpDAT5szm8meLHgf2beXC star_normalization +ENSP00000493243.1 YhrPozOQOjywrkWpQWPgu49GdiZqFZk8 vax-AZkewD5SpDAT5szm8meLHgf2beXC star_normalization +ENSP00000493291.1 _CbilrTwIJnxQ8UDDZ3ieqBW5Jz6IlVd 8GVt4hPQBAH0n47yoEBplEk176LMj6fZ star_normalization +ENSP00000493317.1 AlKSb7ywKLef2uY_oqJ97OqTeZ1Cdb1D SzZa7T9y81S_V8iQp9RnpbnKshXX9IHp star_normalization +ENSP00000493452.1 mICKQK_3Sn5lQNys1NRoohE3GrKeB5eh HO0I-zZFjRZAJMqZ7io8-u7lxa3y9P6_ star_normalization +ENSP00000495205.1 Nmuo8TKv6WCrdaDDfamxvpgrA26KLpLS qR9P0vEzxBhBS34XQHLkcbx9GCG08qoa star_normalization +ENSP00000496497.1 546P9zkeSm9jcCq9v5rNou3-U-9wOXa0 9gXvvtU-EONZkDaTQZTRkqqAeX9YbCjD star_normalization +ENSP00000498832.1 PZVNRQLqkqRCjj4klpDJmfxJqLHzCSyO Gr7F6_lh_SAfec-UJ4tgY0vLz7oag0LV star_normalization +ENST00000668831.1 NllKO15Wb60jTlfVHY5YOFfN3WFZyn5M 2EdYgXlBf5xGHlhCifMYtWt9QZPGOyxA sequence_divergence diff --git a/misc/refgetstore/ensembl_translator_equivalence.py b/misc/refgetstore/ensembl_translator_equivalence.py new file mode 100644 index 00000000..7ba1c794 --- /dev/null +++ b/misc/refgetstore/ensembl_translator_equivalence.py @@ -0,0 +1,217 @@ +#!/usr/bin/env python +"""Ensembl-specific translator equivalence: seqrepo vs refgetstore. + +Verifies that VRS Allele IDs agree between backends for SPDI expressions +using Ensembl transcript accessions. This complements the bulk_equivalence.py +test (which uses genomic VCF inputs) by exercising the Ensembl alias +resolution code paths in the data proxy. + +SPDI format is used instead of HGVS to avoid the UTA database dependency. +The SPDI path exercises: ``derive_refget_accession`` → ``get_sequence`` → +``normalize`` — the full alias-to-allele pipeline. + +Usage: + SEQREPO_ROOT_DIR=/path/to/seqrepo/2024-12-20 \\ + uv run python misc/refgetstore/ensembl_translator_equivalence.py \\ + [--refget-store misc/refgetstore/store] + +Exits 0 if all Allele IDs match, 1 if any mismatch or error. +""" + +from __future__ import annotations + +import argparse +import os +import sys +from dataclasses import dataclass +from pathlib import Path + +from biocommons.seqrepo import SeqRepo + +from ga4gh.vrs.dataproxy import CompositeDataProxy, DataProxy +from ga4gh.vrs.dataproxy_refget import ( + RefgetAliasDataProxy, + RefgetSequenceDataProxy, +) +from ga4gh.vrs.dataproxy_seqrepo import SeqRepoDataProxy +from ga4gh.vrs.extras.translator import AlleleTranslator + +HERE = Path(__file__).resolve().parent +DEFAULT_STORE_PATH = HERE / "store" + + +@dataclass +class SPDICase: + """A single SPDI test case with Ensembl accession.""" + + spdi: str + description: str + + +# Well-known, stable Ensembl transcripts present in both seqrepo 2024-12-20 +# and refgetstore (r113). Versions were selected to exist in both stores. +# None of these contain '*' in their sequence (DNA transcripts), so the +# digest normalization difference does not apply. +SPDI_CASES: list[SPDICase] = [ + # BRCA1 — well-known cancer gene, both stores have v9 + SPDICase("ENST00000357654.9:100:1:T", "BRCA1 SNV at 100"), + SPDICase("ENST00000357654.9:5000:1:C", "BRCA1 SNV near 3' end"), + SPDICase("ENST00000357654.9:200:0:A", "BRCA1 insertion at 200"), + SPDICase("ENST00000357654.9:300:3:G", "BRCA1 delins at 300"), + # EGFR — both stores have v9 + SPDICase("ENST00000261584.9:500:1:A", "EGFR SNV at 500"), + SPDICase("ENST00000261584.9:2000:2:CC", "EGFR MNV at 2000"), + # KRAS — both stores have v11 + SPDICase("ENST00000288602.11:100:1:T", "KRAS SNV at 100"), + # BRAF — both stores have v7 + SPDICase("ENST00000275493.7:1000:1:A", "BRAF SNV at 1000"), + # RB1 — both stores have v8 + SPDICase("ENST00000378567.8:500:1:G", "RB1 SNV at 500"), + # Test deletion (empty insertion) + SPDICase("ENST00000357654.9:400:1:", "BRCA1 1bp deletion at 400"), +] + + +def make_seqrepo_proxy(seqrepo_root: Path) -> SeqRepoDataProxy: + sr = SeqRepo(root_dir=str(seqrepo_root), fd_cache_size=None) + return SeqRepoDataProxy(sr) + + +def make_refget_proxy(store_path: Path) -> CompositeDataProxy: + from gtars.refget import RefgetStore + + store = RefgetStore.open_local(str(store_path)) + return CompositeDataProxy( + RefgetSequenceDataProxy(store), + RefgetAliasDataProxy(store), + ) + + +def translate_spdi( + dp: DataProxy, spdi_expr: str, *, do_normalize: bool = True +) -> tuple[str | None, str | None]: + """Translate an SPDI expression and return (allele_id, error).""" + try: + tlr = AlleleTranslator(dp, identify=True) + allele = tlr.translate_from(spdi_expr, fmt="spdi") + if allele is None: + return None, "translate_from returned None" + return allele.id, None + except Exception as exc: # noqa: BLE001 + return None, f"{type(exc).__name__}: {exc}" + + +class Report: + def __init__(self) -> None: + self.passed = 0 + self.failed = 0 + + def check(self, name: str, ok: bool, details: str = "") -> bool: + mark = "PASS" if ok else "FAIL" + line = f" [{mark}] {name}" + if details: + line += f" — {details}" + print(line) + if ok: + self.passed += 1 + else: + self.failed += 1 + return ok + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--refget-store", + type=Path, + default=DEFAULT_STORE_PATH, + help="Path to refgetstore directory (default: %(default)s)", + ) + args = parser.parse_args() + + seqrepo_root = os.environ.get("SEQREPO_ROOT_DIR") + if not seqrepo_root: + print("ERROR: SEQREPO_ROOT_DIR not set", file=sys.stderr) + return 1 + if not args.refget_store.exists(): + print( + f"ERROR: refget store not found at {args.refget_store}", + file=sys.stderr, + ) + return 1 + + print(f"seqrepo: {seqrepo_root}") + print(f"refgetstore: {args.refget_store}") + + sr_dp = make_seqrepo_proxy(Path(seqrepo_root)) + rg_dp = make_refget_proxy(args.refget_store) + + r = Report() + + # Phase 1: Verify both backends can resolve the Ensembl accessions + print("\n=== Phase 1: Accession resolution ===") + accessions_seen: set[str] = set() + for case in SPDI_CASES: + ac = case.spdi.split(":")[0] + if ac in accessions_seen: + continue + accessions_seen.add(ac) + + sr_acc = sr_dp.derive_refget_accession(ac) + rg_acc = rg_dp.derive_refget_accession(ac) + r.check( + f"derive_refget_accession({ac})", + sr_acc is not None and rg_acc is not None and sr_acc == rg_acc, + f"seqrepo={sr_acc} refget={rg_acc}", + ) + + # Phase 2: Translate SPDI expressions and compare Allele IDs + print("\n=== Phase 2: SPDI → VRS Allele equivalence ===") + for case in SPDI_CASES: + sr_id, sr_err = translate_spdi(sr_dp, case.spdi) + rg_id, rg_err = translate_spdi(rg_dp, case.spdi) + + if sr_err: + r.check(f"{case.spdi} ({case.description})", False, f"seqrepo error: {sr_err}") + continue + if rg_err: + r.check(f"{case.spdi} ({case.description})", False, f"refget error: {rg_err}") + continue + + r.check( + f"{case.spdi} ({case.description})", + sr_id == rg_id, + f"seqrepo={sr_id} refget={rg_id}" if sr_id != rg_id else f"id={sr_id}", + ) + + # Phase 3: Verify sequence content agreement for test accessions + print("\n=== Phase 3: Sequence content spot-checks ===") + for ac in sorted(accessions_seen): + try: + sr_len = sr_dp.get_sequence_length(ac) + rg_len = rg_dp.get_sequence_length(ac) + except KeyError: + r.check(f"get_sequence_length({ac})", False, "KeyError on one or both backends") + continue + r.check( + f"get_sequence_length({ac})", + sr_len == rg_len, + f"seqrepo={sr_len} refget={rg_len}", + ) + # Compare first 100 bytes + sr_head = sr_dp.get_sequence(ac, start=0, end=min(100, sr_len)) + rg_head = rg_dp.get_sequence(ac, start=0, end=min(100, rg_len)) + r.check( + f"get_sequence({ac})[0:100]", + sr_head == rg_head, + f"{'match' if sr_head == rg_head else 'MISMATCH'}", + ) + + print() + print(f"{r.passed} passed, {r.failed} failed") + print("OVERALL:", "EQUIVALENT" if r.failed == 0 else "DIVERGENT") + return 0 if r.failed == 0 else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/misc/refgetstore/vcf_equivalence.py b/misc/refgetstore/vcf_equivalence.py new file mode 100644 index 00000000..f7b0bfcf --- /dev/null +++ b/misc/refgetstore/vcf_equivalence.py @@ -0,0 +1,316 @@ +"""Compare VRS annotation outputs from seqrepo vs refgetstore dataproxies. + +Runs the vrs-python VCF annotator over the same input VCF with two +dataproxy configurations and reports match rates + end-to-end wall time: + + 1. SeqRepoDataProxy (unbounded fd cache) — seqrepo baseline + 2. CompositeDataProxy(Refget, Refget-aliases) — fully refget-native + +Seqrepo is always run with ``fd_cache_size=None`` (unbounded file-descriptor +cache) so the comparison reflects seqrepo's steady-state performance rather +than per-record open/close cost. ``SEQREPO_LRU_CACHE_MAXSIZE`` defaults to +1_000_000 in seqrepo; export a larger value if your input has more than +that many distinct sequence fetches so the sequence cache doesn't evict. + +Everything runs single-threaded: pysam.VariantFile defaults to threads=1, +the translator path is Python-serial, and neither refget nor seqrepo uses +workers internally. + +Usage: + SEQREPO_ROOT_DIR=/path/to/seqrepo/2024-12-20 \\ + uv run python misc/refgetstore/vcf_equivalence.py \\ + --input-vcf ~/dev/vrs-rust/gnomad_chr1.1M.vcf.bgz \\ + --refget-store misc/refgetstore/store + +The refget store must already be built by build_store.py. The script does +NOT ingest FASTA — it only opens existing stores. + +Exits non-zero if any backend disagrees with the seqrepo baseline on +VRS_Allele_IDs. +""" + +from __future__ import annotations + +import argparse +import os +import sys +import time +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING + +from biocommons.seqrepo import SeqRepo + +from ga4gh.vrs.dataproxy import CompositeDataProxy, DataProxy +from ga4gh.vrs.dataproxy_refget import ( + RefgetAliasDataProxy, + RefgetSequenceDataProxy, +) +from ga4gh.vrs.dataproxy_seqrepo import SeqRepoDataProxy +from ga4gh.vrs.extras.annotator.vcf import VcfAnnotator + +if TYPE_CHECKING: + from gtars.refget import RefgetStore + + +# --------------------------------------------------------------------------- +# Dataproxy construction +# --------------------------------------------------------------------------- + + +def open_refget_store(store_path: Path) -> RefgetStore: + """Open an existing on-disk refget store. Errors if not present.""" + from gtars.refget import RefgetStore + + if not (store_path / "rgstore.json").exists(): + raise SystemExit( + f"refget store not found at {store_path}; " + f"run `uv run python misc/refgetstore/build_store.py` first" + ) + return RefgetStore.open_local(str(store_path)) + + +def make_seqrepo_proxy(seqrepo_root: Path) -> SeqRepoDataProxy: + """Construct a seqrepo-backed proxy with an unbounded fd cache. + + ``SEQREPO_FD_CACHE_MAXSIZE`` env var overrides this if set; otherwise + ``fd_cache_size=None`` means `functools.lru_cache(maxsize=None)` — no + eviction at any point, so the seqrepo baseline measures steady state. + """ + sr = SeqRepo(root_dir=str(seqrepo_root), fd_cache_size=None) + return SeqRepoDataProxy(sr) + + +def make_refget_native_proxy(store: RefgetStore) -> CompositeDataProxy: + """Composite: refget for both sequence bytes and alias lookups.""" + return CompositeDataProxy( + RefgetSequenceDataProxy(store), + RefgetAliasDataProxy(store), + ) + + +# --------------------------------------------------------------------------- +# Annotation + comparison +# --------------------------------------------------------------------------- + + +def annotate_vcf( + dp: DataProxy, + input_vcf: Path, + output_vcf: Path, + assembly: str, +) -> float: + """Run VCF annotation and return wall-clock seconds.""" + annotator = VcfAnnotator(dp) + t0 = time.perf_counter() + annotator.annotate( + input_vcf_path=input_vcf, + output_vcf_path=output_vcf, + vrs_attributes=True, + assembly=assembly, + compute_for_ref=True, + require_validation=False, + ) + return time.perf_counter() - t0 + + +@dataclass +class CompareResult: + total_records: int + id_matches: int + full_info_matches: int + id_mismatches: list[str] + + +def compare_vcf_outputs(path_a: Path, path_b: Path) -> CompareResult: + """Compare two annotated VCF files line-by-line on VRS INFO fields.""" + import pysam + + vcf_a = pysam.VariantFile(str(path_a)) + vcf_b = pysam.VariantFile(str(path_b)) + + total = 0 + id_matches = 0 + id_mismatches: list[str] = [] + full_info_matches = 0 + + vrs_fields = [ + "VRS_Allele_IDs", + "VRS_Starts", + "VRS_Ends", + "VRS_States", + "VRS_Lengths", + "VRS_Error", + ] + + for rec_a, rec_b in zip(vcf_a, vcf_b, strict=False): + total += 1 + ids_a = rec_a.info.get("VRS_Allele_IDs", None) + ids_b = rec_b.info.get("VRS_Allele_IDs", None) + + if ids_a == ids_b: + id_matches += 1 + else: + coord = ( + f"{rec_a.chrom}:{rec_a.pos}:{rec_a.ref}>{','.join(rec_a.alts or [])}" + ) + id_mismatches.append(f" {coord}: {ids_a} != {ids_b}") + + all_match = all( + rec_a.info.get(f, None) == rec_b.info.get(f, None) for f in vrs_fields + ) + if all_match: + full_info_matches += 1 + + vcf_a.close() + vcf_b.close() + + return CompareResult( + total_records=total, + id_matches=id_matches, + full_info_matches=full_info_matches, + id_mismatches=id_mismatches, + ) + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +def parse_args() -> argparse.Namespace: + home = Path.home() + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--input-vcf", + type=Path, + default=home / "dev/vrs-rust/gnomad_chr1.1M.vcf.bgz", + help="Input VCF (bgz/plain). Default: %(default)s", + ) + parser.add_argument( + "--seqrepo-root", + type=Path, + default=Path(os.environ.get("SEQREPO_ROOT_DIR", "")), + help="SeqRepo root dir (default: $SEQREPO_ROOT_DIR).", + ) + parser.add_argument( + "--refget-store", + type=Path, + default=Path("misc/refgetstore/store"), + help="On-disk refget store built by build_store.py.", + ) + parser.add_argument( + "--assembly", + default="GRCh38", + help="Assembly passed to VcfAnnotator.annotate().", + ) + parser.add_argument( + "--output-dir", + type=Path, + default=Path(".pytest_tmp/vcf_equivalence"), + help="Directory for annotated VCF outputs.", + ) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + + if not args.input_vcf.exists(): + print(f"Error: input VCF not found: {args.input_vcf}", file=sys.stderr) + return 2 + if not args.seqrepo_root or not args.seqrepo_root.exists(): + print( + f"Error: seqrepo root not found: {args.seqrepo_root} " + f"(set --seqrepo-root or $SEQREPO_ROOT_DIR)", + file=sys.stderr, + ) + return 2 + + args.output_dir.mkdir(parents=True, exist_ok=True) + out_seqrepo = args.output_dir / "annotated_seqrepo.vcf" + out_refget = args.output_dir / "annotated_refget.vcf" + + # --- Setup --- + print("=" * 70) + print("VCF Equivalence: seqrepo vs refgetstore") + print("=" * 70) + print(f" Input VCF: {args.input_vcf}") + print(f" SeqRepo root: {args.seqrepo_root}") + print(f" Refget store: {args.refget_store}") + print(f" Assembly: {args.assembly}") + print(f" Output dir: {args.output_dir}") + print( + f" SEQREPO_FD_CACHE_MAXSIZE={os.environ.get('SEQREPO_FD_CACHE_MAXSIZE', '(unset, constructor=None=unbounded)')}" + ) + print( + f" SEQREPO_LRU_CACHE_MAXSIZE={os.environ.get('SEQREPO_LRU_CACHE_MAXSIZE', '(unset, default=1_000_000)')}" + ) + print() + + print("[setup] Opening refget store...") + store = open_refget_store(args.refget_store) + + print("[setup] Building dataproxies...") + seqrepo_proxy = make_seqrepo_proxy(args.seqrepo_root) + refget_native_proxy = make_refget_native_proxy(store) + + timings: dict[str, float] = {} + outputs: dict[str, Path] = {} + + # --- Annotations (single-threaded; pysam.VariantFile defaults to threads=1) --- + + print(f"\n[run] seqrepo (unbounded fd cache) -> {out_seqrepo}") + t = annotate_vcf(seqrepo_proxy, args.input_vcf, out_seqrepo, args.assembly) + timings["seqrepo"] = t + outputs["seqrepo"] = out_seqrepo + print(f" done in {t:.2f}s") + + print(f"\n[run] refget native -> {out_refget}") + t = annotate_vcf(refget_native_proxy, args.input_vcf, out_refget, args.assembly) + timings["refget_native"] = t + outputs["refget_native"] = out_refget + print(f" done in {t:.2f}s") + + # --- Compare --- + baseline_key = "seqrepo" + baseline_path = outputs[baseline_key] + + print("\n" + "=" * 70) + print("Results") + print("=" * 70) + + all_match = True + for key, path in outputs.items(): + if key == baseline_key: + continue + r = compare_vcf_outputs(baseline_path, path) + total = r.total_records + print( + f" {key:25s} VRS_Allele_IDs={r.id_matches}/{total} " + f"full={r.full_info_matches}/{total} " + f"mismatches={len(r.id_mismatches)}" + ) + if r.id_mismatches: + all_match = False + n_show = min(5, len(r.id_mismatches)) + for line in r.id_mismatches[:n_show]: + print(line) + if len(r.id_mismatches) > n_show: + print(f" ... and {len(r.id_mismatches) - n_show} more") + + print("\n Timing:") + base = timings[baseline_key] + for key, t in timings.items(): + ratio = f" ({base / t:.2f}x vs seqrepo)" if key != baseline_key else "" + print(f" {key:25s} {t:>8.2f}s{ratio}") + + if all_match: + print("\nPASS: all backends agree on VRS_Allele_IDs") + return 0 + print("\nFAIL: at least one backend diverged from seqrepo baseline") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/misc/refgetstore/verify_store.py b/misc/refgetstore/verify_store.py new file mode 100644 index 00000000..377cbe4f --- /dev/null +++ b/misc/refgetstore/verify_store.py @@ -0,0 +1,479 @@ +#!/usr/bin/env python +"""Verify the custom refgetstore built by build_store.py. + +Checks that every sequence referenced by every assembly report was loaded, +that the alias graph contains only the namespaces vrs-python needs, and +that a handful of digests match the seqrepo 2024-12-20 ground truth. + +Exits 0 on full success, 1 on any failure. Each check prints PASS/FAIL +on its own line; final line is OVERALL PASS or OVERALL FAIL. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +from gtars.refget import RefgetStore + +HERE = Path(__file__).resolve().parent +STORE_PATH = HERE / "store" +DOWNLOAD_DIR = HERE / "downloads" + +EXPECTED_SEQ_NAMESPACES = { + "GRCh38", + "GRCh38.p14", + "GRCh37", + "GRCh37.p13", + "refseq", + "insdc", + "ensembl", +} +EXPECTED_COLL_NAMESPACES = {"refseq", "insdc"} +FORBIDDEN_SEQ_NAMESPACES = { + "sha512t24u", + "ga4gh", + "SEGUID", + "SHA1", + "VMC", + "NCBI", + "MD5", +} + +# (namespace, alias) -> expected sha512t24u, captured from seqrepo 2024-12-20 +# and the refgenie jungle store during earlier analysis in this session. +GROUND_TRUTH: dict[tuple[str, str], str] = { + ("refseq", "NC_000001.11"): "Ya6Rs7DHhDeg7YaOSg1EoNi3U_nQ9SvO", + ("refseq", "NC_000013.11"): "_0wi-qoDrvram155UmcSC-zA5ZK4fpLT", + ("GRCh38", "chr1"): "Ya6Rs7DHhDeg7YaOSg1EoNi3U_nQ9SvO", + ("GRCh38", "chrM"): "k3grVkjY-hoWcCUojHw6VU6GE3MZ8Sct", + ("GRCh38", "chrX"): "w0WZEvgJF0zf_P4yyTzjjv9oW1z61HHP", + ("GRCh38", "chrY"): "8_liLu1aycC0tPQPFmUaGXJLDs5SbPZ5", + ("GRCh38", "chr22"): "7B7SHsmchAR0dFcDCuSFjJAo7tX87krQ", + ("GRCh38.p14", "chr1"): "Ya6Rs7DHhDeg7YaOSg1EoNi3U_nQ9SvO", + ("insdc", "CM000663.2"): "Ya6Rs7DHhDeg7YaOSg1EoNi3U_nQ9SvO", +} + +EXPECTED_COLL_REFSEQ_ALIASES = { + "GCF_000001405.26", + "GCF_000001405.40", + "GCF_000001405.13", + "GCF_000001405.25", +} + +# (namespace, alias) -> (sha512t24u, length). Captured from the build_store.py +# run that ingested Ensembl release 113 FASTAs. These are the canonical +# latest versions in release 113 for a few well-known genes across all three +# biotypes we load (cdna, ncrna, pep). Bumping the Ensembl release will +# likely invalidate some of these — update after the rebuild. +ENSEMBL_GROUND_TRUTH: dict[tuple[str, str], tuple[str, int]] = { + # cdna (protein-coding transcript) + ("ensembl", "ENST00000256474.3"): ("xBKOKptLLDr-k4hTyCetvARn16pDS_rW", 4414), # VHL + ("ensembl", "ENST00000357654.9"): ("oR9jzdHf6J23TozeyuvTXtwlm6PpUHHl", 7088), # BRCA1 + # ncrna (non-coding transcript) + ("ensembl", "ENST00000429829.6"): ("FHnP3_NSzX0WZLjoHIPFjv89fSl18Xwk", 19245), # XIST lncRNA + # pep (protein) + ("ensembl", "ENSP00000256474.3"): ("z-Oa0pZkJ6GHJHOYM7h5mY_umc0SJzTu", 213), # VHL + ("ensembl", "ENSP00000350283.3"): ("nUzIPnHMyQV52hzgBbKl5vlbSwx8M8_Y", 1863), # BRCA1 + ("ensembl", "ENSP00000269305.4"): ("KAxM06sYzBF6zFftFaYq9E_18wsnn7al", 393), # TP53 +} + + +class Report: + def __init__(self) -> None: + self.passed = 0 + self.failed = 0 + + def check(self, name: str, ok: bool, details: str = "") -> bool: + mark = "PASS" if ok else "FAIL" + line = f" [{mark}] {name}" + if details: + line += f" — {details}" + print(line) + if ok: + self.passed += 1 + else: + self.failed += 1 + return ok + + +def parse_assembly_report( + path: Path, +) -> tuple[str | None, str | None, list[dict[str, str]]]: + """Mini copy of build_store.parse_assembly_report kept inline so this + script is self-contained. + """ + refseq_accn: str | None = None + genbank_accn: str | None = None + columns: list[str] | None = None + rows: list[dict[str, str]] = [] + with path.open("r", encoding="utf-8") as fh: + for raw in fh: + line = raw.rstrip("\n\r") + if not line: + continue + if line.startswith("#"): + stripped = line.lstrip("#").strip() + if stripped.startswith("GenBank assembly accession:"): + genbank_accn = stripped.split(":", 1)[1].strip() or None + elif stripped.startswith("RefSeq assembly accession:"): + refseq_accn = stripped.split(":", 1)[1].strip() or None + elif "\t" in stripped and "Sequence-Name" in stripped: + columns = stripped.split("\t") + continue + if columns is None: + continue + parts = line.split("\t") + if len(parts) != len(columns): + continue + rows.append(dict(zip(columns, parts))) + return refseq_accn, genbank_accn, rows + + +def section_inventory(store: RefgetStore, r: Report) -> None: + print("\n[A] Store inventory") + stats = store.stats() + print(f" stats: {stats}") + # 4 assembly collections + 16 RNA shards + 16 protein shards = 36. Lower + # bound of 4 keeps the assembly-only configuration passing too. + r.check( + "n_collections >= 4", + int(stats["n_collections"]) >= 4, + f"got {stats['n_collections']}", + ) + n_seq = int(stats["n_sequences"]) + # 941 observed on the first full build (Phase 1). Heavy dedup across + # GRCh38/p14 and GRCh37/p13 keeps this much lower than the naive sum. + r.check( + "n_sequences >= 900", + n_seq >= 900, + f"got {n_seq}", + ) + seq_ns = set(store.list_sequence_alias_namespaces()) + r.check( + "sequence namespaces == expected set", + seq_ns == EXPECTED_SEQ_NAMESPACES, + f"got {sorted(seq_ns)}", + ) + coll_ns = set(store.list_collection_alias_namespaces()) + r.check( + "collection namespaces == expected set", + coll_ns == EXPECTED_COLL_NAMESPACES, + f"got {sorted(coll_ns)}", + ) + + +def section_forbidden(store: RefgetStore, r: Report) -> None: + print("\n[B] Forbidden namespace check") + seq_ns = set(store.list_sequence_alias_namespaces()) + intersection = FORBIDDEN_SEQ_NAMESPACES & seq_ns + r.check( + "no forbidden sequence alias namespaces", + len(intersection) == 0, + f"intersection: {sorted(intersection)}", + ) + + +def section_ground_truth(store: RefgetStore, r: Report) -> None: + print("\n[C] Ground-truth digest cross-check (vs seqrepo 2024-12-20)") + for (ns, alias), expected in GROUND_TRUTH.items(): + rec = store.get_sequence_by_alias(ns, alias) + actual = rec.metadata.sha512t24u if rec is not None else None + r.check( + f"{ns}:{alias}", + actual == expected, + f"expected {expected}, got {actual}", + ) + # metadata fidelity + rec = store.get_sequence_by_alias("refseq", "NC_000001.11") + if rec is None: + r.check("NC_000001.11 metadata length", False, "record not found") + r.check("NC_000001.11 metadata md5", False, "record not found") + else: + r.check( + "refseq:NC_000001.11 length == 248956422", + rec.metadata.length == 248956422, + f"got {rec.metadata.length}", + ) + r.check( + "refseq:NC_000001.11 md5 matches", + rec.metadata.md5 == "6aef897c3d6ff0c78aff06ac189178dd", + f"got {rec.metadata.md5}", + ) + + +def section_coverage(store: RefgetStore, r: Report) -> None: + print("\n[D] Assembly report coverage") + for namespace in ("GRCh38", "GRCh38.p14", "GRCh37", "GRCh37.p13"): + report_path = DOWNLOAD_DIR / f"{namespace}.assembly_report.txt" + if not report_path.exists(): + r.check(f"{namespace} report present", False, f"{report_path} missing") + continue + _, _, rows = parse_assembly_report(report_path) + miss_refseq: list[str] = [] + miss_ucsc: list[str] = [] + miss_insdc_ns: list[str] = [] + miss_insdc_global: list[str] = [] + total = 0 + for row in rows: + refseq_ac = row.get("RefSeq-Accn", "").strip() + ucsc = row.get("UCSC-style-name", "").strip() + genbank = row.get("GenBank-Accn", "").strip() + # Scope mirrors build_store.add_aliases_for_row: we only alias + # sequences that are actually in the RefSeq dataset, so skip + # rows where RefSeq-Accn is "na" (GenBank-only contigs that + # don't appear in the RefSeq _genomic.fna.gz). + if not refseq_ac or refseq_ac.lower() == "na": + continue + total += 1 + if store.get_sequence_by_alias("refseq", refseq_ac) is None: + miss_refseq.append(refseq_ac) + if ucsc and ucsc.lower() != "na": + if store.get_sequence_by_alias(namespace, ucsc) is None: + miss_ucsc.append(ucsc) + if genbank and genbank.lower() != "na": + if store.get_sequence_by_alias(namespace, genbank) is None: + miss_insdc_ns.append(genbank) + if store.get_sequence_by_alias("insdc", genbank) is None: + miss_insdc_global.append(genbank) + r.check( + f"{namespace}: refseq coverage ({total} rows)", + not miss_refseq, + f"{len(miss_refseq)} missing, first: {miss_refseq[:3]}", + ) + r.check( + f"{namespace}: UCSC coverage", + not miss_ucsc, + f"{len(miss_ucsc)} missing, first: {miss_ucsc[:3]}", + ) + r.check( + f"{namespace}: GenBank coverage ({namespace}:)", + not miss_insdc_ns, + f"{len(miss_insdc_ns)} missing, first: {miss_insdc_ns[:3]}", + ) + r.check( + f"{namespace}: GenBank coverage (insdc:)", + not miss_insdc_global, + f"{len(miss_insdc_global)} missing, first: {miss_insdc_global[:3]}", + ) + + +def section_invariants(store: RefgetStore, r: Report) -> None: + print("\n[E] Cross-patch / cross-build invariants") + + def digest_for(ns: str, alias: str) -> str | None: + rec = store.get_sequence_by_alias(ns, alias) + return rec.metadata.sha512t24u if rec is not None else None + + grch38_chr1 = digest_for("GRCh38", "chr1") + p14_chr1 = digest_for("GRCh38.p14", "chr1") + grch37_chr1 = digest_for("GRCh37", "chr1") + p13_chr1 = digest_for("GRCh37.p13", "chr1") + grch38_chrm = digest_for("GRCh38", "chrM") + # GRCh37 base has no mitochondrion contig; p13 is the first to include it + # as MT/chrM/NC_012920.1. Compare against p13 (and refseq:NC_012920.1). + grch37p13_chrm = digest_for("GRCh37.p13", "chrM") + refseq_mt = digest_for("refseq", "NC_012920.1") + + r.check( + "GRCh38:chr1 == GRCh38.p14:chr1", + grch38_chr1 is not None and grch38_chr1 == p14_chr1, + f"{grch38_chr1} vs {p14_chr1}", + ) + r.check( + "GRCh37:chr1 == GRCh37.p13:chr1", + grch37_chr1 is not None and grch37_chr1 == p13_chr1, + f"{grch37_chr1} vs {p13_chr1}", + ) + r.check( + "GRCh38:chr1 != GRCh37:chr1", + grch38_chr1 is not None + and grch37_chr1 is not None + and grch38_chr1 != grch37_chr1, + f"{grch38_chr1} vs {grch37_chr1}", + ) + r.check( + "GRCh38:chrM == GRCh37.p13:chrM (mito stable across builds)", + grch38_chrm is not None and grch38_chrm == grch37p13_chrm, + f"{grch38_chrm} vs {grch37p13_chrm}", + ) + r.check( + "GRCh38:chrM == refseq:NC_012920.1 (mito accession round-trip)", + grch38_chrm is not None and grch38_chrm == refseq_mt, + f"{grch38_chrm} vs {refseq_mt}", + ) + + +def section_sequence_bytes(store: RefgetStore, r: Report) -> None: + print("\n[F] Sequence data smoke test") + rec = store.get_sequence_by_alias("refseq", "NC_000001.11") + if rec is None: + r.check("retrieve chr1 slice", False, "NC_000001.11 not found") + return + digest = rec.metadata.sha512t24u + try: + store.load_sequence(digest) # type: ignore[attr-defined] + data = store.get_substring(digest, 1_000_000, 1_000_050) + except Exception as exc: # noqa: BLE001 + r.check("retrieve chr1 slice", False, f"load/get_substring raised: {exc}") + return + r.check( + "chr1 slice length == 50", + data is not None and len(data) == 50, + f"got len={len(data) if data else None}", + ) + r.check( + "chr1 slice is ACGTN-only", + data is not None and set(data.upper()) <= set("ACGTN"), + f"unexpected chars: {sorted(set(data.upper()) - set('ACGTN')) if data else None}", + ) + + +def section_collection_aliases(store: RefgetStore, r: Report) -> None: + print("\n[G] Collection aliases") + refseq = set(store.list_collection_aliases("refseq") or []) + r.check( + "refseq collection aliases >= expected set", + EXPECTED_COLL_REFSEQ_ALIASES <= refseq, + f"got {sorted(refseq)}", + ) + insdc = store.list_collection_aliases("insdc") or [] + r.check( + "insdc collection aliases has 4 entries", + len(insdc) >= 4, + f"got {sorted(insdc)}", + ) + + +def section_ensembl(store: RefgetStore, r: Report) -> None: + """Ensembl coverage checks. + + Asserts that each canonical ENST/ENSP identifier in + ``ENSEMBL_GROUND_TRUTH`` resolves, its metadata matches the captured + digest and length, and that at least one sequence's bytes can be + read via ``get_substring`` (smoke-tests the full read path through + ``RefgetSequenceDataProxy._ensure_sequence_loaded``). + """ + print("\n[H] Ensembl coverage smoke test") + for (ns, alias), (expected_digest, expected_length) in ENSEMBL_GROUND_TRUTH.items(): + rec = store.get_sequence_by_alias(ns, alias) + if rec is None: + r.check(f"{ns}:{alias}", False, "record not found") + continue + md = rec.metadata + r.check( + f"{ns}:{alias} digest", + md.sha512t24u == expected_digest, + f"expected {expected_digest}, got {md.sha512t24u}", + ) + r.check( + f"{ns}:{alias} length == {expected_length}", + md.length == expected_length, + f"got {md.length}", + ) + + # Sequence bytes smoke test on VHL cDNA — proves the FASTA payload + # made it to disk for an Ensembl-loaded seqset, not just the alias row. + rec = store.get_sequence_by_alias("ensembl", "ENST00000256474.3") + if rec is None: + r.check("VHL cDNA byte fetch", False, "record not found") + return + digest = rec.metadata.sha512t24u + try: + store.load_sequence(digest) # type: ignore[attr-defined] + data = store.get_substring(digest, 0, 50) + except Exception as exc: # noqa: BLE001 + r.check("VHL cDNA byte fetch", False, f"load/get_substring raised: {exc}") + return + r.check( + "VHL cDNA slice length == 50", + data is not None and len(data) == 50, + f"got len={len(data) if data else None}", + ) + r.check( + "VHL cDNA slice is ACGTN-only", + data is not None and set(data.upper()) <= set("ACGTN"), + f"unexpected chars: {sorted(set(data.upper()) - set('ACGTN')) if data else None}", + ) + + +KNOWN_DIVERGENT_PATH = HERE / "ensembl_known_divergent.txt" + + +def section_known_divergent(store: RefgetStore, r: Report) -> None: + """Regression test for Ensembl accessions with known seqrepo digest divergence. + + Loads the list of accessions from ``ensembl_known_divergent.txt`` and + verifies that each one still resolves in the store with the expected + sha512t24u digest. A change here means the store was rebuilt with + different Ensembl data and the divergent list needs refreshing. + """ + print("\n[I] Known-divergent Ensembl accessions") + if not KNOWN_DIVERGENT_PATH.exists(): + r.check("known-divergent file exists", False, f"not found at {KNOWN_DIVERGENT_PATH}") + return + + entries: list[tuple[str, str]] = [] + with KNOWN_DIVERGENT_PATH.open("r") as f: + for line in f: + line = line.strip() + if not line or line.startswith("#"): + continue + parts = line.split("\t") + if len(parts) >= 3: + # accession, seqrepo_seq_id, refget_sha512t24u, [cause] + entries.append((parts[0], parts[2])) + + r.check( + f"known-divergent entries loaded ({len(entries)})", + len(entries) >= 100, + f"got {len(entries)}", + ) + + # Spot-check first 10 entries: verify each resolves with expected digest + checked = 0 + for accession, expected_digest in entries[:10]: + rec = store.get_sequence_by_alias("ensembl", accession) + if rec is None: + r.check(f"{accession} resolves", False, "not found in store") + continue + r.check( + f"{accession} digest unchanged", + rec.metadata.sha512t24u == expected_digest, + f"expected {expected_digest}, got {rec.metadata.sha512t24u}", + ) + checked += 1 + + r.check( + f"spot-checked {checked} known-divergent entries", + checked >= 10, + f"checked {checked}", + ) + + +def main() -> int: + if not STORE_PATH.exists(): + print(f"ERROR: store not found at {STORE_PATH}", file=sys.stderr) + return 2 + store = RefgetStore.open_local(str(STORE_PATH)) + store.pull_aliases() + + r = Report() + section_inventory(store, r) + section_forbidden(store, r) + section_ground_truth(store, r) + section_coverage(store, r) + section_invariants(store, r) + section_sequence_bytes(store, r) + section_collection_aliases(store, r) + section_ensembl(store, r) + section_known_divergent(store, r) + + print() + print(f"{r.passed} passed, {r.failed} failed") + print("OVERALL:", "PASS" if r.failed == 0 else "FAIL") + return 0 if r.failed == 0 else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/pyproject.toml b/pyproject.toml index afd6613c..c818ec04 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,6 +72,18 @@ notebooks = [ "jupyter", "pyyaml" ] +refget = [ + "gtars>=0.11.0", +] + +[tool.uv.sources] +# No gtars release yet includes databio/gtars#273, which fixes residues that +# RefgetStore's encoded mode returned differently than imported, and stores +# written with it are misread by gtars <= 0.10 without error. This pins the +# PR head plus the fix that makes its Python bindings compile. Replace with the +# version floor alone once a release includes #273. Building it needs a Rust +# toolchain. +gtars = { git = "https://github.com/theferrit32/gtars", rev = "7831e09f596042e3983c7a123c3f6c019ca6c05f", subdirectory = "gtars-python" } [project.urls] Homepage = "https://github.com/ga4gh/vrs-python" @@ -171,7 +183,8 @@ ignore = [ # ANN001 - missing-type-function-argument # ANN2 - missing-return-type # D100 - undocumented-public-module -# D102 - undocumented-public-class +# D101 - undocumented-public-class +# D102 - undocumented-public-method # D103 - undocumented-public-function # S101 - assert # B011 - assert-false @@ -181,6 +194,7 @@ ignore = [ "ANN001", "ANN2", "D100", + "D101", "D102", "D103", "S101", diff --git a/src/ga4gh/vrs/dataproxy.py b/src/ga4gh/vrs/dataproxy.py index 32b8d561..ffc95d7e 100644 --- a/src/ga4gh/vrs/dataproxy.py +++ b/src/ga4gh/vrs/dataproxy.py @@ -1,285 +1,495 @@ -"""provides an abstract class for all data access required for -VRS, and a concrete implementation based on seqrepo. +"""Provides abstract interfaces for all data access required by VRS, +and a concrete implementation based on seqrepo. See https://vr-spec.readthedocs.io/en/1.1/impl-guide/required_data.html """ -import datetime +from __future__ import annotations + import functools import logging import os +import re from abc import ABC, abstractmethod -from collections.abc import Sequence +from collections.abc import Callable, Sequence from urllib.parse import urlparse -import requests -from bioutils.accessions import coerce_namespace +from ga4gh.vrs.models import Range _logger = logging.getLogger(__name__) +def resolve_cache_size(env_var: str, default: str) -> int | None: + """Resolve an :func:`functools.lru_cache` ``maxsize`` from the environment. + + Accepts an integer, or ``-1``/``"none"`` for an unbounded cache. ``0`` + disables caching, per ``functools`` semantics. Read at import time, since + the decorators are applied when the class bodies are evaluated. + + :raises ValueError: if the value is neither an integer nor ``"none"`` + """ + value = os.environ.get(env_var) or default + if value.strip().lower() == "none": + return None + try: + size = int(value) + except ValueError as e: + msg = f"{env_var} must be an integer or 'none', got {value!r}" + raise ValueError(msg) from e + return None if size < 0 else size + + +# maxsize for caches of small fixed-size values: lengths, digests, aliases. +DATAPROXY_CACHE_SIZE = resolve_cache_size("GA4GH_VRS_DATAPROXY_CACHE_SIZE", "8192") + +# maxsize for the REST backend's sequence payload cache. Much smaller by default +# because entries are bounded only by the caller's span: the normalization path +# stores 1-2 base strings, but an unbounded fetch stores a whole contig per +# entry. This is the only get_sequence cache; the local SeqRepo and refget +# backends do not cache sequence payloads. +# +# Note this memoizes fetched *regions*, keyed on the exact (identifier, start, +# end) triple — not sequences. An overlapping span is a miss, and a cached +# full-sequence entry never serves a sub-range request, so raising this only +# helps when identical regions are re-fetched. +REST_SEQUENCE_CACHE_SIZE = resolve_cache_size( + "GA4GH_VRS_REST_DATAPROXY_SEQUENCE_CACHE_SIZE", "128" +) + + +# LRG genomic (LRG_1, LRG_1g), transcript (LRG_1t1) and protein (LRG_1p1) +# accessions. bioutils.accessions does not recognize LRG. +_LRG_ACCESSION_RE = re.compile(r"^LRG_\d+(?:g|[tp]\d+)?$") + + +def coerce_accession_namespace(ac: str) -> str: + """Prefix a bare accession with its inferred namespace. + + As :func:`bioutils.accessions.coerce_namespace`, but also infers ``lrg`` + for Locus Reference Genomic accessions. An accession that already has a + namespace is returned unchanged. + + >>> coerce_accession_namespace("LRG_1t1") + 'lrg:LRG_1t1' + >>> coerce_accession_namespace("NM_000551.3") + 'refseq:NM_000551.3' + + :raises ValueError: if no namespace can be inferred + """ + if ":" not in ac and _LRG_ACCESSION_RE.match(ac): + return f"lrg:{ac}" + from bioutils.accessions import coerce_namespace + + return coerce_namespace(ac) + + class DataProxyValidationError(Exception): """Class for validation errors during data proxy methods""" -class _DataProxy(ABC): - """abstract class / interface for VRS data needs +def validate_start_le_end(identifier: str, start: int | None, end: int | None) -> None: + """Reject an interval whose ``start`` is after its ``end``, since vrs-python treats + every sequence as linear. - The proxy MUST support the use of GA4GH sequence identifers (i.e., - `ga4gh:SQ...`) as keys, and return these identifiers among the - aliases for a sequence. These identifiers may be supported - natively by the data source or synthesized by the proxy from the - data source or synthesized. + :raises DataProxyValidationError: If both are defined and ``start > end`` + """ + if start is not None and end is not None and start > end: + err_msg = f"Interval start must not be after end on {identifier}: start={start}, end={end}" + raise DataProxyValidationError(err_msg) + +class UnresolvableAccessionError(KeyError): + """An accession that the data proxy cannot resolve to a refget accession. + + A ``KeyError``, like any other lookup miss, but distinguishable from one + raised deeper in translation, so callers can skip an input on a sequence + the backend lacks without masking other failures. + """ + + def __init__(self, accession: str) -> None: + """Initialize with the accession that failed to resolve.""" + super().__init__(accession) + self.accession = accession + + def __str__(self) -> str: + """Describe the failure (``KeyError`` would show only the quoted key).""" + return f"Unable to resolve {self.accession} to a refget accession" + + +class SequenceDataProxy(ABC): + """Abstract interface for sequence data access. + + Provides methods for retrieving genomic sequences and their lengths. """ @abstractmethod def get_sequence( self, identifier: str, start: int | None = None, end: int | None = None ) -> str: - """Return the specified sequence or subsequence + """Return the specified sequence or subsequence. - start and end are optional + start and end are optional. - If the given sequence does not exist, KeyError is raised. + If the given sequence does not exist, KeyError is raised. If start is + after end, DataProxyValidationError may be raised. >> dp.get_sequence("NM_000551.3", 0, 10) 'CCTCGCCTCC' - """ @abstractmethod - def get_metadata(self, identifier: str) -> dict: - """For a given identifier, return a structure (dict) containing - sequence length, aliases, and other optional info + def get_sequence_length(self, identifier: str) -> int: + """Return the length of the specified sequence. If the given sequence does not exist, KeyError is raised. - >> dp.get_metadata("NM_000551.3") - {'added': '2016-08-24T05:03:11Z', - 'aliases': ['MD5:215137b1973c1a5afcf86be7d999574a', - 'RefSeq:NM_000551.3', - 'SEGUID:T12L0p2X5E8DbnL0+SwI4Wc1S6g', - 'SHA1:4f5d8bd29d97e44f036e72f4f92c08e167354ba8', - 'ga4gh:SQ.v_QTc1p-MUYdgrRv4LMT6ByXIOsdw3C_', - 'gi:319655736'], - 'alphabet': 'ACGT', - 'length': 4560} + >> dp.get_sequence_length("NM_000551.3") + 4560 + """ + def validate_ref_seq( + self, + sequence_id: str, + start_pos: int, + end_pos: int, + ref: str, + require_validation: bool = True, + ) -> None: + """Determine whether or not the expected reference sequence matches the actual + reference sequence. Returns ``None``, but invalid results are logged at level + WARN by default. If ``require_validation`` is ``True``, then invalid data will + cause a ``DataProxyValidationError`` to be raised. + + :param sequence_id: Sequence ID to use + :param start_pos: Start pos (inter-residue) on the sequence_id + :param end_pos: End pos (inter-residue) on the sequence_id + :param ref: The expected reference sequence on the sequence_id given the + start_pos and end_pos + :param require_validation: If ``True`` and if validation checks fail, a + ``DataProxyValidationError`` will be raised. Error message will always be + logged. + :raises DataProxyValidationError: If excepted reference sequence does not match + the actual reference sequence and ``require_validation`` is ``True``. + Raised regardless of ``require_validation`` if the location is invalid + (see ``validate_location_bounds``). """ + # An out-of-bounds fetch may be silently truncated, which would be misreported + # as a reference mismatch + self.validate_location_bounds(sequence_id, start_pos, end_pos) + correct_ref = self.get_sequence(sequence_id, start_pos, end_pos) - @staticmethod - def extract_sequence_type(alias: str) -> str | None: - """Provide a convenient way to extract the sequence type from an accession by matching its prefix to a known set of prefixes. + if correct_ref != ref: + err_msg = f"Reference mismatch at {sequence_id} position {start_pos}-{end_pos} (input gave '{ref}' but correct ref is '{correct_ref}')" + _logger.warning(err_msg) - Args: - alias (str): The accession string. + if require_validation: + raise DataProxyValidationError(err_msg) - Returns: - str or None: The sequence type associated with the accession string, or None if no matching prefix is found. + def validate_location_bounds( + self, + sequence_id: str, + start_pos: int | Range | None, + end_pos: int | Range | None, + ) -> None: + """Ensure that ``start_pos`` and ``end_pos`` are representable on ``sequence_id``. + + Each defined coordinate must be within ``[0, len(sequence)]`` (inter-residue). + Undefined (``None``) endpoints are skipped, and for a ``Range`` every defined + member is checked. + ``start_pos`` must also not be after ``end_pos``, comparing a ``Range`` by its + lowest possible start and highest possible end. + + There is no ``require_validation`` option: an out-of-bounds location has no + meaning, so the error is always raised, including from ``validate_ref_seq``. + Sequence backends may silently truncate out-of-range fetches, so this check + must be made before relying on fetched sequence. + + :param sequence_id: Sequence ID to use + :param start_pos: Start pos (inter-residue) on the sequence_id + :param end_pos: End pos (inter-residue) on the sequence_id + :raises DataProxyValidationError: If a defined coordinate is out of bounds, or + ``start_pos`` is after ``end_pos`` + :raises KeyError: If ``sequence_id`` is not found """ - prefix_dict = { - "refseq:NM_": "c", - "refseq:NC_012920": "m", - "refseq:NG_": "g", - "refseq:NC_00": "g", - "refseq:NW_": "g", - "refseq:NT_": "g", - "refseq:NR_": "n", - "refseq:NP_": "p", - "refseq:XM_": "c", - "refseq:XR_": "n", - "refseq:XP_": "p", - "GRCh": "g", - } + # same cache key as derive_refget_accession + sequence_id = coerce_accession_namespace(sequence_id) + seq_len = self.get_sequence_length(sequence_id) + bad = [] + for name, pos in (("start", start_pos), ("end", end_pos)): + values = pos.root if isinstance(pos, Range) else [pos] + if any(v is not None and not 0 <= v <= seq_len for v in values): + bad.append((name, pos)) + if bad: + # Range is shown as its list form, e.g. end=[4500, 4600] + detail = ", ".join( + f"{name}={pos.root if isinstance(pos, Range) else pos}" + for name, pos in bad + ) + err_msg = ( + f"Location out of bounds on {self._sequence_label(sequence_id)}: " + f"{detail} not within [0, {seq_len}]" + ) + raise DataProxyValidationError(err_msg) - for prefix, seq_type in prefix_dict.items(): - if alias.startswith(prefix): - return seq_type - return None + validate_start_le_end( + sequence_id, + start_pos.root[0] if isinstance(start_pos, Range) else start_pos, + end_pos.root[1] if isinstance(end_pos, Range) else end_pos, + ) + + def _sequence_label(self, sequence_id: str) -> str: + """Name a sequence in an error message. A proxy that can resolve aliases + also names the refget accession. + """ + return sequence_id + + +class AliasDataProxy(ABC): + """Abstract interface for sequence alias / identifier resolution. + + Provides methods for translating between different sequence identifier + namespaces and deriving refget accessions. + """ + + @abstractmethod + def get_aliases(self, identifier: str) -> list[str]: + """Return a list of aliases for the given sequence identifier. + + Each alias is a string of the form ``namespace:accession``. + + If the given sequence does not exist, KeyError is raised. + + >> dp.get_aliases("NM_000551.3") + ['MD5:215137b1973c1a5afcf86be7d999574a', + 'RefSeq:NM_000551.3', + 'SEGUID:T12L0p2X5E8DbnL0+SwI4Wc1S6g', + 'SHA1:4f5d8bd29d97e44f036e72f4f92c08e167354ba8', + 'ga4gh:SQ.v_QTc1p-MUYdgrRv4LMT6ByXIOsdw3C_', + 'gi:319655736'] + """ - @functools.lru_cache def translate_sequence_identifier( self, identifier: str, namespace: str | None = None ) -> list[str]: """Translate given identifier to a list of identifiers in the specified namespace. - `identifier` must be a string - `namespace` is case-sensitive + ``identifier`` must be a string. + ``namespace`` is case-sensitive. - On success, returns string identifier. Raises KeyError if given + On success, returns list of string identifiers. Raises KeyError if given identifier isn't found. - """ try: - md = self.get_metadata(identifier) + aliases = self.get_aliases(identifier) except (ValueError, KeyError, IndexError) as e: raise KeyError(identifier) from e - aliases = list(set(md["aliases"])) # ensure uniqueness + aliases = list(set(aliases)) # ensure uniqueness if namespace is not None: nsd = namespace + ":" aliases = [a for a in aliases if a.startswith(nsd)] return aliases def derive_refget_accession(self, ac: str) -> str | None: - """Derive the refget accession from a public accession identifier + """Derive the refget accession from a public accession identifier. :param ac: public accession in simple or curie form from which to derive the refget accession - :return: Refget Accession if found + :return: Refget Accession if found, else ``None`` (including for a bare + accession whose namespace cannot be inferred) + :raises ValueError: if the backend reports more than one ``ga4gh:`` alias """ if ac is None: return None if ":" not in ac[1:]: # always coerce the namespace if none provided - ac = coerce_namespace(ac) + try: + ac = coerce_accession_namespace(ac) + except ValueError: + _logger.debug("Unable to infer a namespace for %s", ac) + return None refget_accession = None try: aliases = self.translate_sequence_identifier(ac, namespace="ga4gh") except KeyError: - _logger.exception("KeyError when getting refget accession: %s", ac) + # A missing accession is a normal backend coverage outcome. Avoid + # formatting and emitting a traceback for every lookup miss. + _logger.debug("Unable to derive refget accession for %s", ac) else: + # A sequence can only have 1 refget accession because it is based on + # content digest, so taking [0] is sufficient. Assert that rather than + # assuming it: translate_sequence_identifier filters on the ``ga4gh:`` + # namespace, not on ``SQ.``, and returns a set-derived list whose order + # is arbitrary, so silently taking [0] of a longer list would pick + # unpredictably between genuinely different accessions. + if len(aliases) > 1: + msg = ( + f"Expected at most one ga4gh alias for {ac}, got {sorted(aliases)}" + ) + raise ValueError(msg) if aliases: refget_accession = aliases[0].split("ga4gh:")[-1] return refget_accession - def validate_ref_seq( - self, - sequence_id: str, - start_pos: int, - end_pos: int, - ref: str, - require_validation: bool = True, - ) -> None: - """Determine whether or not the expected reference sequence matches the actual - reference sequence. Returns ``None``, but invalid results are logged at level - WARN by default. If ``require_validation`` is ``True``, then invalid data will - cause a ``DataProxyValidationError`` to be raised. + @staticmethod + def extract_sequence_type(alias: str) -> str | None: + """Provide a convenient way to extract the sequence type from an accession by matching its prefix to a known set of prefixes. + + Args: + alias (str): The accession string. + + Returns: + str or None: The sequence type associated with the accession string, or None if no matching prefix is found. - :param sequence_id: Sequence ID to use - :param start_pos: Start pos (inter-residue) on the sequence_id - :param end_pos: End pos (inter-residue) on the sequence_id - :param ref: The expected reference sequence on the sequence_id given the - start_pos and end_pos - :param require_validation: If ``True`` and if validation checks fail, a - ``DataProxyValidationError`` will be raised. Error message will always be - logged. - :raises DataProxyValidationError: If excepted reference sequence does not match - the actual reference sequence and ``require_validation`` is ``True``. """ - correct_ref = self.get_sequence(sequence_id, start_pos, end_pos) + prefix_dict = { + "refseq:NM_": "c", + "refseq:NC_012920": "m", + "refseq:NG_": "g", + "refseq:NC_00": "g", + "refseq:NW_": "g", + "refseq:NT_": "g", + "refseq:NR_": "n", + "refseq:NP_": "p", + "refseq:XM_": "c", + "refseq:XR_": "n", + "refseq:XP_": "p", + "GRCh": "g", + } - if correct_ref != ref: - err_msg = f"Reference mismatch at {sequence_id} position {start_pos}-{end_pos} (input gave '{ref}' but correct ref is '{correct_ref}')" - _logger.warning(err_msg) + for prefix, seq_type in prefix_dict.items(): + if alias.startswith(prefix): + return seq_type + return None - if require_validation: - raise DataProxyValidationError(err_msg) +class DataProxy(SequenceDataProxy, AliasDataProxy): + """Combined abstract interface for sequence data access and alias mapping. -class _SeqRepoDataProxyBase(_DataProxy): - # wraps seqreqpo classes in order to provide translation to/from - # `ga4gh` identifiers. + The proxy MUST support the use of GA4GH sequence identifiers (i.e., + ``ga4gh:SQ...``) as keys, and return these identifiers among the + aliases for a sequence. These identifiers may be supported + natively by the data source or synthesized by the proxy from the + data source. + """ - @functools.lru_cache def get_metadata(self, identifier: str) -> dict: - md = self._get_metadata(identifier) - md["aliases"] = list(a for a in md["aliases"]) # noqa: C400 - return md + """Return sequence length and aliases for the given identifier. - @functools.lru_cache - def get_sequence( - self, identifier: str, start: int | None = None, end: int | None = None - ) -> str: - return self._get_sequence(identifier, start=start, end=end) + Composes :meth:`get_sequence_length` and :meth:`get_aliases`. Backends + with a cheaper native path (e.g. a single REST call that returns both) + should override this method. - @abstractmethod - def _get_metadata(self, identifier: str) -> dict: # pragma: no cover - pass + If the given sequence does not exist, KeyError is raised. - @abstractmethod - def _get_sequence( - self, identifier: str, start: int | None = None, end: int | None = None - ) -> str: # pragma: no cover - pass + >> dp.get_metadata("NM_000551.3") + {'length': 4560, + 'aliases': ['MD5:215137b1973c1a5afcf86be7d999574a', + 'RefSeq:NM_000551.3', + 'ga4gh:SQ.v_QTc1p-MUYdgrRv4LMT6ByXIOsdw3C_', + ...]} + """ + return { + "length": self.get_sequence_length(identifier), + "aliases": self.get_aliases(identifier), + } + def _sequence_label(self, sequence_id: str) -> str: + """Name the sequence as given, plus the refget accession it resolves to""" + refget_accession = self.derive_refget_accession(sequence_id) + if not refget_accession or sequence_id == f"ga4gh:{refget_accession}": + return sequence_id + return f"{sequence_id} (ga4gh:{refget_accession})" -class SeqRepoDataProxy(_SeqRepoDataProxyBase): - """DataProxy based on a local instance of SeqRepo""" - def __init__(self, sr) -> None: # noqa: ANN001 - """Initialize DataProxy instance. +def require_refget_accession(data_proxy: AliasDataProxy, ac: str) -> str: + """Derive the refget accession for ``ac``, raising if the backend has none. - :param sr: SeqRepo instance - """ - super().__init__() - self.sr = sr + For translators that have already recognized their input format: an + accession the backend cannot resolve is a failure of that input, not a + sign that the input is in some other format. - def _get_sequence( - self, identifier: str, start: int | None = None, end: int | None = None - ) -> str: - # fetch raises KeyError if not found - return self.sr.fetch_uri(coerce_namespace(identifier), start, end) - - def _get_metadata(self, identifier: str) -> dict: - ns, a = coerce_namespace(identifier).split(":", 2) - r = list(self.sr.aliases.find_aliases(namespace=ns, alias=a)) - if len(r) == 0: - raise KeyError(identifier) - seq_id = r[0]["seq_id"] - seqinfo = self.sr.sequences.fetch_seqinfo(seq_id) - aliases = self.sr.aliases.find_aliases(seq_id=seq_id) - return { - "length": seqinfo["len"], - "alphabet": seqinfo["alpha"], - "added": _isoformat(seqinfo["added"]), - "aliases": [f"{a['namespace']}:{a['alias']}" for a in aliases], - } + :raises UnresolvableAccessionError: if ``ac`` does not resolve to a refget + accession + """ + refget_accession = data_proxy.derive_refget_accession(ac) + if not refget_accession: + raise UnresolvableAccessionError(ac) + return refget_accession -class SeqRepoRESTDataProxy(_SeqRepoDataProxyBase): - """DataProxy based on a REST instance of SeqRepo, as provided by seqrepo-rest-services""" +class CompositeDataProxy(DataProxy): + """Composes a SequenceDataProxy and AliasDataProxy into a full DataProxy. - rest_version = "1" + For sequence methods: tries the sequence backend first; on KeyError, + translates the identifier to a GA4GH digest via the alias backend and retries. + For alias methods: delegates directly to the alias backend. + """ - def __init__(self, base_url: str, disable_healthcheck: bool = False) -> None: - """Initialize REST-based dataproxy instance. + def __init__( + self, sequence_proxy: SequenceDataProxy, alias_proxy: AliasDataProxy + ) -> None: + """Initialize CompositeDataProxy. - :param base_url: root URL to server - :param disable_healthcheck: Whether healthcheck should be disabled + :param sequence_proxy: Backend for sequence retrieval + :param alias_proxy: Backend for alias / identifier resolution """ super().__init__() - self.base_url = f"{base_url}/{self.rest_version}/" - if not disable_healthcheck: - ping_url = self.base_url + "ping" - ping_resp = requests.get(ping_url) # noqa: S113 - ping_resp.raise_for_status() + self._sequence_proxy = sequence_proxy + self._alias_proxy = alias_proxy + + @functools.lru_cache(maxsize=DATAPROXY_CACHE_SIZE) + def _resolve_ga4gh_identifier(self, identifier: str) -> str: + """Translate identifier to a GA4GH SQ digest via the alias backend. + + :param identifier: Sequence identifier in any namespace + :return: GA4GH identifier (``ga4gh:SQ.…``) + :raises KeyError: if no GA4GH alias is found + """ + refget_accession = self._alias_proxy.derive_refget_accession(identifier) + if refget_accession is None: + raise KeyError(identifier) + return f"ga4gh:{refget_accession}" - def _get_sequence( + def get_sequence( self, identifier: str, start: int | None = None, end: int | None = None ) -> str: - url = self.base_url + f"sequence/{identifier}" - _logger.info("Fetching %s", url) - params = {"start": start, "end": end} - resp = requests.get(url, params=params) # noqa: S113 - if resp.status_code == 404: - raise KeyError(identifier) - resp.raise_for_status() - return resp.text - - def _get_metadata(self, identifier: str) -> dict: - url = self.base_url + f"metadata/{identifier}" - _logger.info("Fetching %s", url) - resp = requests.get(url) # noqa: S113 - if resp.status_code == 404: - raise KeyError(identifier) - resp.raise_for_status() - return resp.json() + """Return the specified sequence or subsequence. + + Tries the sequence backend first; on KeyError, resolves the identifier + to a GA4GH digest via the alias backend and retries. + """ + try: + return self._sequence_proxy.get_sequence(identifier, start, end) + except KeyError: + pass + ga4gh_id = self._resolve_ga4gh_identifier(identifier) + return self._sequence_proxy.get_sequence(ga4gh_id, start, end) + + def get_sequence_length(self, identifier: str) -> int: + """Return the length of the specified sequence. + + Tries the sequence backend first; on KeyError, resolves the identifier + to a GA4GH digest via the alias backend and retries. + """ + try: + return self._sequence_proxy.get_sequence_length(identifier) + except KeyError: + pass + ga4gh_id = self._resolve_ga4gh_identifier(identifier) + return self._sequence_proxy.get_sequence_length(ga4gh_id) + + def get_aliases(self, identifier: str) -> list[str]: + """Return all aliases for the specified sequence, delegating to the alias backend.""" + return self._alias_proxy.get_aliases(identifier) + + def derive_refget_accession(self, ac: str) -> str | None: + """Derive a refget accession through the alias backend's optimized path.""" + return self._alias_proxy.derive_refget_accession(ac) class SequenceProxy(Sequence): @@ -289,16 +499,16 @@ class SequenceProxy(Sequence): """ - def __init__(self, dp: _DataProxy, alias: str) -> None: # noqa: D107 + def __init__(self, dp: SequenceDataProxy, alias: str) -> None: # noqa: D107 self.dp = dp self.alias = alias - self._md = self.dp.get_metadata(self.alias) + self._length = self.dp.get_sequence_length(self.alias) def __str__(self) -> str: # noqa: D105 return self.dp.get_sequence(self.alias) def __len__(self): # noqa: D105 ANN204 - return self._md["length"] + return self._length def __reversed__(self): # noqa: D105 ANN204 msg = "Reversed iteration of a SequenceProxy is not implemented" @@ -315,39 +525,22 @@ def __getitem__(self, key) -> str: # noqa: ANN001 return self.dp.get_sequence(self.alias, key.start, key.stop) -def _isoformat(o: datetime.datetime) -> str: - """Convert datetime.datetime to iso formatted timestamp +# Provider registry for extensible create_dataproxy +_provider_registry: dict[str, Callable[..., DataProxy]] = {} - >>> dt = datetime.datetime(2019, 10, 15, 10, 23, 41, 115927) - >>> _isoformat(dt) - '2019-10-15T10:23:41.115927Z' - - :param o: date object to format - :return: ISO8601-formatted string equivalent - :raise TypeError: if object given isn't a Python datetime instance - """ - # stolen from connexion flask_app.py - if not isinstance(o, datetime.datetime): - raise TypeError - if o.tzinfo: - # eg: '2015-09-25T23:14:42.588601+00:00' - return o.isoformat("T") - # No timezone present - assume UTC. - # eg: '2015-09-25T23:14:42.588601Z' - return o.isoformat("T") + "Z" +def register_provider(name: str, factory: Callable[..., DataProxy]) -> None: + """Register a custom DataProxy provider factory. -# Future implementations -# * The RefGetDataProxy is waiting on support for sequence lookup by alias -# class RefGetDataProxy(_DataProxy): -# def __init__(self, base_url): -# super().__init__() -# self.base_url = base_url + :param name: Provider name used in URIs (e.g. "seqrepo" for "seqrepo+http://...") + :param factory: Callable that accepts (parsed_uri, proto, disable_healthcheck) and returns a DataProxy + """ + _provider_registry[name] = factory def create_dataproxy( uri: str | None = None, disable_healthcheck: bool = False -) -> _DataProxy: +) -> DataProxy: """Create a dataproxy from uri or GA4GH_VRS_DATAPROXY_URI :param uri: Dataproxy URI. Currently accepted URI schemes: @@ -356,9 +549,16 @@ def create_dataproxy( * seqrepo+:../relative/path/to/seqrepo/root * seqrepo+http://localhost:5000/seqrepo * seqrepo+https://somewhere:5000/seqrepo + * refget+file:///path/to/refgetstore/root + * refget+http://host[:port]/refgetstore/root + * refget+https://host/refgetstore/root + + Remote refget stores cache their local working set under + ``GA4GH_VRS_REFGET_CACHE_DIR`` when set. If unset, the cache defaults + to ``$XDG_CACHE_HOME/gtars/refget`` or ``~/.cache/gtars/refget``. :param disable_healthcheck: Whether healthcheck should be disabled in REST dataproxy :raise ValueError: if URI doesn't match recognized schemes, e.g. is missing provider - prefix (`"seqrepo+"`) + prefix (``"seqrepo+"``) """ uri = uri or os.environ.get("GA4GH_VRS_DATAPROXY_URI", None) @@ -375,22 +575,24 @@ def create_dataproxy( provider, proto = scheme.split("+") + if provider in _provider_registry: + return _provider_registry[provider]( + parsed_uri, proto, disable_healthcheck=disable_healthcheck + ) + if provider == "seqrepo": - if proto in ("", "file"): - from biocommons.seqrepo import SeqRepo - - sr = SeqRepo(root_dir=parsed_uri.path) - dp = SeqRepoDataProxy(sr) - elif proto in ("http", "https"): - dp = SeqRepoRESTDataProxy( - uri[len(provider) + 1 :], disable_healthcheck=disable_healthcheck - ) - else: - msg = f"SeqRepo URI scheme {parsed_uri.scheme} not implemented" - raise ValueError(msg) + from ga4gh.vrs.dataproxy_seqrepo import _create_seqrepo_dataproxy - else: - msg = f"DataProxy provider {provider} not implemented" - raise ValueError(msg) + return _create_seqrepo_dataproxy( + uri, parsed_uri, proto, disable_healthcheck=disable_healthcheck + ) + + if provider == "refget": + from ga4gh.vrs.dataproxy_refget import _create_refget_dataproxy + + return _create_refget_dataproxy( + parsed_uri, proto, disable_healthcheck=disable_healthcheck + ) - return dp + msg = f"DataProxy provider {provider} not implemented" + raise ValueError(msg) diff --git a/src/ga4gh/vrs/dataproxy_refget.py b/src/ga4gh/vrs/dataproxy_refget.py new file mode 100644 index 00000000..821e3188 --- /dev/null +++ b/src/ga4gh/vrs/dataproxy_refget.py @@ -0,0 +1,364 @@ +"""Refget/gtars-based implementations of the DataProxy interfaces. + +Provides a ``SequenceDataProxy`` backed by a gtars ``RefgetStore`` for +high-performance sequence retrieval by GA4GH digest, and a matching +``AliasDataProxy`` that uses the store's sequence alias index to resolve +cross-namespace identifiers. The module factory composes both into a full +``DataProxy`` from a single URI. Stores may be opened from a local directory +or from a remote static HTTP(S) tree. Remote stores use a local cache rooted +at ``GA4GH_VRS_REFGET_CACHE_DIR`` when set, otherwise under an XDG-style +default cache directory. + +Requires the ``refget`` optional dependency group: ``pip install ga4gh.vrs[refget]`` + +Known divergence from SeqRepo +----------------------------- +For protein sequences containing embedded stop-codon ``*`` characters, a +RefgetStore can yield a different ``ga4gh:SQ.`` digest than SeqRepo for the +*same underlying bytes*. SeqRepo runs the FASTA through +``bioutils.sequences.normalize_sequence()``, which strips ``*`` before +computing the digest; gtars hashes the raw bytes. The RefgetStore digest is +the spec-correct one (VRS does not prescribe ``*`` stripping). In practice +this only affects a small fraction (~0.1%) of Ensembl protein accessions; +NCBI RefSeq protein FASTAs contain no ``*`` and DNA/RNA sequences are never +affected. See ``docs/dataproxy_backends.md`` for details. +""" + +from __future__ import annotations + +import functools +import hashlib +import logging +import os +from pathlib import Path +from typing import TYPE_CHECKING +from urllib.parse import ParseResult, urlunparse + +from ga4gh.vrs.dataproxy import ( + DATAPROXY_CACHE_SIZE, + AliasDataProxy, + CompositeDataProxy, + DataProxy, + SequenceDataProxy, + coerce_accession_namespace, + validate_start_le_end, +) + +if TYPE_CHECKING: + from gtars.refget import RefgetStore + +_logger = logging.getLogger(__name__) + +_GA4GH_SQ_PREFIX = "ga4gh:SQ." +_SQ_PREFIX = "SQ." +_SHA512T24U_PREFIX = "sha512t24u:" +_REFGET_CACHE_ENV_VAR = "GA4GH_VRS_REFGET_CACHE_DIR" + + +def _extract_digest(identifier: str) -> str | None: + """Strip GA4GH SQ prefix, returning the raw sha512t24u digest. + + Accepts identifiers in ``ga4gh:SQ.``, ``SQ.``, or + ``sha512t24u:`` form. + + :param identifier: Sequence identifier, possibly with GA4GH prefix + :return: Raw digest string, or ``None`` if identifier is not in GA4GH SQ form + """ + if identifier.startswith(_GA4GH_SQ_PREFIX): + return identifier[len(_GA4GH_SQ_PREFIX) :] + if identifier.startswith(_SQ_PREFIX): + return identifier[len(_SQ_PREFIX) :] + if identifier.startswith(_SHA512T24U_PREFIX): + return identifier[len(_SHA512T24U_PREFIX) :] + return None + + +def _slug_remote_url(remote_url: str) -> str: + """Return a short deterministic cache slug for a remote refget URL.""" + return hashlib.sha256(remote_url.encode("utf-8")).hexdigest()[:16] + + +def _resolve_refget_cache_base_dir() -> Path: + """Return the base directory for remote refget caches. + + This applies only to remote ``refget+http://...`` and + ``refget+https://...`` URIs. Resolution order is: + + 1. ``GA4GH_VRS_REFGET_CACHE_DIR``, if set + 2. ``$XDG_CACHE_HOME/gtars/refget``, if ``XDG_CACHE_HOME`` is set + 3. ``~/.cache/gtars/refget`` + + The returned path is a cache *root*. The actual cache directory passed to + ``RefgetStore.open_remote()`` is a deterministic per-remote subdirectory + under this root; see :func:`_resolve_remote_cache_dir`. + """ + cache_root = os.environ.get(_REFGET_CACHE_ENV_VAR) + if cache_root: + return Path(cache_root).expanduser() + + xdg_cache_home = os.environ.get("XDG_CACHE_HOME") + if xdg_cache_home: + return Path(xdg_cache_home).expanduser() / "gtars" / "refget" + + return Path.home() / ".cache" / "gtars" / "refget" + + +def _resolve_remote_cache_dir(remote_url: str) -> Path: + """Return the per-remote cache directory for a remote refget URL.""" + return _resolve_refget_cache_base_dir() / _slug_remote_url(remote_url) + + +class RefgetSequenceDataProxy(SequenceDataProxy): + """SequenceDataProxy backed by a gtars RefgetStore. + + Only accepts ``ga4gh:SQ.`` or ``SQ.`` identifiers. + Non-GA4GH identifiers raise KeyError — use CompositeDataProxy with an + AliasDataProxy to resolve namespace identifiers first. + """ + + def __init__(self, store: RefgetStore) -> None: + """Initialize RefgetSequenceDataProxy. + + :param store: A gtars RefgetStore instance (local, on-disk, or in-memory) + """ + super().__init__() + self.store = store + # Track which digests we've already promoted from Stub to Full so + # subsequent substring fetches skip the per-digest file read. + self._loaded_digests: set[str] = set() + + def _ensure_sequence_loaded(self, digest: str) -> None: + """Lazily materialize sequence bytes for ``digest`` in the store. + + ``open_local()`` reads ``sequences.rgsi`` and inserts a Stub record + for every sequence in the store, so metadata lookups work without + any preload. But byte-level access (``get_substring``) needs the + per-digest payload read in via ``load_sequence(digest)``. We call + that on first access for each digest and memoize the result — no + sequence bytes are loaded until a caller actually asks for them. + """ + if digest in self._loaded_digests: + return + self.store.load_sequence(digest) # type: ignore[attr-defined] + self._loaded_digests.add(digest) + + def get_sequence( + self, identifier: str, start: int | None = None, end: int | None = None + ) -> str: + """Return the specified sequence or subsequence from the refget store. + + :param identifier: GA4GH sequence identifier (``ga4gh:SQ.…`` or ``SQ.…``) + :param start: Optional start position (interbase, 0-based) + :param end: Optional end position (interbase) + :return: Sequence string + :raises KeyError: if identifier is not a GA4GH SQ form or not found in store + :raises DataProxyValidationError: if ``start`` is after ``end`` + """ + # A slice would otherwise return "" for it + validate_start_le_end(identifier, start, end) + digest = _extract_digest(identifier) + if digest is None: + raise KeyError(identifier) + + self._ensure_sequence_loaded(digest) + + # Full sequence shortcut — no bounds needed + if start is None and end is None: + record = self.store.get_sequence(digest) + if record is None: + raise KeyError(identifier) + result = record.decode() + if result is None: + raise KeyError(identifier) + return result + + # Resolve None bounds and clamp to [0, length] for gtars (requires usize). + # This matches Python slice semantics where negative/out-of-range indices + # are silently clamped. + if start is None or end is None or start < 0: + metadata = self.store.get_sequence_metadata(digest) + if metadata is None: + raise KeyError(identifier) + length: int = metadata.length + s = max(start or 0, 0) + e = min(end, length) if end is not None else length + else: + s = start + e = end + + if s >= e: + return "" + + result = self.store.get_substring(digest, s, e) + if result is None: + raise KeyError(identifier) + return result + + @functools.lru_cache(maxsize=DATAPROXY_CACHE_SIZE) # noqa: B019 + def get_sequence_length(self, identifier: str) -> int: + """Return the length of the specified sequence from the refget store. + + :param identifier: GA4GH sequence identifier (``ga4gh:SQ.…`` or ``SQ.…``) + :return: Sequence length + :raises KeyError: if identifier is not a GA4GH SQ form or not found in store + """ + digest = _extract_digest(identifier) + if digest is None: + raise KeyError(identifier) + metadata = self.store.get_sequence_metadata(digest) + if metadata is None: + raise KeyError(identifier) + return metadata.length + + +class RefgetAliasDataProxy(AliasDataProxy): + """AliasDataProxy backed by a gtars RefgetStore. + + Resolves any-namespace identifiers to sha512t24u digests via the store's + sequence alias index, and enumerates all known aliases for a digest via + the store's reverse-lookup API. The ``sha512t24u:`` and ``ga4gh:SQ.`` + aliases are synthesized from the digest rather than stored on disk. + """ + + def __init__(self, store: RefgetStore) -> None: + """Initialize RefgetAliasDataProxy. + + :param store: A gtars RefgetStore instance. The store's alias + sidecars are pulled eagerly for remote stores so subsequent + lookups do not need extra network / disk I/O for the alias index. + """ + super().__init__() + self.store = store + # pull_aliases is a no-op for local stores but is required for remote + # stores so namespace alias lookups work from the local cache. + try: + store.pull_aliases() # type: ignore[attr-defined] + except Exception as exc: + _logger.debug("pull_aliases failed (ignored): %s", exc) + + @functools.lru_cache(maxsize=DATAPROXY_CACHE_SIZE) # noqa: B019 + def _resolve_to_digest(self, identifier: str) -> str | None: + """Resolve an identifier in any supported form to a sha512t24u digest. + + Supported forms: + - ``ga4gh:SQ.``, ``SQ.``, ``sha512t24u:`` + (handled via :func:`_extract_digest`) + - ``namespace:alias`` (looked up via ``get_sequence_by_alias``) + - Bare accession like ``NC_000001.11`` or ``LRG_1t1`` (coerced to + ``refseq:`` / ``lrg:`` / ... via :func:`coerce_accession_namespace`) + + Returns None if the identifier cannot be resolved. + """ + digest = _extract_digest(identifier) + if digest is not None: + # Digest-prefix branch: confirm the sequence actually exists. + md = self.store.get_sequence_metadata(digest) + return digest if md is not None else None + + # Namespaced or bare accession: coerce and look up. + normalized = identifier + if ":" not in normalized[1:]: + try: + normalized = coerce_accession_namespace(normalized) + except ValueError: + return None + + ns, _, alias = normalized.partition(":") + if not alias: + return None + metadata = self.store.get_sequence_metadata_by_alias(ns, alias) + if metadata is None: + return None + return metadata.sha512t24u + + def derive_refget_accession(self, ac: str) -> str | None: + """Resolve an accession directly without scanning its reverse aliases.""" + if ac is None: + return None + digest = self._resolve_to_digest(ac) + return f"SQ.{digest}" if digest is not None else None + + @functools.lru_cache(maxsize=DATAPROXY_CACHE_SIZE) # noqa: B019 + def get_aliases(self, identifier: str) -> list[str]: + """Return all known aliases for the specified sequence. + + Aliases are emitted as ``namespace:alias`` strings. In addition to + whatever the store's reverse-lookup index returns, two synthesized + entries are always appended when the sequence exists: + + - ``sha512t24u:`` + - ``ga4gh:SQ.`` + + :raises KeyError: if the identifier cannot be resolved in the store. + """ + digest = self._resolve_to_digest(identifier) + if digest is None: + raise KeyError(identifier) + + aliases: list[str] = [] + try: + rev = self.store.get_aliases_for_sequence(digest) or [] + except Exception as exc: + _logger.debug( + "get_aliases_for_sequence(%s) raised, treating as empty: %s", + digest, + exc, + ) + rev = [] + for entry in rev: + if isinstance(entry, tuple) and len(entry) == 2: + ns, alias = entry + aliases.append(f"{ns}:{alias}") + else: + aliases.append(str(entry)) + + aliases.append(f"sha512t24u:{digest}") + aliases.append(f"ga4gh:SQ.{digest}") + return aliases + + +def _create_refget_dataproxy( + parsed_uri: ParseResult, + proto: str, + *, + disable_healthcheck: bool = False, # noqa: ARG001 +) -> DataProxy: + """Create a refget-backed DataProxy from a parsed URI. + + Composes a ``RefgetSequenceDataProxy`` and ``RefgetAliasDataProxy`` over + a single shared ``RefgetStore`` instance, wrapped in a + ``CompositeDataProxy`` so the result satisfies the full ``DataProxy`` + interface. ``refget+file://`` URIs open a local store directly. + ``refget+http://`` and ``refget+https://`` URIs open a remote store and + cache its index, aliases, and fetched sequence payloads locally under + ``GA4GH_VRS_REFGET_CACHE_DIR`` or the default XDG-style cache root. + + :param parsed_uri: Parsed URI components + :param proto: Protocol portion of the scheme (e.g. "file", "http", "") + :param disable_healthcheck: Unused, accepted for interface consistency + :return: A ``CompositeDataProxy`` wrapping the sequence + alias proxies + :raises ValueError: if the protocol is not supported + """ + if proto in ("", "file"): + from gtars.refget import RefgetStore + + store = RefgetStore.open_local(parsed_uri.path) + elif proto in ("http", "https"): + from gtars.refget import RefgetStore + + remote_url = urlunparse(parsed_uri._replace(scheme=proto)) + cache_dir = _resolve_remote_cache_dir(remote_url) + cache_dir.mkdir(parents=True, exist_ok=True) + store = RefgetStore.open_remote(str(cache_dir), remote_url) + # open_remote defers the sequence index, and alias lookups do not load + # it, so they would find nothing. Metadata lookups load it; the digest + # is irrelevant. + store.get_sequence_metadata("") + else: + msg = f"Refget URI protocol {proto!r} not supported" + raise ValueError(msg) + + store.set_quiet(True) + sequence_proxy = RefgetSequenceDataProxy(store) + alias_proxy = RefgetAliasDataProxy(store) + return CompositeDataProxy(sequence_proxy, alias_proxy) diff --git a/src/ga4gh/vrs/dataproxy_seqrepo.py b/src/ga4gh/vrs/dataproxy_seqrepo.py new file mode 100644 index 00000000..9605f36a --- /dev/null +++ b/src/ga4gh/vrs/dataproxy_seqrepo.py @@ -0,0 +1,186 @@ +"""SeqRepo-based implementations of the DataProxy interface. + +Provides concrete DataProxy implementations backed by a local SeqRepo +instance or a SeqRepo REST service. +""" + +from __future__ import annotations + +import datetime +import functools +import logging +from urllib.parse import ParseResult + +import requests + +from ga4gh.vrs.dataproxy import ( + DATAPROXY_CACHE_SIZE, + REST_SEQUENCE_CACHE_SIZE, + DataProxy, + coerce_accession_namespace, + validate_start_le_end, +) + +_logger = logging.getLogger(__name__) + + +class SeqRepoDataProxy(DataProxy): + """DataProxy based on a local instance of SeqRepo.""" + + def __init__(self, sr) -> None: # noqa: ANN001 + """Initialize DataProxy instance. + + :param sr: SeqRepo instance + """ + super().__init__() + self.sr = sr + + def get_sequence( + self, identifier: str, start: int | None = None, end: int | None = None + ) -> str: + """Return the specified sequence or subsequence from local SeqRepo.""" + validate_start_le_end(identifier, start, end) + return self.sr.fetch_uri(coerce_accession_namespace(identifier), start, end) + + @functools.lru_cache(maxsize=DATAPROXY_CACHE_SIZE) # noqa: B019 + def get_sequence_length(self, identifier: str) -> int: + """Return the length of the specified sequence from local SeqRepo.""" + ns, a = coerce_accession_namespace(identifier).split(":", 2) + r = list(self.sr.aliases.find_aliases(namespace=ns, alias=a)) + if len(r) == 0: + raise KeyError(identifier) + seq_id = r[0]["seq_id"] + seqinfo = self.sr.sequences.fetch_seqinfo(seq_id) + return seqinfo["len"] + + @functools.lru_cache(maxsize=DATAPROXY_CACHE_SIZE) # noqa: B019 + def get_aliases(self, identifier: str) -> list[str]: + """Return all aliases for the specified sequence from local SeqRepo.""" + ns, a = coerce_accession_namespace(identifier).split(":", 2) + r = list(self.sr.aliases.find_aliases(namespace=ns, alias=a)) + if len(r) == 0: + raise KeyError(identifier) + seq_id = r[0]["seq_id"] + aliases = self.sr.aliases.find_aliases(seq_id=seq_id) + return [f"{a['namespace']}:{a['alias']}" for a in aliases] + + +class SeqRepoRESTDataProxy(DataProxy): + """DataProxy based on a REST instance of SeqRepo, as provided by seqrepo-rest-services.""" + + rest_version = "1" + + def __init__(self, base_url: str, disable_healthcheck: bool = False) -> None: + """Initialize REST-based dataproxy instance. + + :param base_url: root URL to server + :param disable_healthcheck: Whether healthcheck should be disabled + """ + super().__init__() + self.base_url = f"{base_url}/{self.rest_version}/" + if not disable_healthcheck: + ping_url = self.base_url + "ping" + ping_resp = requests.get(ping_url) # noqa: S113 + ping_resp.raise_for_status() + + @functools.lru_cache(maxsize=REST_SEQUENCE_CACHE_SIZE) # noqa: B019 + def get_sequence( + self, identifier: str, start: int | None = None, end: int | None = None + ) -> str: + """Return the specified sequence or subsequence from REST SeqRepo.""" + validate_start_le_end(identifier, start, end) + url = self.base_url + f"sequence/{identifier}" + _logger.info("Fetching %s", url) + params = {"start": start, "end": end} + resp = requests.get(url, params=params) # noqa: S113 + if resp.status_code == 404: + raise KeyError(identifier) + resp.raise_for_status() + return resp.text + + @functools.lru_cache(maxsize=DATAPROXY_CACHE_SIZE) # noqa: B019 + def _get_metadata(self, identifier: str) -> dict: + """Fetch metadata from the REST service. + + Internal helper that returns the full metadata dict from the REST API. + Used by both get_sequence_length and get_aliases to avoid duplicate requests. + """ + url = self.base_url + f"metadata/{identifier}" + _logger.info("Fetching %s", url) + resp = requests.get(url) # noqa: S113 + if resp.status_code == 404: + raise KeyError(identifier) + resp.raise_for_status() + return resp.json() + + def get_sequence_length(self, identifier: str) -> int: + """Return the length of the specified sequence from REST SeqRepo.""" + md = self._get_metadata(identifier) + return md["length"] + + def get_aliases(self, identifier: str) -> list[str]: + """Return all aliases for the specified sequence from REST SeqRepo.""" + md = self._get_metadata(identifier) + return list(md["aliases"]) + + def get_metadata(self, identifier: str) -> dict: + """Return length and aliases in a single REST round-trip. + + Overrides the default :meth:`DataProxy.get_metadata` to avoid the two + sequential calls that would otherwise each hit ``_get_metadata``. + """ + md = self._get_metadata(identifier) + return {"length": md["length"], "aliases": list(md["aliases"])} + + +def _isoformat(o: datetime.datetime) -> str: + """Convert datetime.datetime to iso formatted timestamp. + + >>> dt = datetime.datetime(2019, 10, 15, 10, 23, 41, 115927) + >>> _isoformat(dt) + '2019-10-15T10:23:41.115927Z' + + :param o: date object to format + :return: ISO8601-formatted string equivalent + :raise TypeError: if object given isn't a Python datetime instance + """ + # stolen from connexion flask_app.py + if not isinstance(o, datetime.datetime): + raise TypeError + if o.tzinfo: + # eg: '2015-09-25T23:14:42.588601+00:00' + return o.isoformat("T") + # No timezone present - assume UTC. + # eg: '2015-09-25T23:14:42.588601Z' + return o.isoformat("T") + "Z" + + +def _create_seqrepo_dataproxy( + uri: str, + parsed_uri: ParseResult, + proto: str, + *, + disable_healthcheck: bool = False, +) -> DataProxy: + """Create a SeqRepo-backed DataProxy from a parsed URI. + + :param uri: Original URI string + :param parsed_uri: Parsed URI components + :param proto: Protocol portion of the scheme (e.g. "file", "http", "https", "") + :param disable_healthcheck: Whether healthcheck should be disabled in REST dataproxy + :return: A SeqRepoDataProxy or SeqRepoRESTDataProxy instance + """ + if proto in ("", "file"): + from biocommons.seqrepo import SeqRepo + + sr = SeqRepo(root_dir=parsed_uri.path) + return SeqRepoDataProxy(sr) + + if proto in ("http", "https"): + provider = parsed_uri.scheme.split("+")[0] + return SeqRepoRESTDataProxy( + uri[len(provider) + 1 :], disable_healthcheck=disable_healthcheck + ) + + msg = f"SeqRepo URI scheme {parsed_uri.scheme} not implemented" + raise ValueError(msg) diff --git a/src/ga4gh/vrs/extras/annotator/cli.py b/src/ga4gh/vrs/extras/annotator/cli.py index 4ebdb206..34abb186 100644 --- a/src/ga4gh/vrs/extras/annotator/cli.py +++ b/src/ga4gh/vrs/extras/annotator/cli.py @@ -188,7 +188,8 @@ def _annotate_vcf_cli( option or the environment variable `GA4GH_VRS_DATAPROXY_URI` to define its location (the former will take priority over the latter when both are set). This option falls back upon the VRS-Python library default `seqrepo+http://localhost:5000/seqrepo` - value if it is not otherwise declared. + value if it is not otherwise declared. Remote refget stores use + `GA4GH_VRS_REFGET_CACHE_DIR` as the local cache root when set. Currently accepted URI schemes: @@ -197,6 +198,9 @@ def _annotate_vcf_cli( * seqrepo+https://somewhere:5000/seqrepo * seqrepo+file:///path/to/seqrepo/root * seqrepo+:../relative/path/to/seqrepo/root + * refget+file:///path/to/refgetstore/root + * refget+http://host[:port]/refgetstore/root + * refget+https://host/refgetstore/root """ # noqa: D301 data_proxy = create_dataproxy(dataproxy_uri) annotator = VcfAnnotator(data_proxy) diff --git a/src/ga4gh/vrs/extras/annotator/vcf.py b/src/ga4gh/vrs/extras/annotator/vcf.py index 3dd70dcf..928a1331 100644 --- a/src/ga4gh/vrs/extras/annotator/vcf.py +++ b/src/ga4gh/vrs/extras/annotator/vcf.py @@ -14,7 +14,7 @@ use_ga4gh_compute_identifier_when, ) from ga4gh.vrs import VRS_VERSION, VrsType, __version__ -from ga4gh.vrs.dataproxy import _DataProxy +from ga4gh.vrs.dataproxy import DataProxy, UnresolvableAccessionError from ga4gh.vrs.extras.translator import AlleleTranslator from ga4gh.vrs.models import Allele, Range @@ -111,7 +111,7 @@ class AbstractVcfAnnotator(abc.ABC): collect_alleles: bool = False - def __init__(self, data_proxy: _DataProxy, **kwargs) -> None: # noqa: ARG002 + def __init__(self, data_proxy: DataProxy, **kwargs) -> None: # noqa: ARG002 """Initialize the VCFAnnotator class. :param data_proxy: GA4GH sequence dataproxy instance. @@ -389,6 +389,11 @@ def _get_vrs_object( assembly_name=assembly, require_validation=require_validation, ) + except UnresolvableAccessionError as e: + # A contig the backend lacks: leave this record's fields empty + # rather than abort the file. + vrs_obj = None + _logger.debug("%s; skipping %s", e, vcf_coords) except Exception: vrs_obj = None _logger.exception( diff --git a/src/ga4gh/vrs/extras/translator.py b/src/ga4gh/vrs/extras/translator.py index c903d4d2..b354917c 100644 --- a/src/ga4gh/vrs/extras/translator.py +++ b/src/ga4gh/vrs/extras/translator.py @@ -13,7 +13,12 @@ from ga4gh.core import ga4gh_identify from ga4gh.vrs import models, normalize -from ga4gh.vrs.dataproxy import SequenceProxy, _DataProxy +from ga4gh.vrs.dataproxy import ( + DataProxy, + DataProxyValidationError, + SequenceProxy, + require_refget_accession, +) from ga4gh.vrs.extras.decorators import lazy_property from ga4gh.vrs.normalize import denormalize_reference_length_expression from ga4gh.vrs.utils.hgvs_tools import HgvsTools @@ -66,7 +71,7 @@ class _Translator(ABC): # noqa: B024 def __init__( self, - data_proxy: _DataProxy, + data_proxy: DataProxy, default_assembly_name: str = "GRCh38", identify: bool = True, rle_seq_limit: int | None = 50, @@ -122,14 +127,32 @@ def translate_from( raise ValueError(msg) return o - for t in self.from_translators.values(): - o = t(var, **kwargs) - if o: + # A format that recognizes `var` but fails to translate it does not end + # the guess: an earlier pattern can match spuriously, and a later format + # may still succeed. Its error is kept and reported if none does. + errors: list[tuple[str, Exception]] = [] + for name, t in self.from_translators.items(): + try: + o = t(var, **kwargs) + except (KeyError, ValueError, DataProxyValidationError) as e: + errors.append((name, e)) + continue + if o is not None: return o - formats = list(self.from_translators.keys()) - msg = f"Unable to parse data as {', '.join(formats)}" - raise ValueError(msg) + if not errors: + formats = list(self.from_translators.keys()) + msg = f"Unable to parse data as {', '.join(formats)}" + raise ValueError(msg) + + # Recognized but untranslatable: report why, rather than "cannot parse". + _, first = errors[0] + if len(errors) > 1 and hasattr(first, "add_note"): # Python 3.11+ + first.add_note( + "Also recognized as: " + + "; ".join(f"{n} ({type(e).__name__}: {e})" for n, e in errors[1:]) + ) + raise first def translate_to(self, vo: models._VariationBase, fmt: str, **kwargs) -> list[str]: """Translate vrs object `vo` to named format `fmt` @@ -172,7 +195,7 @@ class AlleleTranslator(_Translator): def __init__( self, - data_proxy: _DataProxy, + data_proxy: DataProxy, default_assembly_name: str = "GRCh38", identify: bool = True, ) -> None: @@ -196,6 +219,8 @@ def _create_allele(self, values: dict, **kwargs) -> models.Allele: Args: values (dict): The values to use for creating the allele object. + 'sequence_id' (str): The sequence identifier from the input + expression, used to validate `start` and `end`. 'refget_accession' (str): The accession ID of the reference genome. 'start' (int): The start position of the allele. 'end' (int): The end position of the allele. @@ -206,6 +231,9 @@ def _create_allele(self, values: dict, **kwargs) -> models.Allele: models.Allele: The created allele object. """ + self.data_proxy.validate_location_bounds( + values["sequence_id"], values["start"], values["end"] + ) seq_ref = models.SequenceReference(refgetAccession=values["refget_accession"]) location = models.SequenceLocation( sequenceReference=seq_ref, start=values["start"], end=values["end"] @@ -260,9 +288,7 @@ def _from_gnomad(self, gnomad_expr: str, **kwargs) -> models.Allele | None: g = m.groupdict() assembly_name = kwargs.get("assembly_name", self.default_assembly_name) sequence = assembly_name + ":" + g["chr"] - refget_accession = self.data_proxy.derive_refget_accession(sequence) - if not refget_accession: - return None + refget_accession = require_refget_accession(self.data_proxy, sequence) start = int(g["pos"]) - 1 ref = g["ref"].upper() @@ -280,6 +306,7 @@ def _from_gnomad(self, gnomad_expr: str, **kwargs) -> models.Allele | None: ) values = { + "sequence_id": sequence, "refget_accession": refget_accession, "start": start, "end": end, @@ -345,9 +372,7 @@ def _from_spdi(self, spdi_expr: str, **kwargs) -> models.Allele | None: return None g = m.groupdict() - refget_accession = self.data_proxy.derive_refget_accession(g["ac"]) - if not refget_accession: - return None + refget_accession = require_refget_accession(self.data_proxy, g["ac"]) start = int(g["pos"]) try: @@ -358,6 +383,7 @@ def _from_spdi(self, spdi_expr: str, **kwargs) -> models.Allele | None: ins_seq = g["ins_seq"] values = { + "sequence_id": g["ac"], "refget_accession": refget_accession, "start": start, "end": end, @@ -475,7 +501,7 @@ class CnvTranslator(_Translator): def __init__( self, - data_proxy: _DataProxy, + data_proxy: DataProxy, default_assembly_name: str = "GRCh38", identify: bool = True, ) -> None: @@ -510,11 +536,10 @@ def _from_hgvs( msg = "Intronic HGVS variants are not supported" raise ValueError(msg) - refget_accession = self.data_proxy.derive_refget_accession(sv.ac) - if not refget_accession: - return None + refget_accession = require_refget_accession(self.data_proxy, sv.ac) location = self.hgvs_tools.build_cnv_location(sv, refget_accession) + self.data_proxy.validate_location_bounds(sv.ac, location.start, location.end) copies = kwargs.get("copies") if copies is not None: diff --git a/src/ga4gh/vrs/normalize.py b/src/ga4gh/vrs/normalize.py index fad73d87..d3e87e63 100644 --- a/src/ga4gh/vrs/normalize.py +++ b/src/ga4gh/vrs/normalize.py @@ -15,7 +15,7 @@ from ga4gh.core import ga4gh_digest, is_pydantic_instance, pydantic_copy from ga4gh.vrs import models -from ga4gh.vrs.dataproxy import SequenceProxy, _DataProxy +from ga4gh.vrs.dataproxy import DataProxy, SequenceProxy _logger = logging.getLogger(__name__) @@ -85,7 +85,7 @@ def _get_new_allele_location_pos( def _normalize_allele( - input_allele: models.Allele, data_proxy: _DataProxy, rle_seq_limit: int = 50 + input_allele: models.Allele, data_proxy: DataProxy, rle_seq_limit: int = 50 ): """Normalize Allele using "fully-justified" normalization adapted from NCBI's VOCA. Fully-justified normalization expands such ambiguous representation over the @@ -111,6 +111,8 @@ def _normalize_allele( of the `sequence`. To exclude `sequence` from the response, set to 0. For no limit, set to `None`. + :raises DataProxyValidationError: If the allele location is out of bounds on its + sequence """ # Algorithm applies to LiteralSequenceExpression alleles only; other states are returned unchanged if not isinstance(input_allele.state, models.LiteralSequenceExpression): @@ -130,6 +132,14 @@ def _normalize_allele( # 0: Get reference sequence and interval ref_seq = SequenceProxy(data_proxy, alias) + + # Reject locations that do not exist on the sequence before anything is fetched, + # since out-of-range fetches may be silently truncated by the sequence backend. + # Done before the early returns below, which skip definite ranges. + data_proxy.validate_location_bounds( + alias, input_allele.location.start, input_allele.location.end + ) + start = _get_allele_location_pos(input_allele, use_start=True) if start is None: return input_allele @@ -352,7 +362,7 @@ def _is_valid_cycle(template_start, template: str, target) -> bool: def _normalize_cis_phased_block( o, - data_proxy: _DataProxy | None = None, # noqa: ARG001 + data_proxy: DataProxy | None = None, # noqa: ARG001 ): o.members = sorted(o.members, key=ga4gh_digest) return o @@ -364,7 +374,7 @@ def _normalize_cis_phased_block( } -def normalize(vo, data_proxy: _DataProxy | None = None, **kwargs): +def normalize(vo, data_proxy: DataProxy | None = None, **kwargs): """Normalize given vrs object, regardless of type :param vo: diff --git a/src/ga4gh/vrs/utils/hgvs_tools.py b/src/ga4gh/vrs/utils/hgvs_tools.py index 3ba0496d..b169fae6 100644 --- a/src/ga4gh/vrs/utils/hgvs_tools.py +++ b/src/ga4gh/vrs/utils/hgvs_tools.py @@ -15,7 +15,7 @@ from hgvs.sequencevariant import SequenceVariant as HgvsSequenceVariant from ga4gh.vrs import models -from ga4gh.vrs.dataproxy import _DataProxy +from ga4gh.vrs.dataproxy import DataProxy, require_refget_accession _logger = logging.getLogger(__name__) @@ -185,13 +185,13 @@ class HgvsTools: uta_conn: The UTA (Universal Transcript Archive) connection object. normalizer: The HGVS normalizer object. variant_mapper: The HGVS variant mapper object. - data_proxy (_DataProxy): The data proxy object. + data_proxy (DataProxy): The data proxy object. """ hgvs_re = re.compile(r"[^:]+:[cgmnpr]\.") - def __init__(self, data_proxy: _DataProxy | None = None) -> None: + def __init__(self, data_proxy: DataProxy | None = None) -> None: """Initialize object. :param data_proxy: GA4GH data proxy instance @@ -285,6 +285,8 @@ def get_position_and_state( ValueError: If the HGVS variant type is unsupported, or if ``pos`` is an intronic :class:`~hgvs.location.BaseOffsetPosition`. TypeError: If ``pos`` is a :class:`~hgvs.location.BaseOffsetInterval`. + DataProxyValidationError: If an identity or duplication's location is + invalid. """ pos = sv.posedit.pos @@ -308,11 +310,14 @@ def get_position_and_state( if edit_type in ("sub", "del", "delins", "identity"): if edit_type == "identity": + # Checked before fetching, which a backend may silently truncate + self.data_proxy.validate_location_bounds(sv.ac, start, end) state = self.data_proxy.get_sequence(sv.ac, start=start, end=end) else: state = sv.posedit.edit.alt or "" elif edit_type == "dup": + self.data_proxy.validate_location_bounds(sv.ac, start, end) ref = self.data_proxy.get_sequence(sv.ac, start=start, end=end) state = ref + ref @@ -362,9 +367,7 @@ def extract_allele_values(self, hgvs_expr: str) -> dict | None: msg = "Intronic HGVS variants are not supported" raise ValueError(msg) - refget_accession = self.data_proxy.derive_refget_accession(sv.ac) - if not refget_accession: - return None + refget_accession = require_refget_accession(self.data_proxy, sv.ac) # translate coding coordinates to positional coordinates, if necessary if sv.type == "c": @@ -373,6 +376,7 @@ def extract_allele_values(self, hgvs_expr: str) -> dict | None: (start, end, state) = self.get_position_and_state(sv) return { + "sequence_id": sv.ac, "refget_accession": refget_accession, "start": start, "end": end, diff --git a/tests/cassettes/test_normalize_allele.yaml b/tests/cassettes/test_normalize_allele.yaml index a953adce..d447a97f 100644 --- a/tests/cassettes/test_normalize_allele.yaml +++ b/tests/cassettes/test_normalize_allele.yaml @@ -165,6 +165,30 @@ interactions: status: code: 200 message: OK +- request: + body: null + headers: {} + method: GET + uri: http://localhost:5000/seqrepo/1/metadata/ga4gh:SQ.IIB53T8CNeJJdUqzn9V_JnRtQadwWCbl + response: + body: + string: "{\n \"added\": \"2016-08-24T08:19:02Z\",\n \"aliases\": [\n \"Ensembl:19\",\n + \ \"ensembl:19\",\n \"GRCh38:19\",\n \"GRCh38:chr19\",\n \"GRCh38.p1:19\",\n + \ \"GRCh38.p1:chr19\",\n \"GRCh38.p10:19\",\n \"GRCh38.p10:chr19\",\n + \ \"GRCh38.p11:19\",\n \"GRCh38.p11:chr19\",\n \"GRCh38.p12:19\",\n + \ \"GRCh38.p12:chr19\",\n \"GRCh38.p2:19\",\n \"GRCh38.p2:chr19\",\n + \ \"GRCh38.p3:19\",\n \"GRCh38.p3:chr19\",\n \"GRCh38.p4:19\",\n \"GRCh38.p4:chr19\",\n + \ \"GRCh38.p5:19\",\n \"GRCh38.p5:chr19\",\n \"GRCh38.p6:19\",\n \"GRCh38.p6:chr19\",\n + \ \"GRCh38.p7:19\",\n \"GRCh38.p7:chr19\",\n \"GRCh38.p8:19\",\n \"GRCh38.p8:chr19\",\n + \ \"GRCh38.p9:19\",\n \"GRCh38.p9:chr19\",\n \"MD5:b0eba2c7bb5c953d1e06a508b5e487de\",\n + \ \"NCBI:NC_000019.10\",\n \"refseq:NC_000019.10\",\n \"SEGUID:AHxM5/L8jIX08UhBBkKXkiO5rhY\",\n + \ \"SHA1:007c4ce7f2fc8c85f4f148410642979223b9ae16\",\n \"VMC:GS_IIB53T8CNeJJdUqzn9V_JnRtQadwWCbl\",\n + \ \"sha512t24u:IIB53T8CNeJJdUqzn9V_JnRtQadwWCbl\",\n \"ga4gh:SQ.IIB53T8CNeJJdUqzn9V_JnRtQadwWCbl\"\n + \ ],\n \"alphabet\": \"ACGNT\",\n \"length\": 58617616\n}\n" + headers: {} + status: + code: 200 + message: OK - request: body: null headers: {} @@ -273,6 +297,29 @@ interactions: status: code: 200 message: OK +- request: + body: null + headers: {} + method: GET + uri: http://localhost:5000/seqrepo/1/metadata/ga4gh:SQ.Ya6Rs7DHhDeg7YaOSg1EoNi3U_nQ9SvO + response: + body: + string: "{\n \"added\": \"2016-08-27T21:17:00Z\",\n \"aliases\": [\n \"GRCh38:1\",\n + \ \"GRCh38:chr1\",\n \"GRCh38.p1:1\",\n \"GRCh38.p1:chr1\",\n \"GRCh38.p10:1\",\n + \ \"GRCh38.p10:chr1\",\n \"GRCh38.p11:1\",\n \"GRCh38.p11:chr1\",\n + \ \"GRCh38.p12:1\",\n \"GRCh38.p12:chr1\",\n \"GRCh38.p2:1\",\n \"GRCh38.p2:chr1\",\n + \ \"GRCh38.p3:1\",\n \"GRCh38.p3:chr1\",\n \"GRCh38.p4:1\",\n \"GRCh38.p4:chr1\",\n + \ \"GRCh38.p5:1\",\n \"GRCh38.p5:chr1\",\n \"GRCh38.p6:1\",\n \"GRCh38.p6:chr1\",\n + \ \"GRCh38.p7:1\",\n \"GRCh38.p7:chr1\",\n \"GRCh38.p8:1\",\n \"GRCh38.p8:chr1\",\n + \ \"GRCh38.p9:1\",\n \"GRCh38.p9:chr1\",\n \"MD5:6aef897c3d6ff0c78aff06ac189178dd\",\n + \ \"NCBI:NC_000001.11\",\n \"refseq:NC_000001.11\",\n \"SEGUID:FCUd6VJ6uikS/VWLbhGdVmj2rOA\",\n + \ \"SHA1:14251de9527aba2912fd558b6e119d5668f6ace0\",\n \"VMC:GS_Ya6Rs7DHhDeg7YaOSg1EoNi3U_nQ9SvO\",\n + \ \"sha512t24u:Ya6Rs7DHhDeg7YaOSg1EoNi3U_nQ9SvO\",\n \"ga4gh:SQ.Ya6Rs7DHhDeg7YaOSg1EoNi3U_nQ9SvO\"\n + \ ],\n \"alphabet\": \"ACGMNRT\",\n \"length\": 248956422\n}\n" + headers: {} + status: + code: 200 + message: OK - request: body: null headers: {} diff --git a/tests/conftest.py b/tests/conftest.py index 4c843539..cc83f17f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,7 +3,7 @@ import pytest from biocommons.seqrepo import SeqRepo -from ga4gh.vrs.dataproxy import SeqRepoDataProxy, SeqRepoRESTDataProxy +from ga4gh.vrs.dataproxy_seqrepo import SeqRepoDataProxy, SeqRepoRESTDataProxy def remove_request_headers(request): diff --git a/tests/extras/cassettes/test_in_bounds[spdi-insertion-at-end].yaml b/tests/extras/cassettes/test_in_bounds[spdi-insertion-at-end].yaml new file mode 100644 index 00000000..b88723ee --- /dev/null +++ b/tests/extras/cassettes/test_in_bounds[spdi-insertion-at-end].yaml @@ -0,0 +1,58 @@ +interactions: +- request: + body: null + headers: {} + method: GET + uri: http://localhost:5000/seqrepo/1/metadata/refseq:NM_000551.3 + response: + body: + string: "{\n \"added\": \"2016-08-24T05:03:11Z\",\n \"aliases\": [\n \"MD5:215137b1973c1a5afcf86be7d999574a\",\n + \ \"NCBI:NM_000551.3\",\n \"refseq:NM_000551.3\",\n \"SEGUID:T12L0p2X5E8DbnL0+SwI4Wc1S6g\",\n + \ \"SHA1:4f5d8bd29d97e44f036e72f4f92c08e167354ba8\",\n \"VMC:GS_v_QTc1p-MUYdgrRv4LMT6ByXIOsdw3C_\",\n + \ \"sha512t24u:v_QTc1p-MUYdgrRv4LMT6ByXIOsdw3C_\",\n \"ga4gh:SQ.v_QTc1p-MUYdgrRv4LMT6ByXIOsdw3C_\"\n + \ ],\n \"alphabet\": \"ACGT\",\n \"length\": 4560\n}\n" + headers: {} + status: + code: 200 + message: OK +- request: + body: null + headers: {} + method: GET + uri: http://localhost:5000/seqrepo/1/metadata/ga4gh:SQ.v_QTc1p-MUYdgrRv4LMT6ByXIOsdw3C_ + response: + body: + string: "{\n \"added\": \"2016-08-24T05:03:11Z\",\n \"aliases\": [\n \"MD5:215137b1973c1a5afcf86be7d999574a\",\n + \ \"NCBI:NM_000551.3\",\n \"refseq:NM_000551.3\",\n \"SEGUID:T12L0p2X5E8DbnL0+SwI4Wc1S6g\",\n + \ \"SHA1:4f5d8bd29d97e44f036e72f4f92c08e167354ba8\",\n \"VMC:GS_v_QTc1p-MUYdgrRv4LMT6ByXIOsdw3C_\",\n + \ \"sha512t24u:v_QTc1p-MUYdgrRv4LMT6ByXIOsdw3C_\",\n \"ga4gh:SQ.v_QTc1p-MUYdgrRv4LMT6ByXIOsdw3C_\"\n + \ ],\n \"alphabet\": \"ACGT\",\n \"length\": 4560\n}\n" + headers: {} + status: + code: 200 + message: OK +- request: + body: null + headers: {} + method: GET + uri: http://localhost:5000/seqrepo/1/sequence/ga4gh:SQ.v_QTc1p-MUYdgrRv4LMT6ByXIOsdw3C_?start=4560&end=4560 + response: + body: + string: '' + headers: {} + status: + code: 200 + message: OK +- request: + body: null + headers: {} + method: GET + uri: http://localhost:5000/seqrepo/1/sequence/ga4gh:SQ.v_QTc1p-MUYdgrRv4LMT6ByXIOsdw3C_?start=4559&end=4560 + response: + body: + string: G + headers: {} + status: + code: 200 + message: OK +version: 1 diff --git a/tests/extras/cassettes/test_out_of_bounds[cnv-hgvs-copy-number-change-past-end].yaml b/tests/extras/cassettes/test_out_of_bounds[cnv-hgvs-copy-number-change-past-end].yaml new file mode 100644 index 00000000..1e5e9d6b --- /dev/null +++ b/tests/extras/cassettes/test_out_of_bounds[cnv-hgvs-copy-number-change-past-end].yaml @@ -0,0 +1,25 @@ +interactions: +- request: + body: null + headers: {} + method: GET + uri: http://localhost:5000/seqrepo/1/metadata/refseq:NC_000007.14 + response: + body: + string: "{\n \"added\": \"2016-08-27T21:23:35Z\",\n \"aliases\": [\n \"GRCh38:7\",\n + \ \"GRCh38:chr7\",\n \"GRCh38.p1:7\",\n \"GRCh38.p1:chr7\",\n \"GRCh38.p10:7\",\n + \ \"GRCh38.p10:chr7\",\n \"GRCh38.p11:7\",\n \"GRCh38.p11:chr7\",\n + \ \"GRCh38.p12:7\",\n \"GRCh38.p12:chr7\",\n \"GRCh38.p2:7\",\n \"GRCh38.p2:chr7\",\n + \ \"GRCh38.p3:7\",\n \"GRCh38.p3:chr7\",\n \"GRCh38.p4:7\",\n \"GRCh38.p4:chr7\",\n + \ \"GRCh38.p5:7\",\n \"GRCh38.p5:chr7\",\n \"GRCh38.p6:7\",\n \"GRCh38.p6:chr7\",\n + \ \"GRCh38.p7:7\",\n \"GRCh38.p7:chr7\",\n \"GRCh38.p8:7\",\n \"GRCh38.p8:chr7\",\n + \ \"GRCh38.p9:7\",\n \"GRCh38.p9:chr7\",\n \"MD5:cc044cc2256a1141212660fb07b6171e\",\n + \ \"NCBI:NC_000007.14\",\n \"refseq:NC_000007.14\",\n \"SEGUID:4+JjCcBVhPCr8vdIhUKFycPv8bY\",\n + \ \"SHA1:e3e26309c05584f0abf2f748854285c9c3eff1b6\",\n \"VMC:GS_F-LrLMe1SRpfUZHkQmvkVKFEGaoDeHul\",\n + \ \"sha512t24u:F-LrLMe1SRpfUZHkQmvkVKFEGaoDeHul\",\n \"ga4gh:SQ.F-LrLMe1SRpfUZHkQmvkVKFEGaoDeHul\"\n + \ ],\n \"alphabet\": \"ACGNRSTY\",\n \"length\": 159345973\n}\n" + headers: {} + status: + code: 200 + message: OK +version: 1 diff --git a/tests/extras/cassettes/test_out_of_bounds[gnomad-negative-start].yaml b/tests/extras/cassettes/test_out_of_bounds[gnomad-negative-start].yaml new file mode 100644 index 00000000..2c11ee79 --- /dev/null +++ b/tests/extras/cassettes/test_out_of_bounds[gnomad-negative-start].yaml @@ -0,0 +1,25 @@ +interactions: +- request: + body: null + headers: {} + method: GET + uri: http://localhost:5000/seqrepo/1/metadata/GRCh38:1 + response: + body: + string: "{\n \"added\": \"2016-08-27T21:17:00Z\",\n \"aliases\": [\n \"GRCh38:1\",\n + \ \"GRCh38:chr1\",\n \"GRCh38.p1:1\",\n \"GRCh38.p1:chr1\",\n \"GRCh38.p10:1\",\n + \ \"GRCh38.p10:chr1\",\n \"GRCh38.p11:1\",\n \"GRCh38.p11:chr1\",\n + \ \"GRCh38.p12:1\",\n \"GRCh38.p12:chr1\",\n \"GRCh38.p2:1\",\n \"GRCh38.p2:chr1\",\n + \ \"GRCh38.p3:1\",\n \"GRCh38.p3:chr1\",\n \"GRCh38.p4:1\",\n \"GRCh38.p4:chr1\",\n + \ \"GRCh38.p5:1\",\n \"GRCh38.p5:chr1\",\n \"GRCh38.p6:1\",\n \"GRCh38.p6:chr1\",\n + \ \"GRCh38.p7:1\",\n \"GRCh38.p7:chr1\",\n \"GRCh38.p8:1\",\n \"GRCh38.p8:chr1\",\n + \ \"GRCh38.p9:1\",\n \"GRCh38.p9:chr1\",\n \"MD5:6aef897c3d6ff0c78aff06ac189178dd\",\n + \ \"NCBI:NC_000001.11\",\n \"refseq:NC_000001.11\",\n \"SEGUID:FCUd6VJ6uikS/VWLbhGdVmj2rOA\",\n + \ \"SHA1:14251de9527aba2912fd558b6e119d5668f6ace0\",\n \"VMC:GS_Ya6Rs7DHhDeg7YaOSg1EoNi3U_nQ9SvO\",\n + \ \"sha512t24u:Ya6Rs7DHhDeg7YaOSg1EoNi3U_nQ9SvO\",\n \"ga4gh:SQ.Ya6Rs7DHhDeg7YaOSg1EoNi3U_nQ9SvO\"\n + \ ],\n \"alphabet\": \"ACGMNRT\",\n \"length\": 248956422\n}\n" + headers: {} + status: + code: 200 + message: OK +version: 1 diff --git a/tests/extras/cassettes/test_out_of_bounds[gnomad-past-end-no-require-validation].yaml b/tests/extras/cassettes/test_out_of_bounds[gnomad-past-end-no-require-validation].yaml new file mode 100644 index 00000000..2c11ee79 --- /dev/null +++ b/tests/extras/cassettes/test_out_of_bounds[gnomad-past-end-no-require-validation].yaml @@ -0,0 +1,25 @@ +interactions: +- request: + body: null + headers: {} + method: GET + uri: http://localhost:5000/seqrepo/1/metadata/GRCh38:1 + response: + body: + string: "{\n \"added\": \"2016-08-27T21:17:00Z\",\n \"aliases\": [\n \"GRCh38:1\",\n + \ \"GRCh38:chr1\",\n \"GRCh38.p1:1\",\n \"GRCh38.p1:chr1\",\n \"GRCh38.p10:1\",\n + \ \"GRCh38.p10:chr1\",\n \"GRCh38.p11:1\",\n \"GRCh38.p11:chr1\",\n + \ \"GRCh38.p12:1\",\n \"GRCh38.p12:chr1\",\n \"GRCh38.p2:1\",\n \"GRCh38.p2:chr1\",\n + \ \"GRCh38.p3:1\",\n \"GRCh38.p3:chr1\",\n \"GRCh38.p4:1\",\n \"GRCh38.p4:chr1\",\n + \ \"GRCh38.p5:1\",\n \"GRCh38.p5:chr1\",\n \"GRCh38.p6:1\",\n \"GRCh38.p6:chr1\",\n + \ \"GRCh38.p7:1\",\n \"GRCh38.p7:chr1\",\n \"GRCh38.p8:1\",\n \"GRCh38.p8:chr1\",\n + \ \"GRCh38.p9:1\",\n \"GRCh38.p9:chr1\",\n \"MD5:6aef897c3d6ff0c78aff06ac189178dd\",\n + \ \"NCBI:NC_000001.11\",\n \"refseq:NC_000001.11\",\n \"SEGUID:FCUd6VJ6uikS/VWLbhGdVmj2rOA\",\n + \ \"SHA1:14251de9527aba2912fd558b6e119d5668f6ace0\",\n \"VMC:GS_Ya6Rs7DHhDeg7YaOSg1EoNi3U_nQ9SvO\",\n + \ \"sha512t24u:Ya6Rs7DHhDeg7YaOSg1EoNi3U_nQ9SvO\",\n \"ga4gh:SQ.Ya6Rs7DHhDeg7YaOSg1EoNi3U_nQ9SvO\"\n + \ ],\n \"alphabet\": \"ACGMNRT\",\n \"length\": 248956422\n}\n" + headers: {} + status: + code: 200 + message: OK +version: 1 diff --git a/tests/extras/cassettes/test_out_of_bounds[gnomad-past-end].yaml b/tests/extras/cassettes/test_out_of_bounds[gnomad-past-end].yaml new file mode 100644 index 00000000..2c11ee79 --- /dev/null +++ b/tests/extras/cassettes/test_out_of_bounds[gnomad-past-end].yaml @@ -0,0 +1,25 @@ +interactions: +- request: + body: null + headers: {} + method: GET + uri: http://localhost:5000/seqrepo/1/metadata/GRCh38:1 + response: + body: + string: "{\n \"added\": \"2016-08-27T21:17:00Z\",\n \"aliases\": [\n \"GRCh38:1\",\n + \ \"GRCh38:chr1\",\n \"GRCh38.p1:1\",\n \"GRCh38.p1:chr1\",\n \"GRCh38.p10:1\",\n + \ \"GRCh38.p10:chr1\",\n \"GRCh38.p11:1\",\n \"GRCh38.p11:chr1\",\n + \ \"GRCh38.p12:1\",\n \"GRCh38.p12:chr1\",\n \"GRCh38.p2:1\",\n \"GRCh38.p2:chr1\",\n + \ \"GRCh38.p3:1\",\n \"GRCh38.p3:chr1\",\n \"GRCh38.p4:1\",\n \"GRCh38.p4:chr1\",\n + \ \"GRCh38.p5:1\",\n \"GRCh38.p5:chr1\",\n \"GRCh38.p6:1\",\n \"GRCh38.p6:chr1\",\n + \ \"GRCh38.p7:1\",\n \"GRCh38.p7:chr1\",\n \"GRCh38.p8:1\",\n \"GRCh38.p8:chr1\",\n + \ \"GRCh38.p9:1\",\n \"GRCh38.p9:chr1\",\n \"MD5:6aef897c3d6ff0c78aff06ac189178dd\",\n + \ \"NCBI:NC_000001.11\",\n \"refseq:NC_000001.11\",\n \"SEGUID:FCUd6VJ6uikS/VWLbhGdVmj2rOA\",\n + \ \"SHA1:14251de9527aba2912fd558b6e119d5668f6ace0\",\n \"VMC:GS_Ya6Rs7DHhDeg7YaOSg1EoNi3U_nQ9SvO\",\n + \ \"sha512t24u:Ya6Rs7DHhDeg7YaOSg1EoNi3U_nQ9SvO\",\n \"ga4gh:SQ.Ya6Rs7DHhDeg7YaOSg1EoNi3U_nQ9SvO\"\n + \ ],\n \"alphabet\": \"ACGMNRT\",\n \"length\": 248956422\n}\n" + headers: {} + status: + code: 200 + message: OK +version: 1 diff --git a/tests/extras/cassettes/test_out_of_bounds[hgvs-n-dup-past-end].yaml b/tests/extras/cassettes/test_out_of_bounds[hgvs-n-dup-past-end].yaml new file mode 100644 index 00000000..ab28a89d --- /dev/null +++ b/tests/extras/cassettes/test_out_of_bounds[hgvs-n-dup-past-end].yaml @@ -0,0 +1,18 @@ +interactions: +- request: + body: null + headers: {} + method: GET + uri: http://localhost:5000/seqrepo/1/metadata/refseq:NM_000551.3 + response: + body: + string: "{\n \"added\": \"2016-08-24T05:03:11Z\",\n \"aliases\": [\n \"MD5:215137b1973c1a5afcf86be7d999574a\",\n + \ \"NCBI:NM_000551.3\",\n \"refseq:NM_000551.3\",\n \"SEGUID:T12L0p2X5E8DbnL0+SwI4Wc1S6g\",\n + \ \"SHA1:4f5d8bd29d97e44f036e72f4f92c08e167354ba8\",\n \"VMC:GS_v_QTc1p-MUYdgrRv4LMT6ByXIOsdw3C_\",\n + \ \"sha512t24u:v_QTc1p-MUYdgrRv4LMT6ByXIOsdw3C_\",\n \"ga4gh:SQ.v_QTc1p-MUYdgrRv4LMT6ByXIOsdw3C_\"\n + \ ],\n \"alphabet\": \"ACGT\",\n \"length\": 4560\n}\n" + headers: {} + status: + code: 200 + message: OK +version: 1 diff --git a/tests/extras/cassettes/test_out_of_bounds[hgvs-n-insertion-past-end].yaml b/tests/extras/cassettes/test_out_of_bounds[hgvs-n-insertion-past-end].yaml new file mode 100644 index 00000000..ab28a89d --- /dev/null +++ b/tests/extras/cassettes/test_out_of_bounds[hgvs-n-insertion-past-end].yaml @@ -0,0 +1,18 @@ +interactions: +- request: + body: null + headers: {} + method: GET + uri: http://localhost:5000/seqrepo/1/metadata/refseq:NM_000551.3 + response: + body: + string: "{\n \"added\": \"2016-08-24T05:03:11Z\",\n \"aliases\": [\n \"MD5:215137b1973c1a5afcf86be7d999574a\",\n + \ \"NCBI:NM_000551.3\",\n \"refseq:NM_000551.3\",\n \"SEGUID:T12L0p2X5E8DbnL0+SwI4Wc1S6g\",\n + \ \"SHA1:4f5d8bd29d97e44f036e72f4f92c08e167354ba8\",\n \"VMC:GS_v_QTc1p-MUYdgrRv4LMT6ByXIOsdw3C_\",\n + \ \"sha512t24u:v_QTc1p-MUYdgrRv4LMT6ByXIOsdw3C_\",\n \"ga4gh:SQ.v_QTc1p-MUYdgrRv4LMT6ByXIOsdw3C_\"\n + \ ],\n \"alphabet\": \"ACGT\",\n \"length\": 4560\n}\n" + headers: {} + status: + code: 200 + message: OK +version: 1 diff --git a/tests/extras/cassettes/test_out_of_bounds[hgvs-p-ter-at-length-plus-one].yaml b/tests/extras/cassettes/test_out_of_bounds[hgvs-p-ter-at-length-plus-one].yaml new file mode 100644 index 00000000..dba45e9a --- /dev/null +++ b/tests/extras/cassettes/test_out_of_bounds[hgvs-p-ter-at-length-plus-one].yaml @@ -0,0 +1,26 @@ +interactions: +- request: + body: null + headers: {} + method: GET + uri: http://localhost:5000/seqrepo/1/metadata/refseq:NP_001346993.1 + response: + body: + string: "{\n \"added\": \"2016-08-24T04:59:23Z\",\n \"aliases\": [\n \"Ensembl:ENSP00000480268.1\",\n + \ \"ensembl:ENSP00000480268.1\",\n \"Ensembl:ENSP00000491180.1\",\n \"ensembl:ENSP00000491180.1\",\n + \ \"Ensembl:ENSP00000491338.1\",\n \"ensembl:ENSP00000491338.1\",\n \"Ensembl:ENSP00000491353.1\",\n + \ \"ensembl:ENSP00000491353.1\",\n \"Ensembl:ENSP00000492701.1\",\n \"ensembl:ENSP00000492701.1\",\n + \ \"Ensembl:ENSP00000498790.1\",\n \"ensembl:ENSP00000498790.1\",\n \"MD5:fecf2eee2cdc50588a641e472e062be1\",\n + \ \"NCBI:NP_001346993.1\",\n \"refseq:NP_001346993.1\",\n \"NCBI:NP_001347000.1\",\n + \ \"refseq:NP_001347000.1\",\n \"NCBI:NP_001355060.1\",\n \"refseq:NP_001355060.1\",\n + \ \"NCBI:XP_011534418.1\",\n \"refseq:XP_011534418.1\",\n \"NCBI:XP_024302319.1\",\n + \ \"refseq:XP_024302319.1\",\n \"NCBI:XP_054212402.1\",\n \"refseq:XP_054212402.1\",\n + \ \"NCBI:XP_054212403.1\",\n \"refseq:XP_054212403.1\",\n \"SEGUID:8Lnknw+hAAOAjiLGHus4bfyDy0k\",\n + \ \"SHA1:f0b9e49f0fa10003808e22c61eeb386dfc83cb49\",\n \"VMC:GS_IPAWzkahAXVA3fBdoFluaU4NA3xTYUer\",\n + \ \"sha512t24u:IPAWzkahAXVA3fBdoFluaU4NA3xTYUer\",\n \"ga4gh:SQ.IPAWzkahAXVA3fBdoFluaU4NA3xTYUer\"\n + \ ],\n \"alphabet\": \"ACDEFGHIKLMNPQRSTVWY\",\n \"length\": 193\n}\n" + headers: {} + status: + code: 200 + message: OK +version: 1 diff --git a/tests/extras/cassettes/test_out_of_bounds[spdi-insertion-past-end].yaml b/tests/extras/cassettes/test_out_of_bounds[spdi-insertion-past-end].yaml new file mode 100644 index 00000000..ab28a89d --- /dev/null +++ b/tests/extras/cassettes/test_out_of_bounds[spdi-insertion-past-end].yaml @@ -0,0 +1,18 @@ +interactions: +- request: + body: null + headers: {} + method: GET + uri: http://localhost:5000/seqrepo/1/metadata/refseq:NM_000551.3 + response: + body: + string: "{\n \"added\": \"2016-08-24T05:03:11Z\",\n \"aliases\": [\n \"MD5:215137b1973c1a5afcf86be7d999574a\",\n + \ \"NCBI:NM_000551.3\",\n \"refseq:NM_000551.3\",\n \"SEGUID:T12L0p2X5E8DbnL0+SwI4Wc1S6g\",\n + \ \"SHA1:4f5d8bd29d97e44f036e72f4f92c08e167354ba8\",\n \"VMC:GS_v_QTc1p-MUYdgrRv4LMT6ByXIOsdw3C_\",\n + \ \"sha512t24u:v_QTc1p-MUYdgrRv4LMT6ByXIOsdw3C_\",\n \"ga4gh:SQ.v_QTc1p-MUYdgrRv4LMT6ByXIOsdw3C_\"\n + \ ],\n \"alphabet\": \"ACGT\",\n \"length\": 4560\n}\n" + headers: {} + status: + code: 200 + message: OK +version: 1 diff --git a/tests/extras/cassettes/test_start_after_end[cnv-hgvs-dup].yaml b/tests/extras/cassettes/test_start_after_end[cnv-hgvs-dup].yaml new file mode 100644 index 00000000..60e147ee --- /dev/null +++ b/tests/extras/cassettes/test_start_after_end[cnv-hgvs-dup].yaml @@ -0,0 +1,24 @@ +interactions: +- request: + body: null + headers: {} + method: GET + uri: http://localhost:5000/seqrepo/1/metadata/refseq:NC_000012.11 + response: + body: + string: "{\n \"added\": \"2016-08-24T05:10:10Z\",\n \"aliases\": [\n \"GRCh37:12\",\n + \ \"GRCh37:chr12\",\n \"GRCh37.p10:12\",\n \"GRCh37.p10:chr12\",\n + \ \"GRCh37.p11:12\",\n \"GRCh37.p11:chr12\",\n \"GRCh37.p12:12\",\n + \ \"GRCh37.p12:chr12\",\n \"GRCh37.p13:12\",\n \"GRCh37.p13:chr12\",\n + \ \"GRCh37.p2:12\",\n \"GRCh37.p2:chr12\",\n \"GRCh37.p5:12\",\n \"GRCh37.p5:chr12\",\n + \ \"GRCh37.p9:12\",\n \"GRCh37.p9:chr12\",\n \"MD5:51851ac0e1a115847ad36449b0015864\",\n + \ \"NCBI:NC_000012.11\",\n \"refseq:NC_000012.11\",\n \"SEGUID:XRw0p+KCGTdtkYmTmv1kmCeNaFE\",\n + \ \"SHA1:5d1c34a7e28219376d9189939afd6498278d6851\",\n \"VMC:GS_105bBysLoDFQHhajooTAUyUkNiZ8LJEH\",\n + \ \"sha512t24u:105bBysLoDFQHhajooTAUyUkNiZ8LJEH\",\n \"ga4gh:SQ.105bBysLoDFQHhajooTAUyUkNiZ8LJEH\",\n + \ \"hs37-1kg:12\",\n \"hs37d5:12\"\n ],\n \"alphabet\": \"ACGNT\",\n + \ \"length\": 133851895\n}\n" + headers: {} + status: + code: 200 + message: OK +version: 1 diff --git a/tests/extras/cassettes/test_start_after_end[hgvs-g-dup].yaml b/tests/extras/cassettes/test_start_after_end[hgvs-g-dup].yaml new file mode 100644 index 00000000..60e147ee --- /dev/null +++ b/tests/extras/cassettes/test_start_after_end[hgvs-g-dup].yaml @@ -0,0 +1,24 @@ +interactions: +- request: + body: null + headers: {} + method: GET + uri: http://localhost:5000/seqrepo/1/metadata/refseq:NC_000012.11 + response: + body: + string: "{\n \"added\": \"2016-08-24T05:10:10Z\",\n \"aliases\": [\n \"GRCh37:12\",\n + \ \"GRCh37:chr12\",\n \"GRCh37.p10:12\",\n \"GRCh37.p10:chr12\",\n + \ \"GRCh37.p11:12\",\n \"GRCh37.p11:chr12\",\n \"GRCh37.p12:12\",\n + \ \"GRCh37.p12:chr12\",\n \"GRCh37.p13:12\",\n \"GRCh37.p13:chr12\",\n + \ \"GRCh37.p2:12\",\n \"GRCh37.p2:chr12\",\n \"GRCh37.p5:12\",\n \"GRCh37.p5:chr12\",\n + \ \"GRCh37.p9:12\",\n \"GRCh37.p9:chr12\",\n \"MD5:51851ac0e1a115847ad36449b0015864\",\n + \ \"NCBI:NC_000012.11\",\n \"refseq:NC_000012.11\",\n \"SEGUID:XRw0p+KCGTdtkYmTmv1kmCeNaFE\",\n + \ \"SHA1:5d1c34a7e28219376d9189939afd6498278d6851\",\n \"VMC:GS_105bBysLoDFQHhajooTAUyUkNiZ8LJEH\",\n + \ \"sha512t24u:105bBysLoDFQHhajooTAUyUkNiZ8LJEH\",\n \"ga4gh:SQ.105bBysLoDFQHhajooTAUyUkNiZ8LJEH\",\n + \ \"hs37-1kg:12\",\n \"hs37d5:12\"\n ],\n \"alphabet\": \"ACGNT\",\n + \ \"length\": 133851895\n}\n" + headers: {} + status: + code: 200 + message: OK +version: 1 diff --git a/tests/extras/cassettes/test_start_after_end[hgvs-g-identity].yaml b/tests/extras/cassettes/test_start_after_end[hgvs-g-identity].yaml new file mode 100644 index 00000000..60e147ee --- /dev/null +++ b/tests/extras/cassettes/test_start_after_end[hgvs-g-identity].yaml @@ -0,0 +1,24 @@ +interactions: +- request: + body: null + headers: {} + method: GET + uri: http://localhost:5000/seqrepo/1/metadata/refseq:NC_000012.11 + response: + body: + string: "{\n \"added\": \"2016-08-24T05:10:10Z\",\n \"aliases\": [\n \"GRCh37:12\",\n + \ \"GRCh37:chr12\",\n \"GRCh37.p10:12\",\n \"GRCh37.p10:chr12\",\n + \ \"GRCh37.p11:12\",\n \"GRCh37.p11:chr12\",\n \"GRCh37.p12:12\",\n + \ \"GRCh37.p12:chr12\",\n \"GRCh37.p13:12\",\n \"GRCh37.p13:chr12\",\n + \ \"GRCh37.p2:12\",\n \"GRCh37.p2:chr12\",\n \"GRCh37.p5:12\",\n \"GRCh37.p5:chr12\",\n + \ \"GRCh37.p9:12\",\n \"GRCh37.p9:chr12\",\n \"MD5:51851ac0e1a115847ad36449b0015864\",\n + \ \"NCBI:NC_000012.11\",\n \"refseq:NC_000012.11\",\n \"SEGUID:XRw0p+KCGTdtkYmTmv1kmCeNaFE\",\n + \ \"SHA1:5d1c34a7e28219376d9189939afd6498278d6851\",\n \"VMC:GS_105bBysLoDFQHhajooTAUyUkNiZ8LJEH\",\n + \ \"sha512t24u:105bBysLoDFQHhajooTAUyUkNiZ8LJEH\",\n \"ga4gh:SQ.105bBysLoDFQHhajooTAUyUkNiZ8LJEH\",\n + \ \"hs37-1kg:12\",\n \"hs37d5:12\"\n ],\n \"alphabet\": \"ACGNT\",\n + \ \"length\": 133851895\n}\n" + headers: {} + status: + code: 200 + message: OK +version: 1 diff --git a/tests/extras/test_allele_translator.py b/tests/extras/test_allele_translator.py index 2e4d3dff..f1726735 100644 --- a/tests/extras/test_allele_translator.py +++ b/tests/extras/test_allele_translator.py @@ -1,9 +1,13 @@ +from unittest.mock import MagicMock + +import hgvs.parser import pytest from pydantic import ValidationError from ga4gh.vrs import models -from ga4gh.vrs.dataproxy import DataProxyValidationError -from ga4gh.vrs.extras.translator import AlleleTranslator +from ga4gh.vrs.dataproxy import DataProxyValidationError, UnresolvableAccessionError +from ga4gh.vrs.extras.translator import AlleleTranslator, CnvTranslator +from ga4gh.vrs.utils.hgvs_tools import HgvsTools @pytest.fixture(scope="module") @@ -351,6 +355,85 @@ def test_from_trailing_input_rejected(tlr, expr, fmt): tlr.translate_from(expr, fmt=fmt) +def _unresolving_translator(cls: type) -> AlleleTranslator | CnvTranslator: + """Build a translator whose data proxy resolves no accession, without UTA.""" + data_proxy = MagicMock() + data_proxy.derive_refget_accession.return_value = None + tlr = cls(data_proxy=data_proxy, identify=False) + hgvs_tools = HgvsTools.__new__(HgvsTools) # skip __init__'s UTA connection + hgvs_tools.parser = hgvs.parser.Parser() + hgvs_tools.data_proxy = data_proxy + tlr._lazy_hgvs_tools = hgvs_tools + return tlr + + +@pytest.mark.parametrize( + ("cls", "fmt", "expr", "accession"), + [ + (AlleleTranslator, "hgvs", "NC_000099.1:g.100A>T", "NC_000099.1"), + (AlleleTranslator, "spdi", "NC_000099.1:100:A:T", "NC_000099.1"), + (AlleleTranslator, "gnomad", "Z-100-A-T", "GRCh38:Z"), + (CnvTranslator, "hgvs", "NC_000099.1:g.100_200del", "NC_000099.1"), + ], +) +@pytest.mark.parametrize("given_fmt", [True, False], ids=["given", "guessed"]) +@pytest.mark.filterwarnings("ignore::DeprecationWarning") +def test_from_unresolvable_accession(cls, fmt, expr, accession, given_fmt): + """Report an unknown accession as a lookup failure, not a parse failure. + + Applies whether the expression's format is given or guessed. + """ + tlr = _unresolving_translator(cls) + with pytest.raises(UnresolvableAccessionError) as excinfo: + tlr.translate_from(expr, fmt=fmt if given_fmt else None) + assert excinfo.value.accession == accession + + +def _raises(exc: Exception): + def translate(*_args, **_kwargs): + raise exc + + return translate + + +def test_guess_continues_past_a_format_that_fails(): + """A format that recognizes the input but fails does not end the guess.""" + tlr = _unresolving_translator(AlleleTranslator) + allele = object() + tlr.from_translators = { + "unmatched": lambda *_a, **_k: None, + "spurious": _raises(UnresolvableAccessionError("X")), + "real": lambda *_a, **_k: allele, + } + assert tlr.translate_from("anything") is allele + + +def test_guess_reports_the_first_recognized_failure(): + tlr = _unresolving_translator(AlleleTranslator) + first = UnresolvableAccessionError("X") + tlr.from_translators = { + "unmatched": lambda *_a, **_k: None, + "a": _raises(first), + "b": _raises(DataProxyValidationError("bad ref")), + } + with pytest.raises(UnresolvableAccessionError) as excinfo: + tlr.translate_from("anything") + assert excinfo.value is first + if hasattr(first, "add_note"): + assert "b (DataProxyValidationError: bad ref)" in excinfo.value.__notes__[0] + + +def test_guess_does_not_absorb_unexpected_errors(): + """Errors outside the diagnostic set are programming errors and abort.""" + tlr = _unresolving_translator(AlleleTranslator) + tlr.from_translators = { + "broken": _raises(TypeError("bug")), + "real": lambda *_a, **_k: object(), + } + with pytest.raises(TypeError, match="bug"): + tlr.translate_from("anything") + + @pytest.mark.vcr def test_from_gnomad(tlr): do_normalize = False diff --git a/tests/extras/test_annotate_vcf.py b/tests/extras/test_annotate_vcf.py index 0d66dbe9..c8c8887f 100644 --- a/tests/extras/test_annotate_vcf.py +++ b/tests/extras/test_annotate_vcf.py @@ -9,7 +9,8 @@ import pysam import pytest -from ga4gh.vrs.dataproxy import DataProxyValidationError, SeqRepoRESTDataProxy +from ga4gh.vrs.dataproxy import DataProxyValidationError +from ga4gh.vrs.dataproxy_seqrepo import SeqRepoRESTDataProxy from ga4gh.vrs.extras.annotator.vcf import VcfAnnotator, VcfAnnotatorError TEST_DATA_DIR = Path("tests/extras/data") @@ -199,7 +200,7 @@ def test_get_vrs_object_invalid_input(vcf_annotator: VcfAnnotator, caplog): # No CHROM vcf_annotator._get_vrs_object(".-140753336-A-T", [], {}, "GRCh38") - assert "KeyError when getting refget accession: GRCh38:." in caplog.text + assert "Unable to derive refget accession for GRCh38:." in caplog.text # No POS vcf_annotator._get_vrs_object("7-.-A-T", [], {}, "GRCh38") diff --git a/tests/extras/test_location_bounds.py b/tests/extras/test_location_bounds.py new file mode 100644 index 00000000..83727d05 --- /dev/null +++ b/tests/extras/test_location_bounds.py @@ -0,0 +1,189 @@ +"""Out-of-bounds locations, and locations whose start is after their end, are +rejected on every translator input path except ``vrs``, whose input is trusted +as-is + +Sequence lengths used below: + NM_000551.3 4560 + NP_001346993.1 193 + NC_000007.14 159345973 + NC_000012.11 133851895 + GRCh38:1 248956422 +""" + +import os +import re + +import pytest + +from ga4gh.vrs.dataproxy import DataProxyValidationError +from ga4gh.vrs.dataproxy_seqrepo import SeqRepoRESTDataProxy +from ga4gh.vrs.extras.translator import AlleleTranslator, CnvTranslator + +# Refget accession each input sequence resolves to, named alongside it in errors +REFGET_ACCESSIONS = { + "GRCh38:1": "ga4gh:SQ.Ya6Rs7DHhDeg7YaOSg1EoNi3U_nQ9SvO", + "refseq:NC_000007.14": "ga4gh:SQ.F-LrLMe1SRpfUZHkQmvkVKFEGaoDeHul", + "refseq:NM_000551.3": "ga4gh:SQ.v_QTc1p-MUYdgrRv4LMT6ByXIOsdw3C_", + "refseq:NP_001346993.1": "ga4gh:SQ.IPAWzkahAXVA3fBdoFluaU4NA3xTYUer", +} + + +@pytest.fixture +def data_proxy() -> SeqRepoRESTDataProxy: + # Metadata lookups are lru_cached per dataproxy instance, so a fresh instance per + # test keeps each cassette self-contained regardless of test order + return SeqRepoRESTDataProxy( + base_url=os.environ.get("SEQREPO_REST_URL", "http://localhost:5000/seqrepo"), + disable_healthcheck=True, + ) + + +@pytest.fixture +def allele_tlr(data_proxy: SeqRepoRESTDataProxy) -> AlleleTranslator: + return AlleleTranslator(data_proxy=data_proxy) + + +@pytest.fixture +def cnv_tlr(data_proxy: SeqRepoRESTDataProxy) -> CnvTranslator: + return CnvTranslator(data_proxy=data_proxy) + + +def _bounds_msg(sequence_id: str, detail: str, seq_len: int) -> str: + return ( + f"Location out of bounds on {sequence_id} ({REFGET_ACCESSIONS[sequence_id]}): " + f"{detail} not within [0, {seq_len}]" + ) + + +out_of_bounds_cases = [ + { + "id": "hgvs-n-insertion-past-end", + "tlr_fixture": "allele_tlr", + "fmt": "hgvs", + "var": "NM_000551.3:n.4561_4562insA", + "msg": _bounds_msg("refseq:NM_000551.3", "start=4561, end=4561", 4560), + }, + # Checked before the duplicated reference is fetched, which would be truncated + { + "id": "hgvs-n-dup-past-end", + "tlr_fixture": "allele_tlr", + "fmt": "hgvs", + "var": "NM_000551.3:n.4560_4561dup", + "msg": _bounds_msg("refseq:NM_000551.3", "end=4561", 4560), + }, + # ClinVar references the stop codon, which is not part of the protein sequence + { + "id": "hgvs-p-ter-at-length-plus-one", + "tlr_fixture": "allele_tlr", + "fmt": "hgvs", + "var": "NP_001346993.1:p.Ter194del", + "msg": _bounds_msg("refseq:NP_001346993.1", "end=194", 193), + }, + # Zero-width: an out-of-range fetch returns "" and would compare equal to the + # empty reference, so only a coordinate check can catch this + { + "id": "spdi-insertion-past-end", + "tlr_fixture": "allele_tlr", + "fmt": "spdi", + "var": "NM_000551.3:5000:0:AAA", + "msg": _bounds_msg("refseq:NM_000551.3", "start=5000, end=5000", 4560), + }, + # Must report the bounds error, not "Reference mismatch ... correct ref is ''" + { + "id": "gnomad-past-end", + "tlr_fixture": "allele_tlr", + "fmt": "gnomad", + "var": "1-248956423-A-T", + "msg": _bounds_msg("GRCh38:1", "end=248956423", 248956422), + }, + { + "id": "gnomad-past-end-no-require-validation", + "tlr_fixture": "allele_tlr", + "fmt": "gnomad", + "var": "1-248956423-A-T", + "kwargs": {"require_validation": False}, + "msg": _bounds_msg("GRCh38:1", "end=248956423", 248956422), + }, + { + "id": "gnomad-negative-start", + "tlr_fixture": "allele_tlr", + "fmt": "gnomad", + "var": "1-0-A-T", + "msg": _bounds_msg("GRCh38:1", "start=-1", 248956422), + }, + { + "id": "cnv-hgvs-copy-number-change-past-end", + "tlr_fixture": "cnv_tlr", + "fmt": "hgvs", + "var": "NC_000007.14:g.159400000_159400100del", + "msg": _bounds_msg( + "refseq:NC_000007.14", "start=159399999, end=159400100", 159345973 + ), + }, +] + + +# ClinVar VariationID 1299657 gives NC_000012.11:g.110593351_110576466dup: both +# coordinates are on the sequence, but the start is after the end. Identity and +# duplication fetch the reference, so they are checked before the fetch, which +# the backend would otherwise reject with an error of its own. +START_AFTER_END_MSG = ( + "Interval start must not be after end on refseq:NC_000012.11: " + "start=110593350, end=110576466" +) + +start_after_end_cases = [ + { + "id": "hgvs-g-dup", + "tlr_fixture": "allele_tlr", + "var": "NC_000012.11:g.110593351_110576466dup", + }, + { + "id": "hgvs-g-identity", + "tlr_fixture": "allele_tlr", + "var": "NC_000012.11:g.110593351_110576466=", + }, + { + "id": "cnv-hgvs-dup", + "tlr_fixture": "cnv_tlr", + "var": "NC_000012.11:g.110593351_110576466dup", + }, +] + + +in_bounds_cases = [ + { + "id": "spdi-insertion-at-end", + "tlr_fixture": "allele_tlr", + "fmt": "spdi", + "var": "NM_000551.3:4560:0:AAA", + "expected_location": {"start": 4560, "end": 4560}, + }, +] + + +@pytest.mark.parametrize("case", out_of_bounds_cases, ids=lambda c: c["id"]) +@pytest.mark.vcr +def test_out_of_bounds(request: pytest.FixtureRequest, case: dict) -> None: + tlr = request.getfixturevalue(case["tlr_fixture"]) + with pytest.raises(DataProxyValidationError, match=f"^{re.escape(case['msg'])}$"): + tlr.translate_from(case["var"], fmt=case["fmt"], **case.get("kwargs", {})) + + +@pytest.mark.parametrize("case", start_after_end_cases, ids=lambda c: c["id"]) +@pytest.mark.vcr +def test_start_after_end(request: pytest.FixtureRequest, case: dict) -> None: + tlr = request.getfixturevalue(case["tlr_fixture"]) + with pytest.raises( + DataProxyValidationError, match=f"^{re.escape(START_AFTER_END_MSG)}$" + ): + tlr.translate_from(case["var"], fmt="hgvs") + + +@pytest.mark.parametrize("case", in_bounds_cases, ids=lambda c: c["id"]) +@pytest.mark.vcr +def test_in_bounds(request: pytest.FixtureRequest, case: dict) -> None: + tlr = request.getfixturevalue(case["tlr_fixture"]) + location = tlr.translate_from(case["var"], fmt=case["fmt"]).location.model_dump() + expected_location = case["expected_location"] + assert {k: location[k] for k in expected_location} == expected_location diff --git a/tests/test_dataproxy.py b/tests/test_dataproxy.py index f306c27a..4213b49f 100644 --- a/tests/test_dataproxy.py +++ b/tests/test_dataproxy.py @@ -1,22 +1,64 @@ -import os import re +import sys +from types import SimpleNamespace +from unittest.mock import MagicMock import pytest -from ga4gh.vrs.dataproxy import create_dataproxy +from ga4gh.vrs import models +from ga4gh.vrs.dataproxy import ( + AliasDataProxy, + DataProxy, + DataProxyValidationError, + UnresolvableAccessionError, + coerce_accession_namespace, + create_dataproxy, + require_refget_accession, + resolve_cache_size, +) +from ga4gh.vrs.dataproxy_seqrepo import SeqRepoDataProxy, SeqRepoRESTDataProxy + + +@pytest.mark.parametrize( + ("ac", "expected"), + [ + ("LRG_1", "lrg:LRG_1"), + ("LRG_1g", "lrg:LRG_1g"), + ("LRG_199t1", "lrg:LRG_199t1"), + ("LRG_199p1", "lrg:LRG_199p1"), + ("NM_000551.3", "refseq:NM_000551.3"), + ("ENST00000530893.6", "ensembl:ENST00000530893.6"), + ("lrg:LRG_1t1", "lrg:LRG_1t1"), + ], +) +def test_coerce_accession_namespace(ac: str, expected: str) -> None: + assert coerce_accession_namespace(ac) == expected + + +@pytest.mark.parametrize( + "ac", ["LRG_", "LRG_1x1", "LRG_1t", "LRG1", "NOT_AN_ACCESSION"] +) +def test_coerce_accession_namespace_rejects(ac: str) -> None: + with pytest.raises(ValueError, match="Could not infer namespace"): + coerce_accession_namespace(ac) @pytest.mark.parametrize("dp", ["rest_dataproxy", "dataproxy"]) @pytest.mark.vcr def test_data_proxies(dp, request): dataproxy = request.getfixturevalue(dp) - r = dataproxy.get_metadata("NM_000551.3") - assert r["length"] == 4560 - assert "ga4gh:SQ.v_QTc1p-MUYdgrRv4LMT6ByXIOsdw3C_" in r["aliases"] - r = dataproxy.get_metadata("NC_000013.11") - assert r["length"] == 114364328 - assert "ga4gh:SQ._0wi-qoDrvram155UmcSC-zA5ZK4fpLT" in r["aliases"] + length = dataproxy.get_sequence_length("NM_000551.3") + assert length == 4560 + + aliases = dataproxy.get_aliases("NM_000551.3") + assert "ga4gh:SQ.v_QTc1p-MUYdgrRv4LMT6ByXIOsdw3C_" in aliases + + length = dataproxy.get_sequence_length("NC_000013.11") + assert length == 114364328 + + aliases = dataproxy.get_aliases("NC_000013.11") + assert "ga4gh:SQ._0wi-qoDrvram155UmcSC-zA5ZK4fpLT" in aliases seq = dataproxy.get_sequence("ga4gh:SQ.v_QTc1p-MUYdgrRv4LMT6ByXIOsdw3C_") assert seq.startswith("CCTCGCCTCCGTTACAACGGCCTACGGTGCTGGAGGATCCTTCTGCGCAC") @@ -27,7 +69,7 @@ def test_data_proxies(dp, request): assert seq == "CCTCGCCTCCGTTACAACGGCCTACGGTGCTGGAGGATCCTTCTGCGCAC" -def test_data_proxy_configs(): +def test_data_proxy_configs(monkeypatch): with pytest.raises( ValueError, match=re.escape( @@ -47,6 +89,7 @@ def test_data_proxy_configs(): ): create_dataproxy("fake-dataprovider+http://localhost:5000") + monkeypatch.delenv("GA4GH_VRS_DATAPROXY_URI", raising=False) with pytest.raises( ValueError, match="No data proxy URI provided or found in GA4GH_VRS_DATAPROXY_URI", @@ -54,7 +97,7 @@ def test_data_proxy_configs(): create_dataproxy(None) # check that fallback on env var works - os.environ["GA4GH_VRS_DATAPROXY_URI"] = "seqrepo+:tests/data/seqrepo/latest" + monkeypatch.setenv("GA4GH_VRS_DATAPROXY_URI", "seqrepo+:tests/data/seqrepo/latest") create_dataproxy(None) # check that method arg takes precedence by passing an invalid arg @@ -65,3 +108,344 @@ def test_data_proxy_configs(): ), ): create_dataproxy("file:///path/to/seqrepo/root") + + +def test_refget_remote_uri_configs(monkeypatch, tmp_path): + remote_store = MagicMock() + + class FakeRefgetStore: + open_local = MagicMock() + open_remote = MagicMock(return_value=remote_store) + + monkeypatch.setitem(sys.modules, "gtars", SimpleNamespace()) + monkeypatch.setitem( + sys.modules, + "gtars.refget", + SimpleNamespace(RefgetStore=FakeRefgetStore), + ) + monkeypatch.setenv("GA4GH_VRS_REFGET_CACHE_DIR", str(tmp_path / "cache-root")) + + create_dataproxy("refget+http://example.org/refget-store") + create_dataproxy("refget+https://example.org/refget-store") + + assert FakeRefgetStore.open_remote.call_count == 2 + + with pytest.raises( + ValueError, + match=re.escape("Refget URI protocol 'fake-scheme' not supported"), + ): + create_dataproxy("refget+fake-scheme://example.org/refget-store") + + +class TestResolveCacheSize: + """Cache sizes are resolved from the environment at import time.""" + + ENV_VAR = "GA4GH_VRS_TEST_CACHE_SIZE" + + @pytest.mark.parametrize( + ("env_value", "expected"), + [ + (None, 8192), # unset falls back to the default + ("", 8192), # empty is treated as unset + ("512", 512), + ("0", 0), # functools semantics: caching disabled + ("-1", None), # unbounded + ("-99", None), + ("none", None), + ("NONE", None), + (" none ", None), + (" 512 ", 512), + ], + ) + def test_valid_values( + self, + monkeypatch: pytest.MonkeyPatch, + env_value: str | None, + expected: int | None, + ) -> None: + if env_value is None: + monkeypatch.delenv(self.ENV_VAR, raising=False) + else: + monkeypatch.setenv(self.ENV_VAR, env_value) + assert resolve_cache_size(self.ENV_VAR, "8192") == expected + + @pytest.mark.parametrize("env_value", ["abc", "1.5", "8k"]) + def test_invalid_values_raise( + self, monkeypatch: pytest.MonkeyPatch, env_value: str + ) -> None: + monkeypatch.setenv(self.ENV_VAR, env_value) + with pytest.raises(ValueError, match="must be an integer or 'none'"): + resolve_cache_size(self.ENV_VAR, "8192") + + +class TestDeriveRefgetAccession: + """A sequence has exactly one refget accession, so >1 ga4gh alias is a bug.""" + + SQ_A = "ga4gh:SQ.v_QTc1p-MUYdgrRv4LMT6ByXIOsdw3C_" + SQ_B = "ga4gh:SQ.Ya6Rs7DHhDeg7YaOSg1EoNi3U_nQ9SvO" + + @staticmethod + def _proxy(aliases: list[str]) -> AliasDataProxy: + class _Proxy(AliasDataProxy): + def get_aliases(self, identifier: str) -> list[str]: # noqa: ARG002 + return aliases + + return _Proxy() + + @pytest.mark.parametrize( + ("aliases", "expected"), + [ + ([SQ_A, "refseq:NM_000551.3"], "SQ.v_QTc1p-MUYdgrRv4LMT6ByXIOsdw3C_"), + (["refseq:NM_000551.3"], None), # no ga4gh alias is a normal miss + ], + ) + def test_resolves(self, aliases: list[str], expected: str | None) -> None: + proxy = self._proxy(aliases) + assert proxy.derive_refget_accession("refseq:NM_000551.3") == expected + + def test_multiple_ga4gh_aliases_raises(self) -> None: + proxy = self._proxy([self.SQ_A, self.SQ_B, "refseq:NM_000551.3"]) + with pytest.raises(ValueError, match="Expected at most one ga4gh alias"): + proxy.derive_refget_accession("refseq:NM_000551.3") + + def test_uninferrable_namespace_is_a_miss(self) -> None: + proxy = self._proxy([self.SQ_A]) + assert proxy.derive_refget_accession("NOT_AN_ACCESSION") is None + + def test_require_refget_accession(self) -> None: + assert ( + require_refget_accession(self._proxy([self.SQ_A]), "refseq:NM_000551.3") + == "SQ.v_QTc1p-MUYdgrRv4LMT6ByXIOsdw3C_" + ) + with pytest.raises(UnresolvableAccessionError) as excinfo: + require_refget_accession(self._proxy([]), "refseq:NM_000551.3") + assert isinstance(excinfo.value, KeyError) + assert excinfo.value.accession == "refseq:NM_000551.3" + assert str(excinfo.value) == ( + "Unable to resolve refseq:NM_000551.3 to a refget accession" + ) + + +class _StubDataProxy(DataProxy): + """Dataproxy serving only lengths and aliases, keyed by identifier""" + + def __init__(self, metadata: dict[str, dict]) -> None: + self.metadata = metadata + + def get_sequence( + self, identifier: str, start: int | None = None, end: int | None = None + ) -> str: + raise NotImplementedError + + def get_sequence_length(self, identifier: str) -> int: + return self.metadata[identifier]["length"] + + def get_aliases(self, identifier: str) -> list[str]: + return self.metadata[identifier]["aliases"] + + +BOUNDS_SEQ_ID = "refseq:NM_000551.3" +BOUNDS_REFGET_ID = "ga4gh:SQ.v_QTc1p-MUYdgrRv4LMT6ByXIOsdw3C_" +BOUNDS_SEQ_LEN = 4560 + + +def _bounds_dp(aliases: list[str]) -> _StubDataProxy: + """Stub serving one sequence's metadata under each of its aliases""" + md = {"length": BOUNDS_SEQ_LEN, "aliases": aliases} + return _StubDataProxy(dict.fromkeys(aliases, md)) + + +# Locations representable on a sequence of length BOUNDS_SEQ_LEN +location_bounds_valid_cases = [ + {"id": "interior", "start": 10, "end": 20}, + {"id": "zero-width-at-start", "start": 0, "end": 0}, + {"id": "terminal-residue", "start": 4559, "end": 4560}, + {"id": "insertion-at-end", "start": 4560, "end": 4560}, + { + "id": "overlapping-ranges", + "start": models.Range([100, 200]), + "end": models.Range([150, 300]), + }, + { + "id": "open-ranges-never-start-after-end", + "start": models.Range([None, 4000]), + "end": models.Range([10, None]), + }, + { + "id": "indefinite-end-open-upper", + "start": 4400, + "end": models.Range([4500, None]), + }, + { + "id": "indefinite-start-open-lower", + "start": models.Range([None, 10]), + "end": 20, + }, + { + "id": "definite-ranges", + "start": models.Range([0, 10]), + "end": models.Range([4500, 4560]), + }, + {"id": "start-undefined", "start": None, "end": 10}, + {"id": "end-undefined", "start": 10, "end": None}, + {"id": "both-undefined", "start": None, "end": None}, +] + +# Locations not representable on a sequence of length BOUNDS_SEQ_LEN, with the +# offending coordinates as reported in the error message +location_bounds_invalid_cases = [ + {"id": "one-past-end", "start": 4559, "end": 4561, "detail": "end=4561"}, + { + "id": "zero-width-past-end", + "start": 5000, + "end": 5000, + "detail": "start=5000, end=5000", + }, + { + "id": "start-past-end-with-start-gt-end", + "start": 99999999, + "end": 5, + "detail": "start=99999999", + }, + {"id": "negative-start", "start": -1, "end": 1, "detail": "start=-1"}, + {"id": "negative-end", "start": 0, "end": -1, "detail": "end=-1"}, + { + "id": "definite-end-past-end", + "start": 4400, + "end": models.Range([4500, 4600]), + "detail": "end=[4500, 4600]", + }, + { + "id": "indefinite-end-lower-bound-past-end", + "start": 4400, + "end": models.Range([4561, None]), + "detail": "end=[4561, None]", + }, + { + "id": "range-with-negative-member", + "start": models.Range([-5, 10]), + "end": 20, + "detail": "start=[-5, 10]", + }, +] + + +# In-bounds locations whose start is after their end. A Range is reported by its +# lowest possible start and highest possible end. +location_start_after_end_cases = [ + {"id": "start-after-end", "start": 4000, "end": 5, "detail": "start=4000, end=5"}, + { + "id": "definite-ranges-disjoint", + "start": models.Range([4000, 4100]), + "end": models.Range([10, 20]), + "detail": "start=4000, end=20", + }, + { + "id": "indefinite-ranges-disjoint", + "start": models.Range([4000, None]), + "end": models.Range([None, 20]), + "detail": "start=4000, end=20", + }, +] + + +def _start_after_end_msg(identifier: str, detail: str) -> str: + return f"Interval start must not be after end on {identifier}: {detail}" + + +@pytest.mark.parametrize("case", location_bounds_valid_cases, ids=lambda c: c["id"]) +def test_validate_location_bounds_valid(case: dict) -> None: + dp = _bounds_dp([BOUNDS_SEQ_ID, BOUNDS_REFGET_ID]) + dp.validate_location_bounds(BOUNDS_SEQ_ID, case["start"], case["end"]) + + +@pytest.mark.parametrize("case", location_bounds_invalid_cases, ids=lambda c: c["id"]) +def test_validate_location_bounds_invalid(case: dict) -> None: + dp = _bounds_dp([BOUNDS_SEQ_ID, BOUNDS_REFGET_ID]) + expected_msg = ( + f"Location out of bounds on {BOUNDS_SEQ_ID} ({BOUNDS_REFGET_ID}): " + f"{case['detail']} not within [0, {BOUNDS_SEQ_LEN}]" + ) + with pytest.raises(DataProxyValidationError, match=f"^{re.escape(expected_msg)}$"): + dp.validate_location_bounds(BOUNDS_SEQ_ID, case["start"], case["end"]) + + +@pytest.mark.parametrize("case", location_start_after_end_cases, ids=lambda c: c["id"]) +def test_validate_location_bounds_start_after_end(case: dict) -> None: + dp = _bounds_dp([BOUNDS_SEQ_ID, BOUNDS_REFGET_ID]) + expected_msg = _start_after_end_msg(BOUNDS_SEQ_ID, case["detail"]) + with pytest.raises(DataProxyValidationError, match=f"^{re.escape(expected_msg)}$"): + dp.validate_location_bounds(BOUNDS_SEQ_ID, case["start"], case["end"]) + + +def test_validate_ref_seq_checks_bounds_before_fetching() -> None: + # The stub's get_sequence raises, so this also shows nothing is fetched + dp = _bounds_dp([BOUNDS_SEQ_ID, BOUNDS_REFGET_ID]) + expected_msg = ( + f"Location out of bounds on {BOUNDS_SEQ_ID} ({BOUNDS_REFGET_ID}): " + f"end=4561 not within [0, {BOUNDS_SEQ_LEN}]" + ) + with pytest.raises(DataProxyValidationError, match=f"^{re.escape(expected_msg)}$"): + dp.validate_ref_seq(BOUNDS_SEQ_ID, 4560, 4561, "A", require_validation=False) + + +def test_seqrepo_get_sequence_rejects_start_after_end_before_the_backend() -> None: + sr = MagicMock() + expected_msg = _start_after_end_msg(BOUNDS_SEQ_ID, "start=4000, end=5") + with pytest.raises(DataProxyValidationError, match=f"^{re.escape(expected_msg)}$"): + SeqRepoDataProxy(sr).get_sequence(BOUNDS_SEQ_ID, 4000, 5) + sr.fetch_uri.assert_not_called() + + +def test_seqrepo_rest_get_sequence_rejects_start_after_end_before_the_backend( + monkeypatch: pytest.MonkeyPatch, +) -> None: + get = MagicMock() + monkeypatch.setattr("ga4gh.vrs.dataproxy_seqrepo.requests.get", get) + dp = SeqRepoRESTDataProxy("http://localhost:5000/seqrepo", disable_healthcheck=True) + expected_msg = _start_after_end_msg(BOUNDS_SEQ_ID, "start=4000, end=5") + with pytest.raises(DataProxyValidationError, match=f"^{re.escape(expected_msg)}$"): + dp.get_sequence(BOUNDS_SEQ_ID, 4000, 5) + get.assert_not_called() + + +# Name used for the sequence in the error message, given the sequence_id passed in +# and the aliases of the sequence +location_bounds_sequence_name_cases = [ + { + "id": "input-and-refget", + "sequence_id": BOUNDS_SEQ_ID, + "aliases": [BOUNDS_SEQ_ID, BOUNDS_REFGET_ID], + "seq_name": f"{BOUNDS_SEQ_ID} ({BOUNDS_REFGET_ID})", + }, + { + "id": "bare-accession-coerced", + "sequence_id": "NM_000551.3", + "aliases": [BOUNDS_SEQ_ID, BOUNDS_REFGET_ID], + "seq_name": f"{BOUNDS_SEQ_ID} ({BOUNDS_REFGET_ID})", + }, + { + "id": "refget-input-named-once", + "sequence_id": BOUNDS_REFGET_ID, + "aliases": [BOUNDS_SEQ_ID, BOUNDS_REFGET_ID], + "seq_name": BOUNDS_REFGET_ID, + }, + { + "id": "no-refget-alias", + "sequence_id": BOUNDS_SEQ_ID, + "aliases": [BOUNDS_SEQ_ID], + "seq_name": BOUNDS_SEQ_ID, + }, +] + + +@pytest.mark.parametrize( + "case", location_bounds_sequence_name_cases, ids=lambda c: c["id"] +) +def test_validate_location_bounds_sequence_name(case: dict) -> None: + dp = _bounds_dp(case["aliases"]) + expected_prefix = f"Location out of bounds on {case['seq_name']}: " + with pytest.raises( + DataProxyValidationError, match=f"^{re.escape(expected_prefix)}" + ): + dp.validate_location_bounds(case["sequence_id"], 0, BOUNDS_SEQ_LEN + 1) diff --git a/tests/test_dataproxy_refget.py b/tests/test_dataproxy_refget.py new file mode 100644 index 00000000..1903e2bb --- /dev/null +++ b/tests/test_dataproxy_refget.py @@ -0,0 +1,442 @@ +"""Tests for RefgetSequenceDataProxy, CompositeDataProxy, and _extract_digest.""" + +import sys +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import MagicMock +from urllib.parse import urlparse + +import pytest + +from ga4gh.vrs.dataproxy import ( + CompositeDataProxy, + DataProxyValidationError, + create_dataproxy, +) +from ga4gh.vrs.dataproxy_refget import ( + RefgetAliasDataProxy, + RefgetSequenceDataProxy, + _create_refget_dataproxy, + _extract_digest, + _resolve_refget_cache_base_dir, + _resolve_remote_cache_dir, + _slug_remote_url, +) + +# --- _extract_digest --- + + +@pytest.mark.parametrize( + ("identifier", "expected"), + [ + ( + "ga4gh:SQ.v_QTc1p-MUYdgrRv4LMT6ByXIOsdw3C_", + "v_QTc1p-MUYdgrRv4LMT6ByXIOsdw3C_", + ), + ("SQ.v_QTc1p-MUYdgrRv4LMT6ByXIOsdw3C_", "v_QTc1p-MUYdgrRv4LMT6ByXIOsdw3C_"), + ("ga4gh:SQ.", ""), + ("SQ.", ""), + ("NM_000551.3", None), + ("refseq:NM_000551.3", None), + ("MD5:215137b1973c1a5afcf86be7d999574a", None), + ("", None), + ("ga4gh:VA.something", None), + ], +) +def test_extract_digest(identifier: str, expected: str | None) -> None: + assert _extract_digest(identifier) == expected + + +# --- Mock helpers --- + + +DIGEST = "v_QTc1p-MUYdgrRv4LMT6ByXIOsdw3C_" +GA4GH_ID = f"ga4gh:SQ.{DIGEST}" +SQ_ID = f"SQ.{DIGEST}" +SEQ = "CCTCGCCTCC" +SEQ_LENGTH = 4560 + + +def _make_mock_store( + *, has_sequence: bool = True, decode_returns: str | None = SEQ +) -> MagicMock: + """Create a mock RefgetStore with configurable behavior.""" + store = MagicMock() + if has_sequence: + metadata = SimpleNamespace(length=SEQ_LENGTH, sha512t24u=DIGEST, md5="abc123") + record = MagicMock() + record.metadata = metadata + record.decode.return_value = decode_returns + store.get_sequence.return_value = record + store.get_sequence_metadata.return_value = metadata + store.get_sequence_metadata_by_alias.return_value = metadata + store.get_substring.return_value = SEQ + else: + store.get_sequence.return_value = None + store.get_sequence_metadata.return_value = None + store.get_sequence_metadata_by_alias.return_value = None + store.get_substring.return_value = None + return store + + +def _install_fake_gtars( + monkeypatch: pytest.MonkeyPatch, + *, + local_store: MagicMock | None = None, + remote_store: MagicMock | None = None, +) -> tuple[MagicMock, MagicMock, type]: + """Install a fake gtars.refget module for factory tests.""" + local_store = local_store or MagicMock() + remote_store = remote_store or MagicMock() + + class FakeRefgetStore: + open_local = MagicMock(return_value=local_store) + open_remote = MagicMock(return_value=remote_store) + + fake_gtars = SimpleNamespace() + fake_refget = SimpleNamespace(RefgetStore=FakeRefgetStore) + monkeypatch.setitem(sys.modules, "gtars", fake_gtars) + monkeypatch.setitem(sys.modules, "gtars.refget", fake_refget) + return local_store, remote_store, FakeRefgetStore + + +# --- RefgetSequenceDataProxy.get_sequence --- + + +class TestRefgetGetSequence: + def test_both_bounds(self) -> None: + store = _make_mock_store() + dp = RefgetSequenceDataProxy(store) + result = dp.get_sequence(GA4GH_ID, 0, 10) + assert result == SEQ + store.get_substring.assert_called_once_with(DIGEST, 0, 10) + + def test_no_bounds(self) -> None: + store = _make_mock_store() + dp = RefgetSequenceDataProxy(store) + result = dp.get_sequence(GA4GH_ID) + assert result == SEQ + store.get_sequence.assert_called_once_with(DIGEST) + + def test_start_only(self) -> None: + store = _make_mock_store() + dp = RefgetSequenceDataProxy(store) + result = dp.get_sequence(GA4GH_ID, start=10) + assert result == SEQ + store.get_substring.assert_called_once_with(DIGEST, 10, SEQ_LENGTH) + + def test_end_only(self) -> None: + store = _make_mock_store() + dp = RefgetSequenceDataProxy(store) + result = dp.get_sequence(GA4GH_ID, end=50) + assert result == SEQ + store.get_substring.assert_called_once_with(DIGEST, 0, 50) + + def test_sq_prefix(self) -> None: + store = _make_mock_store() + dp = RefgetSequenceDataProxy(store) + result = dp.get_sequence(SQ_ID, 0, 10) + assert result == SEQ + store.get_substring.assert_called_once_with(DIGEST, 0, 10) + + def test_not_found_both_bounds(self) -> None: + store = _make_mock_store(has_sequence=False) + dp = RefgetSequenceDataProxy(store) + with pytest.raises(KeyError, match=GA4GH_ID): + dp.get_sequence(GA4GH_ID, 0, 10) + + def test_not_found_no_bounds(self) -> None: + store = _make_mock_store(has_sequence=False) + dp = RefgetSequenceDataProxy(store) + with pytest.raises(KeyError, match=GA4GH_ID): + dp.get_sequence(GA4GH_ID) + + def test_non_ga4gh_identifier(self) -> None: + store = _make_mock_store() + dp = RefgetSequenceDataProxy(store) + with pytest.raises(KeyError, match="NM_000551.3"): + dp.get_sequence("NM_000551.3") + + def test_decode_returns_none(self) -> None: + store = _make_mock_store(decode_returns=None) + dp = RefgetSequenceDataProxy(store) + with pytest.raises(KeyError, match=GA4GH_ID): + dp.get_sequence(GA4GH_ID) + + def test_negative_start_clamped(self) -> None: + """Negative start is clamped to 0 (matching Python slice semantics).""" + store = _make_mock_store() + dp = RefgetSequenceDataProxy(store) + result = dp.get_sequence(GA4GH_ID, -10, 5) + assert result == SEQ + store.get_substring.assert_called_once_with(DIGEST, 0, 5) + + def test_empty_interval_returns_empty(self) -> None: + """A zero-width interval, before or after clamping, is the empty string.""" + store = _make_mock_store() + dp = RefgetSequenceDataProxy(store) + assert dp.get_sequence(GA4GH_ID, 10, 10) == "" + assert dp.get_sequence(GA4GH_ID, -5, 0) == "" + store.get_substring.assert_not_called() + + def test_start_after_end_rejected(self) -> None: + """A start after the end is rejected before the store is consulted.""" + store = _make_mock_store() + dp = RefgetSequenceDataProxy(store) + with pytest.raises(DataProxyValidationError, match="start=10, end=5"): + dp.get_sequence(GA4GH_ID, 10, 5) + store.get_substring.assert_not_called() + store.get_sequence_metadata.assert_not_called() + + +# --- RefgetSequenceDataProxy.get_sequence_length --- + + +class TestRefgetGetSequenceLength: + def test_found(self) -> None: + store = _make_mock_store() + dp = RefgetSequenceDataProxy(store) + assert dp.get_sequence_length(GA4GH_ID) == SEQ_LENGTH + store.get_sequence_metadata.assert_called_once_with(DIGEST) + store.get_sequence.assert_not_called() + + def test_not_found(self) -> None: + store = _make_mock_store(has_sequence=False) + dp = RefgetSequenceDataProxy(store) + with pytest.raises(KeyError, match=GA4GH_ID): + dp.get_sequence_length(GA4GH_ID) + + def test_non_ga4gh_identifier(self) -> None: + store = _make_mock_store() + dp = RefgetSequenceDataProxy(store) + with pytest.raises(KeyError, match="NM_000551.3"): + dp.get_sequence_length("NM_000551.3") + + +# --- RefgetAliasDataProxy --- + + +class TestRefgetAliasDataProxy: + def test_resolve_alias_uses_metadata_only(self) -> None: + store = _make_mock_store() + dp = RefgetAliasDataProxy(store) + + assert dp._resolve_to_digest("refseq:NM_000551.3") == DIGEST + store.get_sequence_metadata_by_alias.assert_called_once_with( + "refseq", "NM_000551.3" + ) + store.get_sequence_by_alias.assert_not_called() + + def test_derive_refget_accession_skips_reverse_alias_scan(self) -> None: + store = _make_mock_store() + dp = RefgetAliasDataProxy(store) + + assert dp.derive_refget_accession("refseq:NM_000551.3") == SQ_ID + store.get_aliases_for_sequence.assert_not_called() + + @pytest.mark.parametrize( + ("bare", "namespace"), + [("NM_000551.3", "refseq"), ("LRG_1", "lrg"), ("LRG_1t1", "lrg")], + ) + def test_bare_accession_namespace_is_inferred( + self, bare: str, namespace: str + ) -> None: + store = _make_mock_store() + dp = RefgetAliasDataProxy(store) + + assert dp.derive_refget_accession(bare) == SQ_ID + store.get_sequence_metadata_by_alias.assert_called_once_with(namespace, bare) + + def test_bare_accession_with_no_inferable_namespace_is_a_miss(self) -> None: + store = _make_mock_store() + dp = RefgetAliasDataProxy(store) + + assert dp.derive_refget_accession("NOT_AN_ACCESSION") is None + store.get_sequence_metadata_by_alias.assert_not_called() + + +# --- CompositeDataProxy --- + + +def _make_composite( + *, + seq_has_sequence: bool = True, +) -> tuple[CompositeDataProxy, MagicMock, MagicMock]: + """Create a CompositeDataProxy with mock backends.""" + seq_store = _make_mock_store(has_sequence=seq_has_sequence) + seq_proxy = RefgetSequenceDataProxy(seq_store) + + alias_proxy = MagicMock() + alias_proxy.get_aliases.return_value = [ + "refseq:NM_000551.3", + GA4GH_ID, + "MD5:abc123", + ] + alias_proxy.translate_sequence_identifier.return_value = [GA4GH_ID] + alias_proxy.derive_refget_accession.return_value = SQ_ID + + composite = CompositeDataProxy(seq_proxy, alias_proxy) + return composite, seq_store, alias_proxy + + +class TestCompositeGetSequence: + def test_direct_ga4gh_hit(self) -> None: + """GA4GH identifier succeeds on first try, no alias lookup.""" + composite, seq_store, alias_proxy = _make_composite() + result = composite.get_sequence(GA4GH_ID, 0, 10) + assert result == SEQ + alias_proxy.translate_sequence_identifier.assert_not_called() + + def test_alias_fallback(self) -> None: + """Non-GA4GH identifier triggers alias resolution then succeeds.""" + composite, seq_store, alias_proxy = _make_composite() + result = composite.get_sequence("NM_000551.3", 0, 10) + assert result == SEQ + alias_proxy.derive_refget_accession.assert_called_once_with("NM_000551.3") + + def test_alias_not_found(self) -> None: + """Non-GA4GH identifier with no alias raises KeyError.""" + composite, _, alias_proxy = _make_composite() + alias_proxy.derive_refget_accession.return_value = None + with pytest.raises(KeyError, match="NM_999999.1"): + composite.get_sequence("NM_999999.1", 0, 10) + + +class TestCompositeGetSequenceLength: + def test_direct_ga4gh_hit(self) -> None: + composite, _, alias_proxy = _make_composite() + assert composite.get_sequence_length(GA4GH_ID) == SEQ_LENGTH + alias_proxy.translate_sequence_identifier.assert_not_called() + + def test_alias_fallback(self) -> None: + composite, _, alias_proxy = _make_composite() + assert composite.get_sequence_length("NM_000551.3") == SEQ_LENGTH + alias_proxy.derive_refget_accession.assert_called_once_with("NM_000551.3") + + +class TestCompositeGetAliases: + def test_delegates_to_alias_proxy(self) -> None: + composite, _, alias_proxy = _make_composite() + aliases = composite.get_aliases("NM_000551.3") + assert GA4GH_ID in aliases + alias_proxy.get_aliases.assert_called_once_with("NM_000551.3") + + def test_derive_refget_accession_delegates_to_alias_proxy(self) -> None: + composite, _, alias_proxy = _make_composite() + + assert composite.derive_refget_accession("NM_000551.3") == SQ_ID + alias_proxy.derive_refget_accession.assert_called_once_with("NM_000551.3") + alias_proxy.get_aliases.assert_not_called() + + +class TestRefgetRemoteCacheHelpers: + def test_slug_remote_url_is_stable(self) -> None: + url = "https://example.org/refget-store" + assert _slug_remote_url(url) == _slug_remote_url(url) + + def test_slug_remote_url_distinguishes_urls(self) -> None: + assert _slug_remote_url("https://example.org/store-a") != _slug_remote_url( + "https://example.org/store-b" + ) + + def test_resolve_cache_base_uses_env_override( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + cache_root = tmp_path / "custom-cache" + monkeypatch.setenv("GA4GH_VRS_REFGET_CACHE_DIR", str(cache_root)) + assert _resolve_refget_cache_base_dir() == cache_root + + def test_resolve_cache_base_uses_xdg( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + monkeypatch.delenv("GA4GH_VRS_REFGET_CACHE_DIR", raising=False) + monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path / "xdg")) + assert _resolve_refget_cache_base_dir() == tmp_path / "xdg" / "gtars" / "refget" + + def test_resolve_cache_base_falls_back_to_home( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + monkeypatch.delenv("GA4GH_VRS_REFGET_CACHE_DIR", raising=False) + monkeypatch.delenv("XDG_CACHE_HOME", raising=False) + monkeypatch.setenv("HOME", str(tmp_path / "home")) + assert ( + _resolve_refget_cache_base_dir() + == tmp_path / "home" / ".cache" / "gtars" / "refget" + ) + + def test_resolve_remote_cache_dir_uses_base_plus_slug( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + cache_root = tmp_path / "cache-root" + remote_url = "https://example.org/refget-store" + monkeypatch.setenv("GA4GH_VRS_REFGET_CACHE_DIR", str(cache_root)) + assert _resolve_remote_cache_dir(remote_url) == cache_root / _slug_remote_url( + remote_url + ) + + +class TestCreateRefgetDataProxyFactory: + def test_file_uri_uses_open_local( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + local_store, _, fake_refget_store = _install_fake_gtars(monkeypatch) + parsed_uri = urlparse(f"refget+file://{tmp_path}/store") + + proxy = _create_refget_dataproxy(parsed_uri, "file") + + assert isinstance(proxy, CompositeDataProxy) + fake_refget_store.open_local.assert_called_once_with(parsed_uri.path) + fake_refget_store.open_remote.assert_not_called() + local_store.set_quiet.assert_called_once_with(True) + local_store.pull_aliases.assert_called_once_with() + + def test_http_uri_uses_open_remote_with_env_cache_root( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + _, remote_store, fake_refget_store = _install_fake_gtars(monkeypatch) + cache_root = tmp_path / "cache-root" + remote_url = "http://example.org:8080/refget-store" + parsed_uri = urlparse(f"refget+{remote_url}") + monkeypatch.setenv("GA4GH_VRS_REFGET_CACHE_DIR", str(cache_root)) + + proxy = _create_refget_dataproxy(parsed_uri, "http") + + assert isinstance(proxy, CompositeDataProxy) + expected_cache_dir = cache_root / _slug_remote_url(remote_url) + fake_refget_store.open_local.assert_not_called() + fake_refget_store.open_remote.assert_called_once_with( + str(expected_cache_dir), remote_url + ) + assert expected_cache_dir.is_dir() + remote_store.get_sequence_metadata.assert_called_once_with("") + remote_store.pull_aliases.assert_called_once_with() + + def test_https_uri_dispatches_via_create_dataproxy( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + _, remote_store, fake_refget_store = _install_fake_gtars(monkeypatch) + cache_root = tmp_path / "cache-root" + remote_url = "https://example.org/refget-store" + monkeypatch.setenv("GA4GH_VRS_REFGET_CACHE_DIR", str(cache_root)) + + proxy = create_dataproxy(f"refget+{remote_url}") + + assert isinstance(proxy, CompositeDataProxy) + fake_refget_store.open_remote.assert_called_once_with( + str(cache_root / _slug_remote_url(remote_url)), + remote_url, + ) + remote_store.pull_aliases.assert_called_once_with() + + def test_local_refget_uri_ignores_cache_env_var( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + local_store, _, fake_refget_store = _install_fake_gtars(monkeypatch) + monkeypatch.setenv("GA4GH_VRS_REFGET_CACHE_DIR", str(tmp_path / "cache-root")) + + proxy = create_dataproxy(f"refget+file://{tmp_path}/store") + + assert isinstance(proxy, CompositeDataProxy) + fake_refget_store.open_local.assert_called_once() + fake_refget_store.open_remote.assert_not_called() + local_store.pull_aliases.assert_called_once_with() diff --git a/tests/test_dataproxy_refget_store.py b/tests/test_dataproxy_refget_store.py new file mode 100644 index 00000000..63259392 --- /dev/null +++ b/tests/test_dataproxy_refget_store.py @@ -0,0 +1,106 @@ +"""Refget data proxies over a real gtars RefgetStore, opened locally and over HTTP. + +The store is built from the sequences and aliases of the SeqRepo in +``tests/data/seqrepo/latest``, so both backends serve the same data. +""" + +import shutil +import sqlite3 +import subprocess +import sys +from collections.abc import Iterator +from pathlib import Path + +import pytest + +from ga4gh.vrs.dataproxy import DataProxy, create_dataproxy + +SEQREPO_DIR = Path(__file__).parent / "data" / "seqrepo" / "latest" +# SeqRepo alias namespace -> RefgetStore alias namespace +NAMESPACES = {"NCBI": "refseq", "GRCh38": "GRCh38"} +VHL_DIGEST = "v_QTc1p-MUYdgrRv4LMT6ByXIOsdw3C_" # NM_000551.3 + + +@pytest.fixture(scope="session") +def refget_store_dir(tmp_path_factory: pytest.TempPathFactory) -> Path: + refget = pytest.importorskip("gtars.refget") + work_dir = tmp_path_factory.mktemp("refget") + store_dir = work_dir / "store" + store = refget.RefgetStore.on_disk(str(store_dir)) + store.set_quiet(True) + # SeqRepo names each FASTA record by its sha512t24u digest. Import copies: + # gtars writes a digest index next to each FASTA it reads. + for fasta in sorted((SEQREPO_DIR / "sequences").rglob("*.fa.bgz")): + copy = work_dir / fasta.name + shutil.copyfile(fasta, copy) + store.add_sequence_collection_from_fasta(str(copy)) + digests = {metadata.sha512t24u for metadata in store.list_sequences()} + with sqlite3.connect(SEQREPO_DIR / "aliases.sqlite3") as aliases: + for seq_id, namespace, alias in aliases.execute( + "SELECT seq_id, namespace, alias FROM seqalias WHERE is_current = 1" + ): + if namespace in NAMESPACES and seq_id in digests: + store.add_sequence_alias(NAMESPACES[namespace], alias, seq_id) + store.write() + return store_dir + + +@pytest.fixture(scope="session") +def refget_store_url(refget_store_dir: Path) -> Iterator[str]: + # A separate process: gtars holds the GIL during its HTTP requests, so an + # in-process server thread could never answer them + server = subprocess.Popen( # noqa: S603 + [ + sys.executable, + "-u", + "-m", + "http.server", + "0", + "--bind", + "127.0.0.1", + "--directory", + str(refget_store_dir), + ], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + text=True, + ) + try: + # "Serving HTTP on 127.0.0.1 port 49482 (http://127.0.0.1:49482/) ..." + port = int(server.stdout.readline().split(" port ")[1].split()[0]) + yield f"http://127.0.0.1:{port}" + finally: + server.terminate() + server.wait() + + +@pytest.fixture(params=["local", "remote"]) +def refget_dataproxy( + request: pytest.FixtureRequest, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + refget_store_dir: Path, +) -> DataProxy: + if request.param == "local": + return create_dataproxy(f"refget+file://{refget_store_dir}") + monkeypatch.setenv("GA4GH_VRS_REFGET_CACHE_DIR", str(tmp_path / "cache")) + return create_dataproxy(f"refget+{request.getfixturevalue('refget_store_url')}") + + +@pytest.mark.parametrize( + "identifier", ["NM_000551.3", "refseq:NM_000551.3", f"ga4gh:SQ.{VHL_DIGEST}"] +) +def test_resolves_identifier(refget_dataproxy: DataProxy, identifier: str) -> None: + assert refget_dataproxy.derive_refget_accession(identifier) == f"SQ.{VHL_DIGEST}" + + +def test_matches_seqrepo(refget_dataproxy: DataProxy) -> None: + seqrepo_dataproxy = create_dataproxy(f"seqrepo+file://{SEQREPO_DIR}") + for identifier in ("NM_000551.3", "NC_000019.10", "GRCh38:chr19"): + assert refget_dataproxy.get_sequence_length(identifier) == ( + seqrepo_dataproxy.get_sequence_length(identifier) + ) + assert refget_dataproxy.get_sequence(identifier, 1000, 1100) == ( + seqrepo_dataproxy.get_sequence(identifier, 1000, 1100) + ) + assert "refseq:NM_000551.3" in refget_dataproxy.get_aliases("NM_000551.3") diff --git a/tests/test_vrs_normalize.py b/tests/test_vrs_normalize.py index 83c27cdd..dc094960 100644 --- a/tests/test_vrs_normalize.py +++ b/tests/test_vrs_normalize.py @@ -1,6 +1,10 @@ +import re + import pytest from ga4gh.vrs import models, normalize +from ga4gh.vrs.dataproxy import DataProxyValidationError +from ga4gh.vrs.dataproxy_seqrepo import SeqRepoDataProxy # Single nucleotide same-as-reference allele. allele_dict1 = { @@ -929,3 +933,76 @@ def test_normalize_partial_rle_del_ins(rest_dataproxy): tail_del_4 = models.Allele(**tail_del_4bp) tail_del_4_norm = normalize(tail_del_4, rest_dataproxy, rle_seq_limit=0) assert tail_del_4_norm == models.Allele(**tail_del_4bp_normalized) + + +# NM_000551.3, length 4560. Present in the test SeqRepo, so these use the local +# dataproxy fixture and need no cassettes. +BOUNDS_REFGET_AC = "SQ.v_QTc1p-MUYdgrRv4LMT6ByXIOsdw3C_" +BOUNDS_SEQ_LEN = 4560 + + +def _bounds_allele( + start: int | list[int | None], end: int | list[int | None], sequence: str +) -> models.Allele: + return models.Allele( + location=models.SequenceLocation( + sequenceReference=models.SequenceReference( + refgetAccession=BOUNDS_REFGET_AC + ), + start=start, + end=end, + ), + state=models.LiteralSequenceExpression(sequence=sequence), + ) + + +normalize_location_in_bounds_cases = [ + # an undefined outer endpoint is representable and must not be rejected + # (the deletion is also rolled right by one base by normalization) + { + "id": "indefinite-ranges-open-outward", + "start": [None, 4400], + "end": [4500, None], + "sequence": "", + "expected_start": [None, 4400], + "expected_end": [4501, None], + }, +] + + +@pytest.mark.parametrize( + "case", normalize_location_in_bounds_cases, ids=lambda c: c["id"] +) +def test_normalize_location_in_bounds(dataproxy: SeqRepoDataProxy, case: dict) -> None: + allele = _bounds_allele(case["start"], case["end"], case["sequence"]) + location = normalize(allele, dataproxy).location.model_dump() + assert (location["start"], location["end"]) == ( + case["expected_start"], + case["expected_end"], + ) + + +normalize_location_out_of_bounds_cases = [ + # Definite ranges are otherwise returned without normalization, so the + # bounds check must run before that early return + { + "id": "definite-range-end-past-end", + "start": 4400, + "end": [4500, 4600], + "detail": "end=[4500, 4600]", + }, +] + + +@pytest.mark.parametrize( + "case", normalize_location_out_of_bounds_cases, ids=lambda c: c["id"] +) +def test_normalize_location_out_of_bounds( + dataproxy: SeqRepoDataProxy, case: dict +) -> None: + expected_msg = ( + f"Location out of bounds on ga4gh:{BOUNDS_REFGET_AC}: {case['detail']} " + f"not within [0, {BOUNDS_SEQ_LEN}]" + ) + with pytest.raises(DataProxyValidationError, match=f"^{re.escape(expected_msg)}$"): + normalize(_bounds_allele(case["start"], case["end"], "A"), dataproxy)