From cd9450c5ce48044ed308d7ae52a10155c0d92a05 Mon Sep 17 00:00:00 2001 From: David Reinhart Date: Tue, 7 Jul 2026 11:17:34 -0700 Subject: [PATCH 001/252] feat(uniprot): support Ensembl protein accessions in UniProt ID mapping infer_db_name_from_sequence_accession now recognizes ENSP accessions and maps them to the Ensembl_Protein source, so accession-based target genes with Ensembl protein IDs resolve to a UniProt ID instead of raising NotImplementedError. Also make validate_ensembl_identifier accept versioned Ensembl stable IDs (e.g. ENSP00000369497.3) by stripping the trailing . before the idutils.is_ensembl check, matching how validate_refseq_identifier already handles versioned RefSeq accessions. Related to varianteffect/dcd_mapping2#110 --- src/mavedb/lib/uniprot/utils.py | 14 +++++++++++--- src/mavedb/lib/validation/identifier.py | 8 +++++++- tests/lib/uniprot/test_utils.py | 8 ++++++++ tests/validation/test_identifier.py | 4 ++++ 4 files changed, 30 insertions(+), 4 deletions(-) diff --git a/src/mavedb/lib/uniprot/utils.py b/src/mavedb/lib/uniprot/utils.py index 7f87524b2..11c13a8f4 100644 --- a/src/mavedb/lib/uniprot/utils.py +++ b/src/mavedb/lib/uniprot/utils.py @@ -1,11 +1,11 @@ from typing import Literal, Union -from mavedb.lib.validation.identifier import validate_refseq_identifier +from mavedb.lib.validation.identifier import validate_ensembl_identifier, validate_refseq_identifier def infer_db_name_from_sequence_accession( sequence_accession: str, -) -> Union[Literal["RefSeq_Nucleotide", "RefSeq_Protein"]]: +) -> Union[Literal["RefSeq_Nucleotide", "RefSeq_Protein", "Ensembl_Protein"]]: """ Infers the database name from a sequence accession. @@ -15,6 +15,12 @@ def infer_db_name_from_sequence_accession( Returns: str: The inferred database name. """ + # Ensembl protein accessions must be handled before validate_refseq_identifier, + # which raises on any non-RefSeq accession. + if sequence_accession.startswith("ENSP"): + validate_ensembl_identifier(sequence_accession) + return "Ensembl_Protein" + validate_refseq_identifier(sequence_accession) if sequence_accession.startswith("NM_"): @@ -22,4 +28,6 @@ def infer_db_name_from_sequence_accession( if sequence_accession.startswith("NP_"): return "RefSeq_Protein" - raise NotImplementedError("Only RefSeq NM and NP identifiers are currently supported for inference.") + raise NotImplementedError( + "Only RefSeq NM/NP and Ensembl protein (ENSP) identifiers are currently supported for inference." + ) diff --git a/src/mavedb/lib/validation/identifier.py b/src/mavedb/lib/validation/identifier.py index e6e2dcde7..077b13fc0 100644 --- a/src/mavedb/lib/validation/identifier.py +++ b/src/mavedb/lib/validation/identifier.py @@ -1,3 +1,5 @@ +import re + import idutils from mavedb.lib.validation.constants.identifier import valid_dbnames @@ -52,6 +54,9 @@ def validate_ensembl_identifier(identifier: str): """ Validates whether the identifier is a valid Ensembl identifier. + Ensembl stable IDs may carry a version suffix (e.g. ``ENSP00000369497.3``), which + ``idutils.is_ensembl`` does not accept; strip it before validating the stable ID. + Parameters __________ identifier : str @@ -62,7 +67,8 @@ def validate_ensembl_identifier(identifier: str): ValidationError If the identifier is not a valid Ensembl identifier. """ - if not idutils.is_ensembl(identifier): + base_identifier = re.sub(r"\.\d+$", "", identifier) + if not idutils.is_ensembl(base_identifier): raise ValidationError(f"'{identifier}' is not a valid Ensembl accession.") diff --git a/tests/lib/uniprot/test_utils.py b/tests/lib/uniprot/test_utils.py index 7afa278c2..5ac773e38 100644 --- a/tests/lib/uniprot/test_utils.py +++ b/tests/lib/uniprot/test_utils.py @@ -21,6 +21,14 @@ def test_infer_db_name_from_sequence_accession_np(): assert result == "RefSeq_Protein" +# Both versioned and unversioned ENSP accessions must route to Ensembl_Protein; the versioned case +# guards that validate_ensembl_identifier's version handling continues to accept it. +@pytest.mark.parametrize("ensembl_protein_accession", ["ENSP00000418960", "ENSP00000418960.3"]) +def test_infer_db_name_from_sequence_accession_ensp(ensembl_protein_accession): + result = infer_db_name_from_sequence_accession(ensembl_protein_accession) + assert result == "Ensembl_Protein" + + @pytest.mark.parametrize("invalid_accession", ["XP_000000", VALID_CHR_ACCESSION]) def test_infer_db_name_from_sequence_accession_invalid(invalid_accession): with pytest.raises(NotImplementedError): diff --git a/tests/validation/test_identifier.py b/tests/validation/test_identifier.py index 005278307..9cbd16b0d 100644 --- a/tests/validation/test_identifier.py +++ b/tests/validation/test_identifier.py @@ -63,6 +63,10 @@ def test_ve_invalid_list(self): def test_passes_valid_id(self): validate_ensembl_identifier("ENSG00000143384") + def test_passes_valid_versioned_id(self): + validate_ensembl_identifier("ENSP00000369497.3") + validate_ensembl_list(["ENSG00000139618.15", "ENST00000380152.8"]) + class TestRefSeqValidators(TestCase): """ From f44a9a131bd98a599bb01be773f8e374a1df05e5 Mon Sep 17 00:00:00 2001 From: David Reinhart Date: Wed, 8 Jul 2026 11:49:19 -0700 Subject: [PATCH 002/252] Add Ensembl transcript (ENST) support to sequence accession inference Extend infer_db_name_from_sequence_accession to recognize ENST identifiers and return "Ensembl_Transcript" as the database name. Restructure control flow to validate each accession type only within its matching branch rather than calling validate_refseq_identifier unconditionally for non-Ensembl accessions. --- src/mavedb/lib/uniprot/utils.py | 18 +++++++++--------- tests/lib/uniprot/test_utils.py | 18 +++++++++++------- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/src/mavedb/lib/uniprot/utils.py b/src/mavedb/lib/uniprot/utils.py index 11c13a8f4..ec5a8a185 100644 --- a/src/mavedb/lib/uniprot/utils.py +++ b/src/mavedb/lib/uniprot/utils.py @@ -5,7 +5,7 @@ def infer_db_name_from_sequence_accession( sequence_accession: str, -) -> Union[Literal["RefSeq_Nucleotide", "RefSeq_Protein", "Ensembl_Protein"]]: +) -> Union[Literal["RefSeq_Nucleotide", "RefSeq_Protein", "Ensembl_Protein", "Ensembl_Transcript"]]: """ Infers the database name from a sequence accession. @@ -15,19 +15,19 @@ def infer_db_name_from_sequence_accession( Returns: str: The inferred database name. """ - # Ensembl protein accessions must be handled before validate_refseq_identifier, - # which raises on any non-RefSeq accession. if sequence_accession.startswith("ENSP"): validate_ensembl_identifier(sequence_accession) return "Ensembl_Protein" - - validate_refseq_identifier(sequence_accession) - - if sequence_accession.startswith("NM_"): + elif sequence_accession.startswith("ENST"): + validate_ensembl_identifier(sequence_accession) + return "Ensembl_Transcript" + elif sequence_accession.startswith("NM_"): + validate_refseq_identifier(sequence_accession) return "RefSeq_Nucleotide" - if sequence_accession.startswith("NP_"): + elif sequence_accession.startswith("NP_"): + validate_refseq_identifier(sequence_accession) return "RefSeq_Protein" raise NotImplementedError( - "Only RefSeq NM/NP and Ensembl protein (ENSP) identifiers are currently supported for inference." + "Only RefSeq (NM_/NP_) and Ensembl (ENSP/ENST) identifiers are currently supported for inference." ) diff --git a/tests/lib/uniprot/test_utils.py b/tests/lib/uniprot/test_utils.py index 5ac773e38..5d51c2afe 100644 --- a/tests/lib/uniprot/test_utils.py +++ b/tests/lib/uniprot/test_utils.py @@ -1,12 +1,10 @@ import pytest -from mavedb.lib.validation.exceptions import ValidationError from mavedb.lib.uniprot.utils import infer_db_name_from_sequence_accession - from tests.helpers.constants import ( + VALID_CHR_ACCESSION, VALID_NT_ACCESSION, VALID_PRO_ACCESSION, - VALID_CHR_ACCESSION, VALID_UNIPROT_ACCESSION, ) @@ -21,14 +19,20 @@ def test_infer_db_name_from_sequence_accession_np(): assert result == "RefSeq_Protein" -# Both versioned and unversioned ENSP accessions must route to Ensembl_Protein; the versioned case -# guards that validate_ensembl_identifier's version handling continues to accept it. +# Both versioned and unversioned ENSP and ENST accessions must route to Ensembl_Protein and Ensembl_Transcript, respectively; +# the versioned case guards that validate_ensembl_identifier's version handling continues to accept it. @pytest.mark.parametrize("ensembl_protein_accession", ["ENSP00000418960", "ENSP00000418960.3"]) def test_infer_db_name_from_sequence_accession_ensp(ensembl_protein_accession): result = infer_db_name_from_sequence_accession(ensembl_protein_accession) assert result == "Ensembl_Protein" +@pytest.mark.parametrize("ensembl_transcript_accession", ["ENST00000418960", "ENST00000418960.3"]) +def test_infer_db_name_from_sequence_accession_enst(ensembl_transcript_accession): + result = infer_db_name_from_sequence_accession(ensembl_transcript_accession) + assert result == "Ensembl_Transcript" + + @pytest.mark.parametrize("invalid_accession", ["XP_000000", VALID_CHR_ACCESSION]) def test_infer_db_name_from_sequence_accession_invalid(invalid_accession): with pytest.raises(NotImplementedError): @@ -36,10 +40,10 @@ def test_infer_db_name_from_sequence_accession_invalid(invalid_accession): def test_infer_db_name_from_non_refseq_accession(): - with pytest.raises(ValidationError): + with pytest.raises(NotImplementedError): infer_db_name_from_sequence_accession(VALID_UNIPROT_ACCESSION) def test_infer_db_name_from_sequence_accession_empty_string(): - with pytest.raises(ValidationError): + with pytest.raises(NotImplementedError): infer_db_name_from_sequence_accession("") From 4e10b20a563d9c2683118c6875e2d4c966a5fce1 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Thu, 9 Jul 2026 10:35:24 -0700 Subject: [PATCH 003/252] chore(export): add changelog to public data archive and document release process MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add CHANGELOG.md to scripts/resources, bundled into every generated archive so each downloaded version carries its own history; documents versions 1–4 (pre-changelog) and v5 in detail - Update export script to include CHANGELOG.md alongside the README and license in the zip output - Add a release-procedure note to the module docstring pointing maintainers to the changelog and deployment docs before each Zenodo publish --- src/mavedb/scripts/export_public_data.py | 7 ++- src/mavedb/scripts/resources/CHANGELOG.md | 62 +++++++++++++++++++++++ src/mavedb/scripts/resources/README.md | 1 + 3 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 src/mavedb/scripts/resources/CHANGELOG.md diff --git a/src/mavedb/scripts/export_public_data.py b/src/mavedb/scripts/export_public_data.py index 4ced338a8..cedc66608 100644 --- a/src/mavedb/scripts/export_public_data.py +++ b/src/mavedb/scripts/export_public_data.py @@ -11,6 +11,10 @@ Unpublished data and data sets licensed other than under the Creative Commons Zero license are not included in the dump, and user details are limited to ORCID IDs and names of contributors to published data sets. + +RELEASING A NEW VERSION: Before publishing a new version of this archive to Zenodo, add an entry describing what +changed to `src/mavedb/scripts/resources/CHANGELOG.md` (it is bundled into the archive). The full release procedure +is documented in `deployment/docs/zenodo-release.md`. """ import json @@ -129,10 +133,11 @@ def export_public_data(db: Session): # Write metadata for all data sets to a single JSON file. zipfile.writestr("main.json", json.dumps(jsonable_encoder(json_data))) - # Copy the CC0 license and README. + # Copy the CC0 license, README, and changelog. resources_dir = os.path.join(os.path.dirname(__file__), "resources") zipfile.write(os.path.join(resources_dir, "CC0_license.txt"), "LICENSE.txt") zipfile.write(os.path.join(resources_dir, "README.md"), "README.md") + zipfile.write(os.path.join(resources_dir, "CHANGELOG.md"), "CHANGELOG.md") # Write score and count files for each score set. num_score_sets = len(score_set_ids) diff --git a/src/mavedb/scripts/resources/CHANGELOG.md b/src/mavedb/scripts/resources/CHANGELOG.md new file mode 100644 index 000000000..3f441fc0c --- /dev/null +++ b/src/mavedb/scripts/resources/CHANGELOG.md @@ -0,0 +1,62 @@ +# MaveDB Public Data Dump — Changelog + +This file records what changed between versions of the MaveDB bulk data archive +published on [Zenodo](https://doi.org/10.5281/zenodo.11201736). It ships inside +every archive so the version you downloaded always carries its own history. + +Versions are the Zenodo archive versions (v1, v2, …), not MaveDB API or website +release numbers. The `asOf` timestamp in `main.json` records the exact moment a +given archive was generated. + +--- + +## [5] — 2026-06-25 + +Version 5 is the first version tracked in this changelog. Versions 1–4 contained +metadata (`main.json`) plus per-score-set score and count CSVs; they predate this +file and are not itemized here. Version 5 adds variant-mapping and annotation +outputs derived from the MaveDB variant mapping pipeline. + +### Added + +- **Annotation CSVs** — `csv/{urn}.annotations.csv` for every score set that has + completed the mapping pipeline. Joins external-database annotations (Ensembl VEP + functional consequence, gnomAD v4.1 allele frequency, ClinGen Allele Registry ID) + with post-mapped HGVS (`g.`/`c.`/`p.` and assay-level) and the GA4GH VRS digest. +- **Mapped-variant JSON** — `mapped/{urn}.mapped-variants.json` for every mapped + score set. Each record carries the pre-mapped and post-mapped VRS alleles, the + VRS schema version, the mapping API version, and the ClinGen allele ID, mirroring + `GET /api/v1/score-sets/{urn}/mapped-variants`. +- **VA-Spec annotation NDJSON** — `va/{urn}.va.ndjson` for every mapped score set. + One line per current mapped variant carrying its highest materialized GA4GH + VA-Spec layer (study result → functional-impact statement → pathogenicity + statement), mirroring the `annotated-variants` streaming endpoints. +- **ClinVar release columns** in the annotation CSVs, covering the `2015_02` + through `2026_01` annual snapshots. + +### Changed + +- **CSV columns are now namespaced.** Every column carries a source prefix + separated by a dot: score columns are `scores.*` (e.g. `scores.score`), counts + are `counts.*`, mapping-pipeline columns are `mavedb.*`, and external annotations + are `vep.*` / `gnomad.*` / `clingen.*`. Core identifier columns (`accession`, + `hgvs_nt`, `hgvs_pro`, `hgvs_splice`) remain unprefixed. **Consumers that parsed + earlier archives by bare column name (e.g. `score`) must update to the namespaced + names.** +- **Cis-phased (haplotype) variants** are now combined into a single HGVS + expression in the CSV outputs where a combined expression can be derived. +- Mapped-variant and VA-Spec outputs include only the **current** mapping for each + variant; superseded mappings are excluded. + +### Fixed + +- VA-Spec annotation output is now valid and round-trippable (it deserializes back + into GA4GH VA-Spec objects without loss). + +### Notes + +- Annotation, mapped-variant, and VA-Spec files are present **only** for score sets + that have completed the mapping pipeline. Unmapped score sets ship scores/counts + CSVs only, as before. +- See `README.md` in this archive for the full file layout, column definitions, and + caveats. diff --git a/src/mavedb/scripts/resources/README.md b/src/mavedb/scripts/resources/README.md index 31ec4e26c..44b8eadf2 100644 --- a/src/mavedb/scripts/resources/README.md +++ b/src/mavedb/scripts/resources/README.md @@ -30,6 +30,7 @@ Unpublished data, private datasets, and datasets published under other licenses ``` mavedb-dump.YYYYMMDDHHMMSS.zip ├── README.md # This file +├── CHANGELOG.md # What changed between archive versions ├── LICENSE.txt # Creative Commons CC0 1.0 license text ├── main.json # Metadata for all included datasets ├── csv/ From 9f10350c36dd291e60348c072edbe4ac3a703d8a Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Thu, 9 Jul 2026 11:23:16 -0700 Subject: [PATCH 004/252] feat(worker): detect stuck RUNNING jobs via progress heartbeat Replace the pure wall-clock RUNNING-stall check with a progress heartbeat, so a healthy long-running job (e.g. large VEP fan-out) is no longer killed while a genuinely wedged one is still caught. Make the DB-driven sweeper authoritative over termination and keep ARQ's timeout only as a last resort. - Add JobRun.progress_updated_at (onupdate=now()) as a zero-code liveness heartbeat, bumped on every committing progress checkpoint; index (status, progress_updated_at) for the sweeper query - Detect stalled RUNNING jobs on stale heartbeat (PROGRESS_STALL_MINUTES = 20) OR a wall-clock backstop (RUNNING_TIMEOUT_MINUTES = 450), and recover them abort-first: abort the current attempt and only re-drive once it is confirmed dead or absent, eliminating the double-run where a retry ran alongside the still-live original - Handle asyncio.CancelledError in with_job_management: mark the row FAILED/TIMEOUT then re-raise, so an ARQ timeout or a stall-recovery abort no longer orphans the row as RUNNING and Job.abort() can confirm - Raise ARQ job_timeout to 8h and enable allow_abort_jobs; tighten the cleanup cron to every 10 minutes - Cover new behavior with unit and integration tests --- ...104_add_progress_updated_at_to_job_runs.py | 44 +++++ src/mavedb/models/job_run.py | 10 ++ src/mavedb/worker/jobs/registry.py | 6 +- src/mavedb/worker/jobs/system/cleanup.py | 140 +++++++++++++-- .../worker/lib/decorators/job_management.py | 40 ++++- src/mavedb/worker/settings/worker.py | 12 +- tests/worker/jobs/system/test_cleanup.py | 167 +++++++++++++++++- .../lib/decorators/test_job_management.py | 63 ++++++- tests/worker/lib/managers/test_job_manager.py | 23 +++ 9 files changed, 472 insertions(+), 33 deletions(-) create mode 100644 alembic/versions/a7f3c2e9b104_add_progress_updated_at_to_job_runs.py diff --git a/alembic/versions/a7f3c2e9b104_add_progress_updated_at_to_job_runs.py b/alembic/versions/a7f3c2e9b104_add_progress_updated_at_to_job_runs.py new file mode 100644 index 000000000..d04ceab9b --- /dev/null +++ b/alembic/versions/a7f3c2e9b104_add_progress_updated_at_to_job_runs.py @@ -0,0 +1,44 @@ +"""Add progress_updated_at heartbeat column to job_runs + +Revision ID: a7f3c2e9b104 +Revises: 4dbf24ed1857 +Create Date: 2026-07-09 00:00:00.000000 + +""" + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "a7f3c2e9b104" +down_revision = "4dbf24ed1857" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column( + "job_runs", + sa.Column( + "progress_updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + ) + op.create_index( + "ix_job_runs_status_progress_updated", + "job_runs", + ["status", "progress_updated_at"], + unique=False, + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index("ix_job_runs_status_progress_updated", table_name="job_runs") + op.drop_column("job_runs", "progress_updated_at") + # ### end Alembic commands ### diff --git a/src/mavedb/models/job_run.py b/src/mavedb/models/job_run.py index 877eeab02..b079a1757 100644 --- a/src/mavedb/models/job_run.py +++ b/src/mavedb/models/job_run.py @@ -70,6 +70,14 @@ class JobRun(Base): progress_total: Mapped[Optional[int]] = mapped_column(Integer, nullable=True) progress_message: Mapped[Optional[str]] = mapped_column(String(500), nullable=True) + # Liveness heartbeat. `onupdate=func.now()` bumps this on *any* commit that dirties the row, + # so every progress checkpoint (and every lifecycle transition) refreshes it with zero per-job + # code. The stalled-job sweeper treats a RUNNING job whose heartbeat has gone stale as wedged, + # which distinguishes a healthy long-running job (still committing progress) from a dead one. + progress_updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now(), onupdate=func.now() + ) + # Correlation for tracing correlation_id: Mapped[Optional[str]] = mapped_column(String(255), nullable=True) @@ -98,6 +106,8 @@ class JobRun(Base): Index("ix_job_runs_created_at", "created_at"), Index("ix_job_runs_correlation_id", "correlation_id"), Index("ix_job_runs_status_scheduled", "status", "scheduled_at"), + # Supports the stalled-job sweeper's RUNNING heartbeat query (status + heartbeat staleness). + Index("ix_job_runs_status_progress_updated", "status", "progress_updated_at"), CheckConstraint( "status IN ('pending', 'queued', 'running', 'succeeded', 'failed', 'errored', 'cancelled', 'skipped')", name="ck_job_runs_status_valid", diff --git a/src/mavedb/worker/jobs/registry.py b/src/mavedb/worker/jobs/registry.py index 94cd76e32..ba9754a05 100644 --- a/src/mavedb/worker/jobs/registry.py +++ b/src/mavedb/worker/jobs/registry.py @@ -71,8 +71,10 @@ cron( cleanup_stalled_jobs, name="cleanup_stalled_jobs_cron", - minute={15, 45}, # Run at :15 and :45 past each hour (every 30 minutes) - keep_result=timedelta(minutes=25).total_seconds(), + # Every 10 minutes. Paired with the 20-min progress-stall window this gives an effective + # stall-detection latency of ~20-30 min (vs. up to 50 min on the old 30-min cadence). + minute={5, 15, 25, 35, 45, 55}, + keep_result=timedelta(minutes=9).total_seconds(), ), ] diff --git a/src/mavedb/worker/jobs/system/cleanup.py b/src/mavedb/worker/jobs/system/cleanup.py index 89089828c..4a2dc975e 100644 --- a/src/mavedb/worker/jobs/system/cleanup.py +++ b/src/mavedb/worker/jobs/system/cleanup.py @@ -13,6 +13,7 @@ The cleanup job acts as a safety net to ensure jobs don't remain in limbo forever. """ +import asyncio import logging from datetime import datetime, timedelta, timezone @@ -37,14 +38,30 @@ logger = logging.getLogger(__name__) # Timeout thresholds for detecting stalled jobs (in minutes). -# RUNNING_TIMEOUT_MINUTES must stay below ArqWorkerSettings.job_timeout (currently 2 hours) -# to avoid marking legitimately running jobs as stalled. -RUNNING_TIMEOUT_MINUTES = 150 # RUNNING jobs should complete within 150 min (30 min buffer under ARQ timeout) +# +# RUNNING jobs are evaluated on two signals: +# 1. PROGRESS_STALL_MINUTES — the primary signal. A healthy job checkpoints progress frequently +# (JobRun.progress_updated_at is bumped on every committing progress update via onupdate=now()). +# If the heartbeat has not advanced in this window the job is considered wedged, regardless of +# how long it has legitimately been running. This is what lets a multi-hour VEP fan-out run to +# completion without being killed, while still catching a job that has genuinely hung. +# 2. RUNNING_TIMEOUT_MINUTES — a wall-clock backstop for jobs that never emit progress (or whose +# heartbeat column is somehow stuck). Sits ~30 min under ArqWorkerSettings.job_timeout (8h) so +# the DB-driven sweeper, not ARQ's hard kill, is what terminates a job — keeping DB state and +# actual work in sync. +PROGRESS_STALL_MINUTES = 20 # RUNNING jobs should checkpoint progress at least this often +RUNNING_TIMEOUT_MINUTES = 450 # Wall-clock backstop (7.5h), 30 min under the 8h ARQ ceiling PENDING_TIMEOUT_MINUTES = 5 # PENDING jobs which are actionable within pipelines should be enqueued within 5 minutes PIPELINE_STUCK_TIMEOUT_MINUTES = ( 5 # Pipelines in non-terminal states with no active jobs should resolve within 5 minutes ) +# How long the sweeper waits for an aborted RUNNING job to actually die before giving up. A job +# cancelled at an await point dies within a poll cycle (sub-second); a job wedged in a blocking sync +# call never honors the abort and will hit this timeout — in which case we do NOT re-drive it (see +# _handle_stalled_running_job) to avoid running it twice. +ABORT_CONFIRM_TIMEOUT_SECONDS = 15 + async def _handle_stalled_job_retry( job: JobRun, @@ -193,19 +210,90 @@ async def _handle_stalled_job_retry( return False +async def _running_job_confirmed_dead(arq_id: str, redis: ArqRedis) -> bool: + """Abort a RUNNING job's current ARQ attempt and report whether it is safe to re-drive. + + RUNNING recovery differs from QUEUED/PENDING recovery: a RUNNING job may still have a live + coroutine holding a worker slot. Re-enqueuing without cancelling it first may mean two + coroutines are running the same job simultaneously, which is unsafe. + + We therefore issue Job.abort() on the current attempt and interpret the outcome: + + - abort() returns True -> the coroutine was alive and has now been cancelled + recorded a + terminal result. Safe to re-drive. + - abort() returns False -> the job is absent from ARQ (crashed worker / never enqueued / already + finished). Nothing is running. Safe to re-drive — this is the common crash-recovery path. + - abort() raises TimeoutError -> the job is still present in ARQ and did not die within the + window: a coroutine wedged in a blocking sync call that cannot honor cooperative cancellation. + NOT safe to re-drive; leave it for the next sweep or ARQ's own ceiling. + - any other error -> treat as not-confirmably-alive and allow re-drive; a genuinely live job + would have either confirmed the abort or timed out, not errored here. + + Returns True when it is safe to re-drive the job, False when the job may still be running. + """ + try: + await ArqJob(arq_id, redis).abort(timeout=ABORT_CONFIRM_TIMEOUT_SECONDS) + return True + except asyncio.TimeoutError: + logger.warning( + f"Abort of RUNNING job attempt {arq_id} did not confirm within " + f"{ABORT_CONFIRM_TIMEOUT_SECONDS}s; job may be wedged in a blocking call — not re-driving" + ) + return False + except Exception as e: + logger.warning(f"Abort of RUNNING job attempt {arq_id} raised {e!r}; treating as not running and re-driving") + return True + + +async def _handle_stalled_running_job( + job: JobRun, + manager: JobManager, + redis: ArqRedis, + stall_reason: str, + db: Session, +) -> bool: + """Recover a stalled RUNNING job via abort-first ordering. + + Aborts the current attempt (urn#N) BEFORE any retry bookkeeping, and only proceeds to re-drive + (which increments retry_count and enqueues urn#N+1) once the abort confirms the original is dead + or absent. If the original refuses to die, we leave it alone rather than risk a double-run. + + Returns True if the sweeper acted on the job (re-drove or permanently failed it), False if it was + left in place because the abort could not be confirmed (so it should not be counted as cleaned). + """ + arq_id = arq_job_id(job) + + # Could not confirm the coroutine is dead. Leave the job RUNNING; the next sweep will retry + # the abort, and ARQ's job_timeout ceiling remains as a final backstop. + if not await _running_job_confirmed_dead(arq_id, redis): + logger.warning( + f"Leaving stalled RUNNING job {job.urn} in place — abort unconfirmed, re-driving would risk a double-run", + extra=manager.logging_context(), + ) + return False + + # Abort confirmed dead (or the job was absent from ARQ). Refresh our view — if the aborted + # coroutine's own lifecycle handler already marked the row terminal, we want to see that — then + # re-drive through the shared retry handler, which preserves pipeline dependency semantics. + db.refresh(job) + await _handle_stalled_job_retry(job, manager, redis, stall_reason, db) + return True + + @with_guaranteed_job_run_record("cron_job") @with_job_management async def cleanup_stalled_jobs(ctx: dict, job_id: int, job_manager: JobManager) -> JobExecutionOutcome: """Detect and handle jobs that have stalled in intermediate states. - This job runs periodically (every 15 minutes) to find jobs that have been - stuck in QUEUED, RUNNING, or PENDING states beyond reasonable timeouts - and handles them appropriately. + This job runs periodically to find jobs that have been stuck in QUEUED, RUNNING, or PENDING + states beyond reasonable timeouts and handles them appropriately. Stalled job detection criteria: - QUEUED: Present in DB as QUEUED but absent from ARQ's Redis queue (process crashed between prepare_queue and redis.enqueue_job) - - RUNNING: Started > 60 minutes ago but not finished (worker likely crashed) + - RUNNING: Progress heartbeat idle > PROGRESS_STALL_MINUTES, or wall-clock runtime past the + RUNNING_TIMEOUT_MINUTES backstop, and not finished. Recovered abort-first (see + _handle_stalled_running_job) so a wedged attempt is never re-driven alongside its own retry. - PENDING: Created > 5 minutes ago in a pipeline and currently runnable (coordination failure) - Pipeline stuck: Non-terminal pipeline with no active jobs older than 5 minutes @@ -230,7 +318,8 @@ async def cleanup_stalled_jobs(ctx: dict, job_id: int, job_manager: JobManager) Job stalled in RUNNING (worker crash): - Worker started job, marked it RUNNING, then crashed - - After 60 minutes (longer than ARQ timeout), janitor detects and retries + - Once the progress heartbeat goes stale (or the wall-clock backstop is hit), the janitor + aborts the dead attempt and re-drives it """ job_manager.save_to_context( { @@ -305,14 +394,19 @@ async def cleanup_stalled_jobs(ctx: dict, job_id: int, job_manager: JobManager) job_manager.save_to_context({"cleaned_queued_jobs": queued_jobs}) logger.debug("Completed cleaning stalled QUEUED jobs.", extra=job_manager.logging_context()) - # Find RUNNING jobs that have been running too long OR have missing started_at - # These likely indicate worker crashes (worker died mid-execution) or data inconsistencies + # Find RUNNING jobs that are wedged. A job is a candidate when EITHER its progress heartbeat has + # gone stale (the primary, runtime-agnostic signal) OR it has blown past the wall-clock backstop + # (or is missing started_at entirely — a data inconsistency). Healthy long-running jobs keep + # bumping progress_updated_at and so are never selected until the backstop. + progress_stall_threshold = now - timedelta(minutes=PROGRESS_STALL_MINUTES) running_threshold = now - timedelta(minutes=RUNNING_TIMEOUT_MINUTES) running_jobs = job_manager.db.scalars( select(JobRun).where( JobRun.status == JobStatus.RUNNING, - (JobRun.started_at < running_threshold) | (JobRun.started_at.is_(None)), JobRun.finished_at.is_(None), + (JobRun.progress_updated_at < progress_stall_threshold) + | (JobRun.started_at < running_threshold) + | (JobRun.started_at.is_(None)), ) ).all() @@ -333,18 +427,26 @@ async def cleanup_stalled_jobs(ctx: dict, job_id: int, job_manager: JobManager) continue elapsed_minutes = (now - job.started_at).total_seconds() / 60 + heartbeat_stale = job.progress_updated_at < progress_stall_threshold + if heartbeat_stale: + heartbeat_age = (now - job.progress_updated_at).total_seconds() / 60 + stall_reason = ( + f"Job stalled in RUNNING state: progress heartbeat idle for {heartbeat_age:.1f} minutes " + f"(running for {elapsed_minutes:.1f} minutes)" + ) + else: + stall_reason = ( + f"Job stalled in RUNNING state for {elapsed_minutes:.1f} minutes " + f"(exceeded {RUNNING_TIMEOUT_MINUTES} min backstop; likely worker crash)" + ) - logger.warning( - f"Detected stalled RUNNING job {job.urn} " - f"(started {job.started_at}, running for {elapsed_minutes:.1f} minutes)", - extra=manager.logging_context(), - ) + logger.warning(f"Detected stalled RUNNING job {job.urn}: {stall_reason}", extra=manager.logging_context()) - stall_reason = f"Job stalled in RUNNING state for {elapsed_minutes:.1f} minutes (likely worker crash)" - await _handle_stalled_job_retry(job, manager, job_manager.redis, stall_reason, job_manager.db) + acted = await _handle_stalled_running_job(job, manager, job_manager.redis, stall_reason, job_manager.db) manager.db.commit() - cleaned_jobs["running"].append(job.urn) + if acted: + cleaned_jobs["running"].append(job.urn) job_manager.save_to_context({"cleaned_running_jobs": running_jobs}) logger.debug("Completed cleaning stalled RUNNING jobs.", extra=job_manager.logging_context()) diff --git a/src/mavedb/worker/lib/decorators/job_management.py b/src/mavedb/worker/lib/decorators/job_management.py index 58aad427c..0dcb463ad 100644 --- a/src/mavedb/worker/lib/decorators/job_management.py +++ b/src/mavedb/worker/lib/decorators/job_management.py @@ -5,6 +5,7 @@ Includes JobManager injection for advanced operations and robust error handling. """ +import asyncio import functools import inspect import logging @@ -15,7 +16,7 @@ from mavedb.lib.slack import send_slack_error, send_slack_job_error, send_slack_job_failure from mavedb.lib.types.workflow import JobExecutionOutcome -from mavedb.models.enums.job_pipeline import JobStatus +from mavedb.models.enums.job_pipeline import FailureCategory, JobStatus from mavedb.worker.lib.decorators.utils import ensure_ctx, ensure_job_id, ensure_session_ctx, is_test_mode from mavedb.worker.lib.managers import JobManager from mavedb.worker.lib.managers.constants import TERMINAL_JOB_STATUSES @@ -143,6 +144,43 @@ async def _execute_managed_job(func: Callable[..., Awaitable[JobExecutionOutcome return result + # The coroutine is being cancelled — either ARQ hit job_timeout, or cleanup_stalled_jobs + # aborted this attempt (Job.abort()) as part of stall recovery. CancelledError is a + # BaseException, so the `except Exception` handler below never sees it; without this branch + # the JobRun row would be orphaned as RUNNING forever while the coroutine dies. + # + # We mark the row terminally (FAILED / TIMEOUT) so DB state matches reality, then RE-RAISE. + # Re-raising is essential: it lets ARQ record the cancellation as the job's result, which is + # what makes a concurrent Job.abort() confirm the job actually died. + except asyncio.CancelledError: + logger.warning(f"Job {job_id} cancelled (timeout or stall-recovery abort); marking FAILED before re-raising") + try: + db_session.rollback() + if job_manager is not None: + job_manager.fail_job( + result=JobExecutionOutcome.failed( + reason="Job cancelled due to timeout, stall-recovery, or internal ARQ abort", + data={"reason": "cancelled"}, + failure_category=FailureCategory.TIMEOUT, + ) + ) + db_session.commit() + job = job_manager.get_job() + send_slack_job_failure( + job_urn=job.urn, + job_function=job.job_function, + reason="Job cancelled due to timeout.", + failure_category=str(FailureCategory.TIMEOUT), + retry_count=job.retry_count, + max_retries=job.max_retries, + will_retry=False, + ) + + except Exception as inner_e: + logger.critical(f"Failed to mark cancelled job {job_id} as failed: {inner_e}") + send_slack_error(inner_e) + raise + except Exception as e: # Prioritize salvaging lifecycle state will_retry = False diff --git a/src/mavedb/worker/settings/worker.py b/src/mavedb/worker/settings/worker.py index 6a558e6b6..3b0ab4e89 100644 --- a/src/mavedb/worker/settings/worker.py +++ b/src/mavedb/worker/settings/worker.py @@ -25,7 +25,13 @@ # driver, enabling incremental migration of job functions without touching # the FastAPI layer. Once all jobs use async sessions, raise MAX_JOBS to 10+. MAX_JOBS = 2 -JOB_TIMEOUT_SECONDS = 3 * 60 * 60 # 3 hours — matches RUNNING_TIMEOUT_MINUTES (150 min) with buffer +# ARQ's hard coroutine-kill ceiling. Kept deliberately high (8h) so ARQ is the *last* resort: +# our own DB-driven sweeper (cleanup_stalled_jobs) should detect and recover stalls long before +# this fires, keeping the JobRun state machine in sync with reality. VEP fan-out over large allele +# sets can legitimately run for hours, so a low ceiling would kill healthy work. +# cleanup's RUNNING_TIMEOUT_MINUTES backstop sits ~30 min under this to try and sweep these jobs before +# ARQ's hard timeout fires. +JOB_TIMEOUT_SECONDS = 8 * 60 * 60 # 8 hours class ArqWorkerSettings: @@ -44,3 +50,7 @@ class ArqWorkerSettings: max_jobs = MAX_JOBS job_timeout = JOB_TIMEOUT_SECONDS + # Required for cleanup_stalled_jobs to abort a wedged RUNNING job's coroutine (via Job.abort()) + # before re-driving it. Without this, ARQ ignores abort requests and the sweeper cannot safely + # confirm a stalled job is dead before retrying it. + allow_abort_jobs = True diff --git a/tests/worker/jobs/system/test_cleanup.py b/tests/worker/jobs/system/test_cleanup.py index 522b6167d..104c3af6d 100644 --- a/tests/worker/jobs/system/test_cleanup.py +++ b/tests/worker/jobs/system/test_cleanup.py @@ -12,6 +12,7 @@ pytest.importorskip("arq") # Skip tests if arq is not installed +import asyncio from datetime import datetime, timedelta, timezone from unittest.mock import AsyncMock, patch @@ -29,6 +30,7 @@ from mavedb.worker.jobs.system.cleanup import ( PENDING_TIMEOUT_MINUTES, PIPELINE_STUCK_TIMEOUT_MINUTES, + PROGRESS_STALL_MINUTES, RUNNING_TIMEOUT_MINUTES, cleanup_stalled_jobs, ) @@ -40,17 +42,55 @@ @pytest.fixture def mock_arq_job_not_found(): - """Mock ArqJob.status() to return not_found, simulating a crashed-enqueue QUEUED job.""" + """Mock ArqJob for a crashed-enqueue QUEUED job: status() is not_found, and abort() confirms dead.""" with patch("mavedb.worker.jobs.system.cleanup.ArqJob") as mock_arq_job: mock_arq_job.return_value.status = AsyncMock(return_value=ArqJobStatus.not_found) + mock_arq_job.return_value.abort = AsyncMock(return_value=True) yield mock_arq_job @pytest.fixture def mock_arq_job_in_redis(): - """Mock ArqJob.status() to return queued, simulating a legitimately queued job in ARQ Redis.""" + """Mock ArqJob for a legitimately queued job: status() is queued, and abort() confirms dead.""" with patch("mavedb.worker.jobs.system.cleanup.ArqJob") as mock_arq_job: mock_arq_job.return_value.status = AsyncMock(return_value=ArqJobStatus.queued) + mock_arq_job.return_value.abort = AsyncMock(return_value=True) + yield mock_arq_job + + +@pytest.fixture +def mock_arq_abort_confirmed(): + """Mock ArqJob.abort() to return True — the stalled RUNNING attempt was alive and is now dead. + + This is the signal that makes it safe for the sweeper to re-drive a stalled RUNNING job. + """ + with patch("mavedb.worker.jobs.system.cleanup.ArqJob") as mock_arq_job: + mock_arq_job.return_value.status = AsyncMock(return_value=ArqJobStatus.not_found) + mock_arq_job.return_value.abort = AsyncMock(return_value=True) + yield mock_arq_job + + +@pytest.fixture +def mock_arq_abort_absent(): + """Mock ArqJob.abort() to return False — the job is absent from ARQ (crashed / never enqueued). + + Nothing is running, so it is still safe to re-drive. + """ + with patch("mavedb.worker.jobs.system.cleanup.ArqJob") as mock_arq_job: + mock_arq_job.return_value.status = AsyncMock(return_value=ArqJobStatus.not_found) + mock_arq_job.return_value.abort = AsyncMock(return_value=False) + yield mock_arq_job + + +@pytest.fixture +def mock_arq_abort_wedged(): + """Mock ArqJob.abort() to time out — the coroutine is wedged in a blocking call and won't die. + + The sweeper must NOT re-drive such a job (it could still be running), to avoid a double-run. + """ + with patch("mavedb.worker.jobs.system.cleanup.ArqJob") as mock_arq_job: + mock_arq_job.return_value.status = AsyncMock(return_value=ArqJobStatus.in_progress) + mock_arq_job.return_value.abort = AsyncMock(side_effect=asyncio.TimeoutError()) yield mock_arq_job @@ -148,7 +188,7 @@ async def test_cleanup_stalled_queued_job_max_retries_reached( assert "stalled" in stalled_job.error_message.lower() async def test_cleanup_stalled_running_job_with_retries( - self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job + self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job, mock_arq_abort_confirmed ): """Test cleanup of a stalled RUNNING job with retries remaining.""" # Create a stalled RUNNING job in the database @@ -184,7 +224,7 @@ async def test_cleanup_stalled_running_job_with_retries( assert stalled_job.finished_at is None async def test_cleanup_stalled_running_job_max_retries_reached( - self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job + self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job, mock_arq_abort_confirmed ): """Test cleanup of a stalled RUNNING job with max retries reached.""" # Create a stalled RUNNING job with max retries @@ -257,6 +297,115 @@ async def test_cleanup_stalled_running_job_missing_started_at( assert stalled_job.status == JobStatus.RUNNING assert stalled_job.retry_count == 0 + async def test_cleanup_detects_running_job_by_stale_heartbeat( + self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job, mock_arq_abort_confirmed + ): + """A RUNNING job with a recent started_at but a stale progress heartbeat is detected and re-driven. + + This is the primary staleness signal: staleness is judged by progress_updated_at, not total runtime. + """ + stalled_job = JobRun( + job_type="test_job", + job_function="test_function", + status=JobStatus.RUNNING, + created_at=datetime.now(timezone.utc) - timedelta(minutes=40), + started_at=datetime.now(timezone.utc) - timedelta(minutes=30), # well within the wall-clock backstop + progress_updated_at=datetime.now(timezone.utc) - timedelta(minutes=PROGRESS_STALL_MINUTES + 5), + finished_at=None, + max_retries=3, + retry_count=0, + job_params={}, + ) + session.add(stalled_job) + session.commit() + + result = await cleanup_stalled_jobs( + mock_worker_ctx, None, JobManager(session, mock_worker_ctx["redis"], sample_cleanup_job_run.id) + ) + + # Abort of the wedged attempt was confirmed, so the job was re-driven. + mock_arq_abort_confirmed.return_value.abort.assert_awaited_once() + mock_worker_ctx["redis"].enqueue_job.assert_called_once() + + assert result.data["total_cleaned"] == 1 + assert stalled_job.urn in result.data["running_jobs"] + + session.refresh(stalled_job) + assert stalled_job.status == JobStatus.QUEUED + assert stalled_job.retry_count == 1 + + async def test_cleanup_leaves_running_job_with_fresh_heartbeat( + self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job + ): + """A RUNNING job that has been running a long time but is still checkpointing progress is left alone. + + This is the VEP fan-out case: legitimate multi-hour runtime, well past the old 150-min limit, + but under the wall-clock backstop and with a fresh heartbeat — must NOT be treated as stalled. + """ + healthy_job = JobRun( + job_type="test_job", + job_function="test_function", + status=JobStatus.RUNNING, + created_at=datetime.now(timezone.utc) - timedelta(minutes=310), + started_at=datetime.now(timezone.utc) - timedelta(minutes=300), # 5h — past old limit, under backstop + progress_updated_at=datetime.now(timezone.utc) - timedelta(minutes=2), # fresh heartbeat + finished_at=None, + max_retries=3, + retry_count=0, + job_params={}, + ) + session.add(healthy_job) + session.commit() + + result = await cleanup_stalled_jobs( + mock_worker_ctx, None, JobManager(session, mock_worker_ctx["redis"], sample_cleanup_job_run.id) + ) + + assert result.data["total_cleaned"] == 0 + assert healthy_job.urn not in result.data["running_jobs"] + + session.refresh(healthy_job) + assert healthy_job.status == JobStatus.RUNNING + assert healthy_job.retry_count == 0 + + async def test_cleanup_does_not_redrive_wedged_running_job( + self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job, mock_arq_abort_wedged + ): + """A stalled RUNNING job whose abort does not confirm is left in place, not re-driven. + + Re-driving a job that may still be executing (wedged in a blocking sync call) would recreate + the historical double-run, so the sweeper takes no action and leaves it for a later sweep. + """ + wedged_job = JobRun( + job_type="test_job", + job_function="test_function", + status=JobStatus.RUNNING, + created_at=datetime.now(timezone.utc) - timedelta(minutes=40), + started_at=datetime.now(timezone.utc) - timedelta(minutes=30), + progress_updated_at=datetime.now(timezone.utc) - timedelta(minutes=PROGRESS_STALL_MINUTES + 5), + finished_at=None, + max_retries=3, + retry_count=0, + job_params={}, + ) + session.add(wedged_job) + session.commit() + + result = await cleanup_stalled_jobs( + mock_worker_ctx, None, JobManager(session, mock_worker_ctx["redis"], sample_cleanup_job_run.id) + ) + + # Abort was attempted but never confirmed, so no retry was enqueued. + mock_arq_abort_wedged.return_value.abort.assert_awaited_once() + mock_worker_ctx["redis"].enqueue_job.assert_not_called() + + assert result.data["total_cleaned"] == 0 + assert wedged_job.urn not in result.data["running_jobs"] + + session.refresh(wedged_job) + assert wedged_job.status == JobStatus.RUNNING + assert wedged_job.retry_count == 0 + async def test_cleanup_stalled_pending_job_with_retries( self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job ): @@ -477,7 +626,7 @@ async def test_cleanup_stalled_queued_standalone_job_enqueue_failure( assert "Failed to enqueue after stall recovery" in stalled_job.error_message async def test_cleanup_stalled_running_standalone_job_enqueue_failure( - self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job + self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job, mock_arq_abort_confirmed ): """Test that stalled standalone RUNNING job is marked FAILED if ARQ enqueue fails.""" @@ -559,7 +708,7 @@ async def test_cleanup_stalled_queued_pipeline_job_dependencies_satisfied( assert stalled_job.retry_count == 1 async def test_cleanup_stalled_running_pipeline_job_dependencies_satisfied( - self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job + self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job, mock_arq_abort_confirmed ): """Test that stalled pipeline RUNNING job with satisfied dependencies is enqueued.""" # Create a pipeline with all dependencies satisfied @@ -735,7 +884,7 @@ async def test_cleanup_stalled_queued_pipeline_job_dependencies_not_ready( assert stalled_job.retry_count == 1 async def test_cleanup_stalled_running_pipeline_job_dependencies_failed( - self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job + self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job, mock_arq_abort_confirmed ): """Test that stalled pipeline RUNNING job with failed dependencies is skipped.""" # Create a pipeline @@ -867,7 +1016,7 @@ async def test_cleanup_stalled_pending_pipeline_job_dependencies_failed( assert stalled_job.retry_count == 0 async def test_cleanup_stalled_running_pipeline_job_dependencies_not_ready( - self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job + self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job, mock_arq_abort_confirmed ): """Test that stalled pipeline RUNNING job with dependencies not ready is skipped.""" # Create a pipeline @@ -1249,7 +1398,7 @@ async def test_cleanup_sends_slack_when_max_retries_reached_queued_job( assert call_kwargs["max_retries"] == 3 async def test_cleanup_sends_slack_when_max_retries_reached_running_job( - self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job + self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job, mock_arq_abort_confirmed ): """Test that send_slack_job_failure is called when sweeper permanently fails a stalled RUNNING job.""" stalled_job = JobRun( diff --git a/tests/worker/lib/decorators/test_job_management.py b/tests/worker/lib/decorators/test_job_management.py index dd2e10ef4..fbfb370d8 100644 --- a/tests/worker/lib/decorators/test_job_management.py +++ b/tests/worker/lib/decorators/test_job_management.py @@ -16,7 +16,7 @@ from sqlalchemy import select from mavedb.lib.types.workflow import JobExecutionOutcome -from mavedb.models.enums.job_pipeline import JobStatus +from mavedb.models.enums.job_pipeline import FailureCategory, JobStatus from mavedb.models.job_run import JobRun from mavedb.worker.lib.decorators.job_management import with_job_management from mavedb.worker.lib.managers.exceptions import JobStateError @@ -133,6 +133,67 @@ async def sample_error(ctx: dict, job_id: int, job_manager: JobManager): mock_start_job.assert_called_once() mock_error_job.assert_called_once() + async def test_decorator_marks_failed_and_reraises_on_cancellation( + self, session, mock_worker_ctx, mock_job_manager + ): + """A CancelledError (ARQ timeout or stall-recovery abort) must mark the row FAILED then re-raise. + + Re-raising is what lets ARQ record the cancellation as the job result so a concurrent + Job.abort() can confirm the job actually died. Without this handler the row would be + orphaned as RUNNING (CancelledError is a BaseException, not caught by `except Exception`). + """ + + @with_job_management + async def sample_cancel(ctx: dict, job_id: int, job_manager: JobManager): + raise asyncio.CancelledError() + + with ( + patch("mavedb.worker.lib.decorators.job_management.JobManager") as mock_job_manager_class, + patch("mavedb.worker.lib.decorators.job_management.send_slack_job_failure") as mock_send_slack_job_failure, + patch.object(mock_job_manager, "start_job", return_value=None) as mock_start_job, + patch.object(mock_job_manager, "fail_job", return_value=None) as mock_fail_job, + ): + mock_job_manager_class.return_value = mock_job_manager + + with pytest.raises(asyncio.CancelledError): + await sample_cancel(mock_worker_ctx, 999) + + mock_start_job.assert_called_once() + mock_fail_job.assert_called_once() + # The row is failed with a TIMEOUT category. + outcome = mock_fail_job.call_args.kwargs["result"] + assert outcome.status == JobStatus.FAILED + assert outcome.failure_category == FailureCategory.TIMEOUT + # A permanent-failure alert is emitted for the cancellation. + mock_send_slack_job_failure.assert_called_once() + + async def test_decorator_reraises_cancellation_even_if_marking_failed_errors( + self, session, mock_worker_ctx, mock_job_manager + ): + """If marking the cancelled job FAILED itself raises, the decorator still re-raises CancelledError. + + The salvage path must never swallow the cancellation — ARQ needs it to record the result. + """ + + @with_job_management + async def sample_cancel(ctx: dict, job_id: int, job_manager: JobManager): + raise asyncio.CancelledError() + + with ( + patch("mavedb.worker.lib.decorators.job_management.JobManager") as mock_job_manager_class, + patch("mavedb.worker.lib.decorators.job_management.send_slack_error") as mock_send_slack_error, + patch("mavedb.worker.lib.decorators.job_management.send_slack_job_failure"), + patch.object(mock_job_manager, "start_job", return_value=None), + patch.object(mock_job_manager, "fail_job", side_effect=RuntimeError("db down")), + ): + mock_job_manager_class.return_value = mock_job_manager + + with pytest.raises(asyncio.CancelledError): + await sample_cancel(mock_worker_ctx, 999) + + # The inner failure is reported, and the original cancellation still propagates. + mock_send_slack_error.assert_called_once() + async def test_decorator_calls_start_job_and_skip_job_when_wrapped_function_returns_skipped_status( self, session, mock_worker_ctx, mock_job_manager ): diff --git a/tests/worker/lib/managers/test_job_manager.py b/tests/worker/lib/managers/test_job_manager.py index 980a30f02..84f709bf4 100644 --- a/tests/worker/lib/managers/test_job_manager.py +++ b/tests/worker/lib/managers/test_job_manager.py @@ -11,6 +11,7 @@ pytest.importorskip("arq") import re +from datetime import datetime, timedelta, timezone from unittest.mock import Mock, PropertyMock, patch from arq import ArqRedis @@ -1348,6 +1349,28 @@ def test_update_progress_success(self, session, arq_redis, with_populated_job_da assert job.progress_total == 100 assert job.progress_message == "Halfway done" + def test_update_progress_bumps_progress_updated_at_heartbeat( + self, session, arq_redis, with_populated_job_data, sample_job_run + ): + """Committing a progress update refreshes progress_updated_at via onupdate=now(). + + This is the liveness heartbeat the stalled-job sweeper reads: it must advance on every + committing progress checkpoint with no explicit code in the job. + """ + + manager = JobManager(session, arq_redis, sample_job_run.id) + + # Pin the heartbeat to a stale value so the bump is unambiguous. + stale = datetime.now(timezone.utc) - timedelta(hours=1) + job = session.execute(select(JobRun).where(JobRun.id == sample_job_run.id)).scalar_one() + job.progress_updated_at = stale + session.commit() + + manager.update_progress(10, 100, "tick", commit=True) # commit triggers the UPDATE + onupdate + + job = session.execute(select(JobRun).where(JobRun.id == sample_job_run.id)).scalar_one() + assert job.progress_updated_at > stale + def test_update_progress_success_does_not_overwrite_old_message_when_no_new_message_is_provided( self, session, arq_redis, with_populated_job_data, sample_job_run ): From 1297468b9b4c75673441814f9f80a6abbfc3a667 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Thu, 4 Jun 2026 11:05:29 -0700 Subject: [PATCH 005/252] feat(data-providers): add seqrepo-backed vrs data proxy - add shared seqrepo and seqrepo data proxy providers in services - reuse shared seqrepo provider in deps to remove duplicate config - align vrs sequence proxy behavior with dcd-mapping expectations --- src/mavedb/data_providers/services.py | 18 ++++++++++++++++++ src/mavedb/deps.py | 6 ++---- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/mavedb/data_providers/services.py b/src/mavedb/data_providers/services.py index a94c16d6e..45b91b756 100644 --- a/src/mavedb/data_providers/services.py +++ b/src/mavedb/data_providers/services.py @@ -2,7 +2,9 @@ from typing import TYPE_CHECKING, Optional import boto3 +from biocommons.seqrepo import SeqRepo from cdot.hgvs.dataproviders import ChainedSeqFetcher, FastaSeqFetcher, RESTDataProvider, SeqFetcher +from ga4gh.vrs.dataproxy import SeqRepoDataProxy from mavedb.lib.mapping import VRSMap @@ -16,6 +18,7 @@ DCD_MAP_URL = os.environ.get("DCD_MAPPING_URL", "http://dcd-mapping:8000") CDOT_URL = os.environ.get("CDOT_URL", "http://cdot-rest:8000") +SEQREPO_DIR = os.environ.get("HGVS_SEQREPO_DIR", "/seqrepo") CSV_UPLOAD_S3_BUCKET_NAME = os.getenv("UPLOAD_S3_BUCKET_NAME", "score-set-csv-uploads-dev") @@ -27,6 +30,21 @@ def cdot_rest() -> RESTDataProvider: return RESTDataProvider(url=CDOT_URL, seqfetcher=seqfetcher()) +def seqrepo() -> SeqRepo: + return SeqRepo(SEQREPO_DIR) + + +def seqrepo_data_proxy() -> SeqRepoDataProxy: + """VRS sequence/refget data proxy backed by local SeqRepo. + + Distinct from cdot_rest(): cdot is an hgvs *coordinate* data provider, while + AlleleTranslator needs a VRS *sequence* proxy that can derive_refget_accession. + Mirrors dcd_mapping's SeqRepo-backed translator. Uses the same HGVS_SEQREPO_DIR + location as deps.get_seqrepo and the refget router. + """ + return SeqRepoDataProxy(seqrepo()) + + def vrs_mapper(url: Optional[str] = None) -> VRSMap: return VRSMap(DCD_MAP_URL) if not url else VRSMap(url) diff --git a/src/mavedb/deps.py b/src/mavedb/deps.py index 8e4ce5106..712a6e5a3 100644 --- a/src/mavedb/deps.py +++ b/src/mavedb/deps.py @@ -1,4 +1,3 @@ -import os from typing import Any, AsyncGenerator, Generator from arq import ArqRedis, create_pool @@ -6,7 +5,7 @@ from cdot.hgvs.dataproviders import RESTDataProvider from sqlalchemy.orm import Session -from mavedb.data_providers.services import cdot_rest +from mavedb.data_providers.services import cdot_rest, seqrepo from mavedb.db.session import SessionLocal from mavedb.worker.settings import RedisWorkerSettings @@ -33,5 +32,4 @@ def hgvs_data_provider() -> RESTDataProvider: def get_seqrepo() -> SeqRepo: - seqrepo_dir = os.environ.get("HGVS_SEQREPO_DIR", "/seqrepo") - return SeqRepo(seqrepo_dir) + return seqrepo() From 7ea450401d5c6134824fc4f980edd8bf1495431d Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Thu, 4 Jun 2026 11:40:32 -0700 Subject: [PATCH 006/252] fix(vrs): centralize allele id recomputation helpers - add shared helpers to translate hgvs, normalize alleles, and compute ids - clear cached merkle digests before identification so mutated alleles do not keep stale ga4gh identifiers - document the invariant that all allele identification must route through the helper to prevent digest correctness regressions --- src/mavedb/lib/vrs_utils.py | 82 +++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 src/mavedb/lib/vrs_utils.py diff --git a/src/mavedb/lib/vrs_utils.py b/src/mavedb/lib/vrs_utils.py new file mode 100644 index 000000000..2720afa68 --- /dev/null +++ b/src/mavedb/lib/vrs_utils.py @@ -0,0 +1,82 @@ +"""VRS allele identification helpers. + +Centralizes the digest-correctness invariant for GA4GH VRS alleles: the +``ga4gh_identify`` Merkle tree caches sub-object digests on the object after +first identification, so any subsequent mutation (refgetAccession swap, +normalization, state coercion) leaves a stale id unless the cached digests +are cleared first. All allele identification in dcd_mapping must route +through :func:`identify_allele` so the digest is always recomputed from +current content. +""" + +from typing import Any + +from ga4gh.core import ga4gh_identify +from ga4gh.vrs.extras.translator import AlleleTranslator +from ga4gh.vrs.models import Allele, LiteralSequenceExpression, SequenceLocation +from ga4gh.vrs.normalize import normalize + + +def translate_hgvs_to_vrs(hgvs: str, translator: AlleleTranslator) -> Allele: + """Convert HGVS variation description to VRS object. + + The AlleleTranslator is supplied by the caller and reused across calls. ga4gh's + AlleleTranslator opens a UTA connection lazily on first translate (via HgvsTools) + and holds it for its lifetime, so constructing one per call opens — and leaks — a + UTA connection per variant, exhausting the server's slot budget. Build one per + job/worker and pass it in. + + :param hgvs: MAVE-HGVS variation string + :param translator: caller-owned AlleleTranslator backed by a sequence/refget proxy + :return: Corresponding VRS allele as a Pydantic class + """ + # coerce tmp HGVS string into formally correct term + if hgvs.startswith("NC_") and ":c." in hgvs: + hgvs = hgvs.replace(":c.", ":g.") + + allele: Allele = translator.translate_from(hgvs, "hgvs", do_normalize=False) + + if ( + not isinstance(allele.location, SequenceLocation) + or not isinstance(allele.location.start, int) + or not isinstance(allele.location.end, int) + or not isinstance(allele.state, LiteralSequenceExpression) + ): + raise ValueError + + return allele + + +def identify_allele(allele: Allele) -> str: + """Clear cached digests and return a fresh GA4GH identifier for *allele*. + + ``ga4gh_identify`` is a Merkle-tree: it calls ``get_or_create_digest`` on + sub-objects, returning any cached value without recomputing. Clearing both + the location digest and the allele digest first ensures the id is always + derived from the current object content — not from a value set before a + refgetAccession mutation or normalization. + """ + if isinstance(allele.location, SequenceLocation): + allele.location.digest = None + + allele.digest = None + digest = ga4gh_identify(allele) + if digest is None: + raise ValueError("Failed to compute GA4GH identifier for allele") # noqa: EM101 + + return digest + + +def normalize_and_identify(allele: Allele, data_proxy: Any) -> Allele: + """Normalize *allele* and stamp it with a freshly computed GA4GH digest. + + Pairs the two finalize steps every VRS allele construction path needs. + Routing identification through :func:`identify_allele` (rather than + ``ga4gh_identify`` directly) is the invariant that protects against the + Merkle-tree's stale-digest behavior after mutation -- so any allele + construction site that bypasses this helper risks reintroducing the + stale-digest bug. + """ + allele = normalize(allele, data_proxy=data_proxy) + allele.id = identify_allele(allele) + return allele From 825cf9d23a7c180b8f8338731549202588052770 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Sat, 6 Jun 2026 18:47:51 -0700 Subject: [PATCH 007/252] feat(db): add ValidTime mixin for valid-time row versioning Introduce a reusable declarative mixin implementing transaction-time SCD Type 2 versioning for rows that change over time, used by either versioned entities or link/association rows. - A row is live while valid_to is NULL; current/as_of express the half-open [valid_from, valid_to) point-in-time predicate so call sites never hand-roll it - supersede_with (single row) and supersede_live_where (bulk) stamp the retired valid_to and successor valid_from with one timestamp for a gap-free handoff regardless of transaction boundaries - retire/retire_live_where are withdrawal primitives; retire cascades to live child links named in __retire_cascade__ - bulk supersede refuses to run on cascade-bearing classes since it cannot fire the cascade - consumers must add a partial unique index over their natural key WHERE valid_to IS NULL as a backstop against duplicate live rows Cover the mixin with unit tests against purpose-built models, using explicit timestamps far from the transaction clock to prove the gap-free handoff and verify the partial unique index backstop. --- src/mavedb/db/mixins.py | 201 ++++++++++++++++++++++++++++ tests/db/test_mixins.py | 289 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 490 insertions(+) create mode 100644 src/mavedb/db/mixins.py create mode 100644 tests/db/test_mixins.py diff --git a/src/mavedb/db/mixins.py b/src/mavedb/db/mixins.py new file mode 100644 index 000000000..4bc5c1ca9 --- /dev/null +++ b/src/mavedb/db/mixins.py @@ -0,0 +1,201 @@ +"""Reusable declarative mixins.""" + +from datetime import datetime +from typing import ClassVar, Optional, Sequence, TypeVar + +from sqlalchemy import ColumnElement, DateTime, Update, func, inspect as sa_inspect, select, update +from sqlalchemy.ext.hybrid import hybrid_property +from sqlalchemy.orm import Mapped, Session, mapped_column + +T = TypeVar("T", bound="ValidTime") + + +class ValidTime: + """Valid-time versioning for rows that change over time (transaction-time SCD Type 2). + + Applies to either a versioned entity (a row that is replaced by a newer version of the same + logical thing — e.g. a mapping record) or a link/association row (a relationship that comes + and goes). Immutable, content-addressed rows (deduplicated entities like an allele or a + source-versioned annotation record) do not use this — their "what applies now" is answered by + the links that reference them, not by versioning the row itself. + + A row is live while ``valid_to`` is NULL. Superseding a row sets its ``valid_to`` to the + successor's ``valid_from`` instead of deleting it, so the full history is retained and a + point-in-time query is a single half-open ``[valid_from, valid_to)`` predicate. ``current`` + and ``as_of`` express that predicate so call sites never hand-roll it. + + Convention: replacing a live row with a successor goes through :meth:`supersede_with` (single + row) or :meth:`supersede_live_where` (bulk), which stamp the retired ``valid_to`` and the new + ``valid_from`` with one timestamp so the handoff is gap-free regardless of transaction + boundaries. ``retire`` (single) / ``retire_live_where`` (bulk) are *withdrawal* primitives — + retiring with no successor; do not pair a bare retire with a separate insert, or a slow job + between the two opens a window where the row is neither live-old nor live-new. The method pairs + mirror: ``retire``/``supersede_with`` act on one row, ``*_live_where`` act on a predicate. + + ``current`` is derived (``valid_to IS NULL``), not stored — one source of truth, no + dual-write to keep consistent. Each table using this mixin should add a partial unique index + over its natural key ``WHERE valid_to IS NULL`` to enforce a single live row per key (the link + key for an association, the logical-entity key for an entity). + + Transaction time only: ``valid_from``/``valid_to`` record when *we* held the row, not when an + external source considered it effective. Tables fed by externally-versioned sources (e.g. a + ClinVar release, a gnomAD version) should add their own source-version column for that axis. + + Consumer contract — a model mixing this in, and any code writing it, MUST honor: + + 1. **Add the partial unique index.** Declare a unique index over the natural key + ``WHERE valid_to IS NULL`` (one live row per key). It is the loud backstop: forgetting to + retire a predecessor before inserting its successor raises ``IntegrityError`` instead of + silently leaving two live rows. + 2. **Replace through supersede, never retire-then-insert.** Use :meth:`supersede_with` (single + row) or :meth:`supersede_live_where` (bulk) to swap a live row for a successor. A bare + :meth:`retire` followed by a separate insert is the gap footgun — if a slow job or a commit + falls between them, the retired ``valid_to`` and the new ``valid_from`` come from different + transaction clocks and a point-in-time query lands in a hole where the row is neither + old-live nor new-live. Reserve ``retire``/``retire_live_where`` for genuine withdrawal (no + successor). Gaps only arise on a *same-key* handoff; the cascade in (3) only ever retires + children under a superseded parent's old key, so it never needs a supersede. + 3. **Declare ``__retire_cascade__`` for parent rows with ``ValidTime`` children.** A live link to + a retired parent is stale; retiring a parent must retire its live child links. The ORM + relationship cascade does not do this (it fires only on hard DELETE). Bulk + :meth:`supersede_live_where` refuses to run on a class that declares ``__retire_cascade__`` + precisely because it cannot cascade — supersede such rows one at a time with + :meth:`supersede_with`. + 4. **Filter reads with ``current``/``as_of``.** Never read live rows by assuming a query returns + only current state; constrain with ``current`` (or ``as_of(ts)``) explicitly. Because of (3), + a live link implies a live parent, so allele/parent-side ``current`` queries do not surface + links dangling off retired parents. + """ + + valid_from: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now()) + valid_to: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True), nullable=True) + + # Names of relationships to child ``ValidTime`` rows that should be retired when this row is. + # A link to a retired row is itself stale, so closing a parent's window must close its live + # links' windows too — the relationship cascade ("all, delete-orphan") only fires on a hard + # DELETE, not on this soft ``valid_to`` close. Each named relationship must target a + # ``ValidTime`` model; ``retire`` issues one bulk UPDATE per relationship. + # + # This covers only the transaction-time axis: "the parent was superseded, so its links are too." + # Links retired on a source-version axis (e.g. a new gnomAD release superseding an + # allele→gnomAD-variant link while the allele itself stays live) are a separate trigger and are + # not driven from here. Cascade is one level deep (bulk UPDATE, not per-row ``retire``); a deeper + # chain would need its own handling. + __retire_cascade__: ClassVar[tuple[str, ...]] = () + + @hybrid_property + def current(self) -> bool: + return self.valid_to is None + + @current.inplace.expression + @classmethod + def _current_expression(cls) -> ColumnElement[bool]: + return cls.valid_to.is_(None) + + @classmethod + def as_of(cls, ts: datetime) -> ColumnElement[bool]: + """Filter clause selecting the rows live at ``ts`` (half-open ``[valid_from, valid_to)``).""" + return (cls.valid_from <= ts) & (cls.valid_to.is_(None) | (cls.valid_to > ts)) + + def retire(self, session: Optional[Session] = None, at: Optional[datetime] = None) -> None: + """Close this row's validity window, cascading to the live child links named in + ``__retire_cascade__``. Idempotent — an already-closed row keeps its original valid_to. + + Pass ``at`` to stamp an explicit ``valid_to`` shared with a successor's ``valid_from``; this + is how :meth:`supersede_with` keeps the handoff gap-free. Without ``at`` the close uses + ``func.now()`` (Postgres ``transaction_timestamp``). This is a *withdrawal* primitive on its + own — to replace a row with a successor, use :meth:`supersede_with` so the handoff is closed. + + ``session`` is required only when ``__retire_cascade__`` is non-empty (the cascade issues a + bulk UPDATE per child relationship). A leaf row with no declared cascades retires without one. + The cascade runs even when this row was already closed, so a half-finished prior retire (row + closed, links not) still converges. + """ + stamp = at if at is not None else func.now() + if self.valid_to is None: + self.valid_to = stamp + + if not self.__retire_cascade__: + return + if session is None: + raise ValueError( + f"{type(self).__name__}.retire() needs a session to cascade-retire {self.__retire_cascade__}." + ) + + mapper = sa_inspect(type(self)) + assert mapper is not None # a mapped class always inspects to a Mapper + for name in self.__retire_cascade__: + rel = mapper.relationships[name] + child_cls = rel.mapper.class_ + conditions = [remote == getattr(self, local.key) for local, remote in rel.local_remote_pairs] + session.execute(child_cls.retire_live_where(*conditions, at=at)) + + @classmethod + def retire_live_where(cls, *conditions: ColumnElement[bool], at: Optional[datetime] = None) -> Update: + """An UPDATE that retires the live rows matching ``conditions`` (closes their valid_to). + Only currently-live rows are touched, so already-retired rows keep their original valid_to. + Pass ``at`` to stamp an explicit ``valid_to`` (used by :meth:`supersede`); defaults to + ``func.now()``. + + This is a *withdrawal* primitive — retiring live rows with no successor. To replace live rows + with new ones, use :meth:`supersede_live_where` so retire and insert share one timestamp. + + Usage: ``session.execute(Model.retire_live_where(Model.foo == bar))``. + """ + return ( + update(cls).where(cls.valid_to.is_(None), *conditions).values(valid_to=at if at is not None else func.now()) + ) + + def supersede_with(self: T, session: Session, replacement: T, at: Optional[datetime] = None) -> T: + """Retire this row (cascading to its child links) and insert ``replacement``, stamping the + retired row's ``valid_to`` and the replacement's ``valid_from`` with one timestamp so the + handoff has no gap — independent of how many transactions or how much wall-clock separate + them. The single-row counterpart to :meth:`supersede_live_where`, for superseding one known + row that may carry ``__retire_cascade__`` children. + + Flushes the retire before the insert: the partial unique index (one live row per natural key) + is checked per statement and the unit of work emits INSERTs before UPDATEs, so without the + flush the replacement and the not-yet-closed original would both be live and trip the index. + """ + if at is None: + at = session.scalar(select(func.now())) + assert at is not None # SELECT now() always returns a timestamp + self.retire(session, at=at) + session.flush() + replacement.valid_from = at + session.add(replacement) + return replacement + + @classmethod + def supersede_live_where( + cls, + session: Session, + new_rows: Sequence[T], + *conditions: ColumnElement[bool], + at: Optional[datetime] = None, + ) -> Sequence[T]: + """Retire the live rows matching ``conditions`` and insert ``new_rows``, stamping both sides + with one timestamp so every retired row's ``valid_to`` equals every new row's ``valid_from`` + — a gap-free handoff independent of transaction boundaries. The bulk counterpart to + :meth:`supersede_with` (and the supersede form of :meth:`retire_live_where`), for replacing a + *set* of live rows in one scope (e.g. a score set's derived links) with a freshly computed + set. + + The retire executes immediately (before the inserts flush), so a reused natural key does not + trip the partial unique index. Only for leaf ``ValidTime`` rows: a bulk retire cannot fire + ``__retire_cascade__``, so this refuses to run on a class that declares one — use + :meth:`supersede_with` per row there. + """ + if cls.__retire_cascade__: + raise ValueError( + f"{cls.__name__} declares __retire_cascade__; bulk supersede cannot cascade to children. " + "Use supersede_with per row so child links retire too." + ) + if at is None: + at = session.scalar(select(func.now())) + assert at is not None # SELECT now() always returns a timestamp + session.execute(cls.retire_live_where(*conditions, at=at)) + for row in new_rows: + row.valid_from = at + session.add_all(new_rows) + return new_rows diff --git a/tests/db/test_mixins.py b/tests/db/test_mixins.py new file mode 100644 index 000000000..ad0333a5d --- /dev/null +++ b/tests/db/test_mixins.py @@ -0,0 +1,289 @@ +"""Unit tests for the ValidTime mixin. + +Tested against minimal, purpose-built models rather than any real MaveDB models. ``_VtParent`` is +a cascade-bearing entity (natural key ``key``); ``_VtChild`` is a leaf link (natural key +``(parent_id, tag)``). Both carry a partial unique index over their natural key +``WHERE valid_to IS NULL``, mirroring the contract real consumers must honor. + +Many checks pass an explicit ``at`` that differs from the DB clock: a single-transaction test cannot +otherwise distinguish a deliberate shared timestamp from the coincidental same-transaction handoff +the gap bug relied on, so the explicit ``at`` is what actually proves the timestamp is threaded. +""" + +from datetime import datetime, timedelta, timezone + +import pytest +from sqlalchemy import Column, ForeignKey, Index, Integer, select, text +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import declarative_base, relationship + +from mavedb.db.mixins import ValidTime + +VtBase = declarative_base() + +# Fixed, deterministic windows — far from the test's transaction clock so an accidental +# server_default (func.now()) stamp would be visibly wrong rather than coincidentally equal. +T0 = datetime(2020, 1, 1, tzinfo=timezone.utc) +T1 = datetime(2021, 1, 1, tzinfo=timezone.utc) +T2 = datetime(2022, 1, 1, tzinfo=timezone.utc) + + +class _VtParent(ValidTime, VtBase): + __tablename__ = "vt_parents" + + id = Column(Integer, primary_key=True) + key = Column(Integer, nullable=False) + children = relationship("_VtChild", back_populates="parent") + + __retire_cascade__ = ("children",) + __table_args__ = (Index("uq_vt_parents_live", "key", unique=True, postgresql_where=text("valid_to IS NULL")),) + + +class _VtChild(ValidTime, VtBase): + __tablename__ = "vt_children" + + id = Column(Integer, primary_key=True) + parent_id = Column(Integer, ForeignKey("vt_parents.id"), nullable=False) + tag = Column(Integer, nullable=False) + parent = relationship("_VtParent", back_populates="children") + + __table_args__ = ( + Index("uq_vt_children_live", "parent_id", "tag", unique=True, postgresql_where=text("valid_to IS NULL")), + ) + + +@pytest.fixture(autouse=True) +def vt_tables(session): + """Create the throwaway ValidTime tables on the test engine, drop them after. Autouse so every + test in the module gets the tables without naming the fixture in its signature.""" + engine = session.get_bind() + VtBase.metadata.create_all(bind=engine) + yield + # Release any locks the test's still-open transaction holds before dropping: a post-commit + # attribute access reopens a transaction holding ACCESS SHARE on these tables, and DROP TABLE + # would block on it forever (this fixture tears down while the session fixture is still open). + session.rollback() + VtBase.metadata.drop_all(bind=engine) + + +def _parent(session, key, *, valid_from=None): + p = _VtParent(key=key, valid_from=valid_from) if valid_from else _VtParent(key=key) + session.add(p) + session.commit() + return p + + +def _child(session, parent, tag, *, valid_from=None): + c = ( + _VtChild(parent_id=parent.id, tag=tag, valid_from=valid_from) + if valid_from + else _VtChild(parent_id=parent.id, tag=tag) + ) + session.add(c) + session.commit() + return c + + +class TestCurrentAndAsOf: + def test_fresh_row_is_current(self, session): + p = _parent(session, 1) + assert p.valid_to is None + assert p.current is True + + def test_retired_row_is_not_current(self, session): + p = _parent(session, 1) + p.retire(session, at=T1) + assert p.current is False + + def test_current_expression_filters_to_live_rows(self, session): + live = _parent(session, 1) + retired = _parent(session, 2) + retired.retire(session, at=T1) + session.commit() + + rows = session.scalars(select(_VtParent).where(_VtParent.current)).all() + assert [r.id for r in rows] == [live.id] + + def test_as_of_selects_rows_live_at_timestamp(self, session): + # Window [T0, T2): explicit valid_from and a retire at T2. + p = _parent(session, 1, valid_from=T0) + p.retire(session, at=T2) + session.commit() + + def live_ids(ts): + return [r.id for r in session.scalars(select(_VtParent).where(_VtParent.as_of(ts))).all()] + + assert live_ids(T0 - timedelta(days=1)) == [] # before the window opens + assert live_ids(T0) == [p.id] # at valid_from (inclusive) + assert live_ids(T1) == [p.id] # inside the window + assert live_ids(T2) == [] # at valid_to (exclusive) + assert live_ids(T2 + timedelta(days=1)) == [] # after the window closes + + +class TestRetire: + def test_retire_closes_valid_to(self, session): + c = _child(session, _parent(session, 1), 1) + c.retire(at=T1) + assert c.valid_to == T1 + + def test_retire_is_idempotent(self, session): + c = _child(session, _parent(session, 1), 1) + c.retire(at=T1) + c.retire(at=T2) # already closed — keeps the original valid_to + assert c.valid_to == T1 + + def test_leaf_retire_needs_no_session(self, session): + c = _child(session, _parent(session, 1), 1) + c.retire() # no cascade declared, so no session required + session.commit() + assert c.valid_to is not None + + def test_retire_cascades_to_live_child_links(self, session): + p = _parent(session, 1) + c1 = _child(session, p, 1) + c2 = _child(session, p, 2) + + p.retire(session, at=T1) + session.commit() + + assert p.valid_to == T1 + assert c1.valid_to == T1 # cascade closes the children at the same instant + assert c2.valid_to == T1 + + def test_cascade_retire_without_session_raises(self, session): + p = _parent(session, 1) + with pytest.raises(ValueError, match="cascade-retire"): + p.retire() # cascade declared but no session to issue the child UPDATE + + def test_cascade_runs_even_when_row_already_closed(self, session): + # A half-finished prior retire (parent closed, child not) must still converge. + p = _parent(session, 1) + c = _child(session, p, 1) + p.valid_to = T0 # simulate parent closed without its child + session.commit() + + p.retire(session, at=T1) # idempotent on the parent, but still cascades + session.commit() + assert p.valid_to == T0 # unchanged (idempotent) + assert c.valid_to == T1 # child still retired + + +class TestRetireLiveWhere: + def test_retires_matching_live_rows(self, session): + p = _parent(session, 1) + c1 = _child(session, p, 1) + c2 = _child(session, p, 2) + + session.execute(_VtChild.retire_live_where(_VtChild.parent_id == p.id, at=T1)) + session.commit() + + assert c1.valid_to == T1 + assert c2.valid_to == T1 + + def test_leaves_already_retired_rows_untouched(self, session): + p = _parent(session, 1) + already = _child(session, p, 1) + already.retire(at=T0) + session.commit() + live = _child(session, p, 2) + + session.execute(_VtChild.retire_live_where(_VtChild.parent_id == p.id, at=T1)) + session.commit() + + assert already.valid_to == T0 # retired row keeps its original valid_to + assert live.valid_to == T1 + + +class TestSupersedeWith: + def test_gap_free_handoff_with_explicit_at(self, session): + old = _parent(session, 1, valid_from=T0) + new = _VtParent(key=1) + + old.supersede_with(session, new, at=T1) + session.commit() + + # The retired valid_to equals the successor's valid_from exactly — no point-in-time hole. + assert old.valid_to == T1 + assert new.valid_from == T1 + + def test_default_at_handoff_is_gap_free(self, session): + old = _parent(session, 1) + new = _VtParent(key=1) + + old.supersede_with(session, new) # at defaults to a single captured func.now() + session.commit() + + assert old.valid_to is not None + assert old.valid_to == new.valid_from + + def test_cascades_child_links_at_the_handoff_timestamp(self, session): + old = _parent(session, 1) + child = _child(session, old, 1) + new = _VtParent(key=1) + + old.supersede_with(session, new, at=T1) + session.commit() + + assert old.valid_to == T1 + assert new.valid_from == T1 + assert child.valid_to == T1 # the old parent's child link retired under the same timestamp + + def test_respects_partial_unique_index_via_flush_ordering(self, session): + # old and new share natural key=1; supersede flushes the retire before inserting the + # successor, so the two are never simultaneously live and the unique index does not trip. + old = _parent(session, 1) + new = _VtParent(key=1) + + old.supersede_with(session, new, at=T1) + session.commit() # would raise IntegrityError if both were live at the INSERT + + live = session.scalars(select(_VtParent).where(_VtParent.key == 1, _VtParent.current)).all() + assert [r.id for r in live] == [new.id] + + +class TestSupersedeLiveWhere: + def test_stamps_explicit_valid_from_on_inserts(self, session): + # Without flushing, valid_from would be None under the server_default path; supersede sets it + # explicitly, which is what keeps the handoff gap-free across a later-transaction flush. + new = _VtChild(parent_id=1, tag=1) + _VtChild.supersede_live_where(session, [new], _VtChild.parent_id == 1, at=T1) + assert new.valid_from == T1 + + def test_gap_free_handoff_with_reused_natural_key(self, session): + p = _parent(session, 1) + old = _child(session, p, 1) + new = _VtChild(parent_id=p.id, tag=1) # same natural key (parent_id, tag) + + _VtChild.supersede_live_where(session, [new], _VtChild.parent_id == p.id, at=T1) + session.commit() # reused key: retire executes before the insert, so the index holds + + assert old.valid_to == T1 + assert new.valid_from == T1 + live = session.scalars(select(_VtChild).where(_VtChild.parent_id == p.id, _VtChild.current)).all() + assert [r.id for r in live] == [new.id] + + def test_empty_new_rows_retires_the_scope(self, session): + # A re-run that produces nothing should still withdraw the prior live set. + p = _parent(session, 1) + old = _child(session, p, 1) + + _VtChild.supersede_live_where(session, [], _VtChild.parent_id == p.id, at=T1) + session.commit() + + assert old.valid_to == T1 + assert session.scalars(select(_VtChild).where(_VtChild.current)).all() == [] + + def test_refuses_a_cascade_bearing_class(self, session): + # Bulk supersede cannot fire __retire_cascade__, so it must refuse rather than orphan links. + with pytest.raises(ValueError, match="__retire_cascade__"): + _VtParent.supersede_live_where(session, [], _VtParent.key == 1) + + +class TestPartialUniqueIndexBackstop: + def test_two_live_rows_for_one_key_raise(self, session): + # The DB index is the loud backstop: forgetting to retire a predecessor before inserting its + # successor fails at flush instead of silently leaving two live rows. + session.add(_VtParent(key=1)) + session.add(_VtParent(key=1)) + with pytest.raises(IntegrityError): + session.commit() From 160318002214792315ea3b2ba60f52497afcc47a Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Sat, 6 Jun 2026 18:48:38 -0700 Subject: [PATCH 008/252] feat(hgvs): add HGVS accession and cis-phased expression helpers Add a lib module with reusable helpers for parsing and rewriting HGVS strings ahead of VRS translation. - extract_accession returns the reference accession (substring before the first colon), tolerant of whitespace and missing separators - split_cis_phased_hgvs expands a bracketed multivariant expression into fully-qualified components carrying the original accession and coordinate prefix, since ga4gh's AlleleTranslator only yields a single Allele per call and each component must translate alone - join_cis_phased_hgvs is the inverse, recombining components into one bracketed block and returning None when they do not share a single accession and coordinate prefix Cover the helpers with unit tests including the split/join round trip and the mixed-accession and mixed-prefix rejection cases. --- src/mavedb/lib/hgvs.py | 80 ++++++++++++++++++++++++++++++++++++++++++ tests/lib/test_hgvs.py | 76 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 156 insertions(+) create mode 100644 src/mavedb/lib/hgvs.py create mode 100644 tests/lib/test_hgvs.py diff --git a/src/mavedb/lib/hgvs.py b/src/mavedb/lib/hgvs.py new file mode 100644 index 000000000..f330bdfd4 --- /dev/null +++ b/src/mavedb/lib/hgvs.py @@ -0,0 +1,80 @@ +import re +from typing import Optional + +# Coordinate prefix of an HGVS variant description: a single type letter plus a dot +# (g. c. n. m. r. p. ...), capturing the prefix and the remaining description separately. +_HGVS_COORD_PREFIX = re.compile(r"^([a-z]\.)(.+)$") + + +def extract_accession(hgvs_string: str) -> str: + """Extract the reference accession from an HGVS string, or return empty string if it cannot + be parsed. + + This function makes no assumptions about the structure of the accession (e.g. whether it starts with "NM_" + or "NC_") and is robust to extra whitespace. It simply returns the substring before the first colon, which + is the standard HGVS separator between the accession and the variant description. Callers must validate + the accession format separately if needed. + """ + token = (hgvs_string or "").strip() + if ":" not in token: + return "" + + return token.split(":", 1)[0].strip() + + +def split_cis_phased_hgvs(hgvs_string: str) -> list[str]: + """Split a cis-phased multivariant HGVS expression into fully-qualified component strings. + + The reverse-translate-variants tool emits the non-adjacent component substitutions of a + single codon change as one bracketed genomic expression (``NC_000001.11:g.[123A>G;125T>C]``) + whenever they are too far apart on the genome to collapse into a single delins. ga4gh's + AlleleTranslator only ever produces a single Allele, so each component must be translated + independently and recombined into a VRS CisPhasedBlock. + + Each returned component carries the original accession and coordinate prefix + (``NC_000001.11:g.123A>G``). A non-bracketed expression is returned unchanged as a + single-element list, so callers can treat both cases uniformly. + + Unlike a bare mavehgvs split, the accession is preserved: the components feed straight + into VRS translation, which requires a reference accession to resolve positions. + """ + if "[" not in hgvs_string: + return [hgvs_string] + + accession, _, remainder = hgvs_string.partition(":") + prefix = remainder[: remainder.index("[")] # e.g. "g." / "c." + inner = remainder[remainder.index("[") + 1 : remainder.rindex("]")] + return [f"{accession}:{prefix}{component}" for component in inner.split(";") if component] + + +def join_cis_phased_hgvs(components: list[str]) -> Optional[str]: + """Join cis-phased component HGVS strings into one bracketed expression. + + Inverse of :func:`split_cis_phased_hgvs`: ``["NC_…:g.123A>G", "NC_…:g.125T>C"]`` becomes + ``"NC_…:g.[123A>G;125T>C]"``. A single component is returned unchanged. + + Returns ``None`` when the components do not share a single accession and coordinate prefix — + they are then not expressible as one cis-phased block (e.g. members on different sequences). + """ + if not components: + return None + if len(components) == 1: + return components[0] + + accessions: set[str] = set() + prefixes: set[str] = set() + descriptions: list[str] = [] + for component in components: + accession, separator, remainder = component.partition(":") + match = _HGVS_COORD_PREFIX.match(remainder) + if not separator or not match: + return None + + accessions.add(accession) + prefixes.add(match.group(1)) + descriptions.append(match.group(2)) + + if len(accessions) != 1 or len(prefixes) != 1: + return None + + return f"{accessions.pop()}:{prefixes.pop()}[{';'.join(descriptions)}]" diff --git a/tests/lib/test_hgvs.py b/tests/lib/test_hgvs.py new file mode 100644 index 000000000..355174fb1 --- /dev/null +++ b/tests/lib/test_hgvs.py @@ -0,0 +1,76 @@ +import pytest + +from mavedb.lib.hgvs import extract_accession, join_cis_phased_hgvs, split_cis_phased_hgvs + + +@pytest.mark.parametrize( + ("hgvs", "expected"), + [ + ("NC_000001.11:g.1000A>G", "NC_000001.11"), + ("NM_000001.1:c.5A>G", "NM_000001.1"), + ("NC_000001.11:g.[1000A>G;1002T>C]", "NC_000001.11"), + ("no-colon-here", ""), + ("", ""), + ], +) +def test_extract_accession(hgvs, expected): + assert extract_accession(hgvs) == expected + + +def test_split_cis_phased_hgvs_passes_through_single_variant(): + assert split_cis_phased_hgvs("NC_000001.11:g.1000A>G") == ["NC_000001.11:g.1000A>G"] + + +def test_split_cis_phased_hgvs_splits_bracketed_genomic_expression(): + # Non-adjacent codon components are emitted as one bracketed genomic expression; each + # component must regain the accession and prefix to be VRS-translatable on its own. + assert split_cis_phased_hgvs("NC_000001.11:g.[1000A>G;1002T>C]") == [ + "NC_000001.11:g.1000A>G", + "NC_000001.11:g.1002T>C", + ] + + +def test_split_cis_phased_hgvs_handles_three_components_and_coding_prefix(): + assert split_cis_phased_hgvs("NM_000001.1:c.[1A>G;3T>C;5G>A]") == [ + "NM_000001.1:c.1A>G", + "NM_000001.1:c.3T>C", + "NM_000001.1:c.5G>A", + ] + + +def test_join_cis_phased_hgvs_passes_through_single_component(): + assert join_cis_phased_hgvs(["NC_000001.11:g.1000A>G"]) == "NC_000001.11:g.1000A>G" + + +def test_join_cis_phased_hgvs_combines_genomic_components(): + assert ( + join_cis_phased_hgvs(["NC_000001.11:g.1000A>G", "NC_000001.11:g.1002T>C"]) == "NC_000001.11:g.[1000A>G;1002T>C]" + ) + + +@pytest.mark.parametrize( + "expression", + [ + "NC_000001.11:g.[1000A>G;1002T>C]", + "NM_000001.1:c.[1A>G;3T>C;5G>A]", + ], +) +def test_join_is_inverse_of_split(expression): + assert join_cis_phased_hgvs(split_cis_phased_hgvs(expression)) == expression + + +def test_join_cis_phased_hgvs_returns_none_for_empty(): + assert join_cis_phased_hgvs([]) is None + + +def test_join_cis_phased_hgvs_returns_none_for_mixed_accessions(): + # Members on different sequences are not one cis-phased block. + assert join_cis_phased_hgvs(["NC_000001.11:g.1A>G", "NC_000002.12:g.2T>C"]) is None + + +def test_join_cis_phased_hgvs_returns_none_for_mixed_coordinate_prefixes(): + assert join_cis_phased_hgvs(["NM_000001.1:c.1A>G", "NM_000001.1:g.2T>C"]) is None + + +def test_join_cis_phased_hgvs_returns_none_for_component_without_accession(): + assert join_cis_phased_hgvs(["g.1A>G", "g.2T>C"]) is None From 0e6246413d65e5961057405fd96c8b2078866eac Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Sat, 6 Jun 2026 18:49:11 -0700 Subject: [PATCH 009/252] feat(vrs): translate cis-phased multivariant HGVS into VRS blocks Add VRS translation support for cis-phased multivariant HGVS, building on the new hgvs split helpers. - translate_hgvs_to_variation translates each component HGVS to an Allele independently, returning a bare Allele for a single component and wrapping two or more in a CisPhasedBlock, mirroring dcd_mapping's vrs_map._construct_vrs_allele; the reverse-translation job emits bracketed genomic forms the AlleleTranslator cannot translate directly - identify_variation generalizes identify_allele to blocks, clearing every member and location digest plus the block's own before identifying so a stale member digest never propagates into the block id; the block digest is order-independent so a set dedups to one row Cover both with unit tests, including the order-independent block digest and the stale-digest clearing. --- src/mavedb/lib/vrs_utils.py | 55 +++++++++++++++++++++++- tests/lib/test_vrs_utils.py | 86 +++++++++++++++++++++++++++++++++++++ 2 files changed, 140 insertions(+), 1 deletion(-) create mode 100644 tests/lib/test_vrs_utils.py diff --git a/src/mavedb/lib/vrs_utils.py b/src/mavedb/lib/vrs_utils.py index 2720afa68..7872f2ed6 100644 --- a/src/mavedb/lib/vrs_utils.py +++ b/src/mavedb/lib/vrs_utils.py @@ -13,9 +13,11 @@ from ga4gh.core import ga4gh_identify from ga4gh.vrs.extras.translator import AlleleTranslator -from ga4gh.vrs.models import Allele, LiteralSequenceExpression, SequenceLocation +from ga4gh.vrs.models import Allele, CisPhasedBlock, LiteralSequenceExpression, SequenceLocation from ga4gh.vrs.normalize import normalize +from mavedb.lib.hgvs import split_cis_phased_hgvs + def translate_hgvs_to_vrs(hgvs: str, translator: AlleleTranslator) -> Allele: """Convert HGVS variation description to VRS object. @@ -47,6 +49,32 @@ def translate_hgvs_to_vrs(hgvs: str, translator: AlleleTranslator) -> Allele: return allele +def translate_hgvs_to_variation(hgvs: str, translator: AlleleTranslator) -> Allele | CisPhasedBlock: + """Translate an HGVS expression — possibly a cis-phased multivariant — into a VRS object. + + Mirrors dcd_mapping's ``vrs_map._construct_vrs_allele``: each component HGVS is translated + to an Allele independently; a single component returns a bare Allele, while two or more are + wrapped in a CisPhasedBlock. The reverse-translation job emits bracketed genomic forms + (``g.[a;b]``) for non-adjacent codon components that ga4gh's AlleleTranslator cannot + translate directly, so splitting and recombining is the only way to represent them. + + The block's GA4GH digest is order-independent, so the same biological cis-phased set always + identifies to one ``ga4gh:CPB.`` digest and dedups to a single row regardless of component + ordering. + + :param hgvs: a single- or cis-phased-multivariant HGVS string + :param translator: caller-owned AlleleTranslator reused across calls + :return: an Allele for a single variant, or a CisPhasedBlock for a cis-phased set + """ + members = [translate_hgvs_to_vrs(component, translator) for component in split_cis_phased_hgvs(hgvs)] + if len(members) == 1: + return members[0] + + block = CisPhasedBlock(members=members) # type: ignore[call-arg] + block.id = identify_variation(block) + return block + + def identify_allele(allele: Allele) -> str: """Clear cached digests and return a fresh GA4GH identifier for *allele*. @@ -67,6 +95,31 @@ def identify_allele(allele: Allele) -> str: return digest +def identify_variation(variation: Allele | CisPhasedBlock) -> str: + """Clear cached digests and return a fresh GA4GH id for an Allele or CisPhasedBlock. + + Generalizes :func:`identify_allele` to cis-phased blocks. A block's Merkle digest is + derived from its members' digests, so a stale member digest would silently propagate into + the block id. Clear every member (and its location) plus the block itself before + identifying so the id always reflects current content. + """ + if isinstance(variation, Allele): + return identify_allele(variation) + + for member in variation.members: + if isinstance(member, Allele): + if isinstance(member.location, SequenceLocation): + member.location.digest = None + member.digest = None + + variation.digest = None + digest = ga4gh_identify(variation) + if digest is None: + raise ValueError("Failed to compute GA4GH identifier for variation") # noqa: EM101 + + return digest + + def normalize_and_identify(allele: Allele, data_proxy: Any) -> Allele: """Normalize *allele* and stamp it with a freshly computed GA4GH digest. diff --git a/tests/lib/test_vrs_utils.py b/tests/lib/test_vrs_utils.py new file mode 100644 index 000000000..6f2bd382f --- /dev/null +++ b/tests/lib/test_vrs_utils.py @@ -0,0 +1,86 @@ +# ruff: noqa: E402 + +import pytest + +pytest.importorskip("ga4gh.vrs") + +from ga4gh.vrs.models import ( + Allele, + CisPhasedBlock, + LiteralSequenceExpression, + SequenceLocation, + SequenceReference, +) + +from mavedb.lib import vrs_utils +from mavedb.lib.vrs_utils import identify_variation, translate_hgvs_to_variation + +_SQ = "SQ." + "a" * 32 + + +def _allele(start: int, alt: str) -> Allele: + return Allele( + location=SequenceLocation( + sequenceReference=SequenceReference(refgetAccession=_SQ), + start=start, + end=start + 1, + ), + state=LiteralSequenceExpression(sequence=alt), + ) + + +def _patch_component_translator(monkeypatch, alleles_by_hgvs: dict[str, Allele]) -> None: + """Stub the per-component single-allele translator that translate_hgvs_to_variation calls.""" + + def _fake(hgvs: str, translator=None) -> Allele: + return alleles_by_hgvs[hgvs] + + monkeypatch.setattr(vrs_utils, "translate_hgvs_to_vrs", _fake) + + +def test_single_variant_returns_a_bare_allele(monkeypatch): + allele = _allele(1000, "G") + _patch_component_translator(monkeypatch, {"NC_000001.11:g.1000A>G": allele}) + + result = translate_hgvs_to_variation("NC_000001.11:g.1000A>G", translator=None) + + assert isinstance(result, Allele) + assert result is allele + + +def test_cis_phased_multivariant_returns_an_identified_block(monkeypatch): + members = { + "NC_000001.11:g.1000A>G": _allele(1000, "G"), + "NC_000001.11:g.1002T>C": _allele(1002, "C"), + } + _patch_component_translator(monkeypatch, members) + + result = translate_hgvs_to_variation("NC_000001.11:g.[1000A>G;1002T>C]", translator=None) + + assert isinstance(result, CisPhasedBlock) + assert len(result.members) == 2 + assert result.id is not None and result.id.startswith("ga4gh:CPB.") + + +def test_cis_phased_block_digest_is_order_independent(monkeypatch): + members = { + "NC_000001.11:g.1000A>G": _allele(1000, "G"), + "NC_000001.11:g.1002T>C": _allele(1002, "C"), + } + _patch_component_translator(monkeypatch, members) + + forward = translate_hgvs_to_variation("NC_000001.11:g.[1000A>G;1002T>C]", translator=None) + reverse = translate_hgvs_to_variation("NC_000001.11:g.[1002T>C;1000A>G]", translator=None) + + # The same biological cis-phased set dedups to one row regardless of component ordering. + assert forward.id == reverse.id + + +def test_identify_variation_clears_stale_block_digest(): + block = CisPhasedBlock(members=[_allele(1000, "G"), _allele(1002, "C")]) + block.digest = "STALE" + + digest = identify_variation(block) + + assert digest != "STALE" + assert digest.startswith("ga4gh:CPB.") # recomputed from content, not the stale cache From ce510c33d527aa9e4cf1fca3ce472116676cd913 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Sat, 6 Jun 2026 18:53:03 -0700 Subject: [PATCH 010/252] feat(variants): combine cis-phased members into one HGVS expression MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit get_hgvs_from_post_mapped can now join the members of a multi-variant block (Haplotype/CisPhasedBlock) into a single bracketed cis-phased expression via the new combine_cis flag, replacing the previous behavior of returning None for any multi-variant block. - combine_cis defaults off because some consumers cannot yet handle a bracketed expression — notably ClinGen submission, which has no single CAID for a multi-variant cis block (#764) - the CSV export fallback opts in, so g./p. HGVS columns are populated from cis-phased post-mapped output instead of left empty - drop the stale commented-out error branches and the resolved TODO --- src/mavedb/lib/score_sets.py | 8 ++++++-- src/mavedb/lib/variants.py | 17 +++++++++++------ .../worker/jobs/external_services/clingen.py | 5 +++++ 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/src/mavedb/lib/score_sets.py b/src/mavedb/lib/score_sets.py index 8e3c8debb..2acfa26f7 100644 --- a/src/mavedb/lib/score_sets.py +++ b/src/mavedb/lib/score_sets.py @@ -906,7 +906,9 @@ def variant_to_csv_row( value = str(mapping.hgvs_g) if mapping and mapping.hgvs_g else na_rep if value == na_rep: fallback_hgvs = ( - get_hgvs_from_post_mapped(mapping.post_mapped) if mapping and mapping.post_mapped else None + get_hgvs_from_post_mapped(mapping.post_mapped, combine_cis=True) + if mapping and mapping.post_mapped + else None ) if fallback_hgvs is not None and is_hgvs_g(fallback_hgvs): value = fallback_hgvs @@ -917,7 +919,9 @@ def variant_to_csv_row( value = str(mapping.hgvs_p) if mapping and mapping.hgvs_p else na_rep if value == na_rep: fallback_hgvs = ( - get_hgvs_from_post_mapped(mapping.post_mapped) if mapping and mapping.post_mapped else None + get_hgvs_from_post_mapped(mapping.post_mapped, combine_cis=True) + if mapping and mapping.post_mapped + else None ) if fallback_hgvs is not None and is_hgvs_p(fallback_hgvs): value = fallback_hgvs diff --git a/src/mavedb/lib/variants.py b/src/mavedb/lib/variants.py index bef9725c0..a941006cf 100644 --- a/src/mavedb/lib/variants.py +++ b/src/mavedb/lib/variants.py @@ -1,6 +1,7 @@ import re from typing import Any, Optional +from mavedb.lib.hgvs import join_cis_phased_hgvs from mavedb.lib.validation.constants.general import hgvs_columns from mavedb.models.target_gene import TargetGene from mavedb.models.variant import Variant @@ -25,7 +26,15 @@ def hgvs_from_vrs_allele(allele: dict) -> str: raise KeyError("Invalid VRS allele structure. Expected 'expressions'.") -def get_hgvs_from_post_mapped(post_mapped_vrs: Optional[Any]) -> Optional[str]: +def get_hgvs_from_post_mapped(post_mapped_vrs: Optional[Any], *, combine_cis: bool = False) -> Optional[str]: + """Extract a single HGVS string from a post-mapped VRS object. + + Multi-variant blocks (Haplotype/CisPhasedBlock) are cis-phased, so their members combine + into one bracketed expression (``NC_…:g.[a;b]``) when ``combine_cis`` is set. It defaults + off because some consumers cannot yet handle a bracketed expression — notably ClinGen + submission, which has no single CAID for a multi-variant cis block (see + https://github.com/VariantEffect/mavedb-api/issues/764). + """ if not post_mapped_vrs: return None @@ -40,13 +49,9 @@ def get_hgvs_from_post_mapped(post_mapped_vrs: Optional[Any]) -> Optional[str]: if len(variations_hgvs) == 0: return None - # raise ValueError(f"No variations found in variant {variant_urn}.") - # TODO (https://github.com/VariantEffect/mavedb-api/issues/468) In a future version, we will be able to generate - # a combined HGVS string for haplotypes and cis phased blocks directly from mapper output. if len(variations_hgvs) > 1: - return None - # raise ValueError(f"Multiple variations found in variant {variant_urn}.") + return join_cis_phased_hgvs(variations_hgvs) if combine_cis else None return variations_hgvs[0] diff --git a/src/mavedb/worker/jobs/external_services/clingen.py b/src/mavedb/worker/jobs/external_services/clingen.py index 639160ed5..5a65cc728 100644 --- a/src/mavedb/worker/jobs/external_services/clingen.py +++ b/src/mavedb/worker/jobs/external_services/clingen.py @@ -137,6 +137,9 @@ async def submit_score_set_mappings_to_car(ctx: dict, job_id: int, job_manager: variant_post_mapped_hgvs: dict[str, list[int]] = {} no_hgvs_count = 0 for mapped_variant_id, post_mapped in variant_post_mapped_objects: + # Intentionally not combine_cis=True: multi-variant cis-phased blocks have no single + # CAID, so they are skipped here pending ClinGen guidance on how to register them + # (https://github.com/VariantEffect/mavedb-api/issues/764). hgvs_for_post_mapped = get_hgvs_from_post_mapped(post_mapped) if not hgvs_for_post_mapped: @@ -357,6 +360,8 @@ async def submit_score_set_mappings_to_ldh(ctx: dict, job_id: int, job_manager: variant_content = [] variant_for_urn = {} for variant, mapped_variant in variant_objects: + # See the note above: cis-phased blocks are skipped here pending ClinGen guidance + # (https://github.com/VariantEffect/mavedb-api/issues/764). variation = get_hgvs_from_post_mapped(mapped_variant.post_mapped) if not variation: From 058ff3f293d5203783e94aa176cc7f07ffcf8e5d Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Sat, 6 Jun 2026 18:54:54 -0700 Subject: [PATCH 011/252] feat(db): add VRS allele closure tables for reverse translation Introduce parallel mapping tables for the Better Reverse Translation epic (#746). The existing mapped_variants table is left untouched (frozen serving) while the new schema is built out separately. - add alleles, mapping_records, and mapping_record_alleles tables with their ORM models; alleles are content-addressed by vrs_digest and shared across mapping records via the link table - mapping_record_alleles.is_authoritative distinguishes the assay's actual measurement from translator-derived links, since the same VRS allele can be authoritative for one record and derived for another - put mapping_records and mapping_record_alleles on valid-time versioning (ValidTime mixin): a re-map retires the prior live row by closing valid_to instead of deleting, so history is retained and point-in-time queries are a single predicate; partial unique indexes promote "one live row per key" to the database - derive Allele.transcript and MappingRecord.transcript as hybrid properties from the HGVS columns rather than storing them, so they cannot drift; drop the stored alleles.transcript column - add the cross_level_translation annotation type, written once per variant to record whether filling unmapped levels succeeded, was skipped, or failed - annotate Variant and TargetGeneMapping with mapping_records relationships and typed primary keys --- ...2c3d4e5f6_add_vrs_allele_closure_tables.py | 151 ++++++++++++++++++ ...f0a2c4d7_drop_alleles_transcript_column.py | 32 ++++ ...7f9a1b2_temporal_mapping_record_alleles.py | 48 ++++++ .../d4e6f8a0b2c3_temporal_mapping_records.py | 76 +++++++++ ...cross_level_translation_annotation_type.py | 49 ++++++ src/mavedb/models/__init__.py | 3 + src/mavedb/models/allele.py | 59 +++++++ src/mavedb/models/enums/annotation_type.py | 1 + src/mavedb/models/mapping_record.py | 96 +++++++++++ src/mavedb/models/mapping_record_allele.py | 60 +++++++ src/mavedb/models/target_gene_mapping.py | 10 +- src/mavedb/models/variant.py | 7 +- .../models/variant_annotation_status.py | 2 +- 13 files changed, 590 insertions(+), 4 deletions(-) create mode 100644 alembic/versions/a1b2c3d4e5f6_add_vrs_allele_closure_tables.py create mode 100644 alembic/versions/b8e1f0a2c4d7_drop_alleles_transcript_column.py create mode 100644 alembic/versions/c3d5e7f9a1b2_temporal_mapping_record_alleles.py create mode 100644 alembic/versions/d4e6f8a0b2c3_temporal_mapping_records.py create mode 100644 alembic/versions/f4d2a9c1b7e3_add_cross_level_translation_annotation_type.py create mode 100644 src/mavedb/models/allele.py create mode 100644 src/mavedb/models/mapping_record.py create mode 100644 src/mavedb/models/mapping_record_allele.py diff --git a/alembic/versions/a1b2c3d4e5f6_add_vrs_allele_closure_tables.py b/alembic/versions/a1b2c3d4e5f6_add_vrs_allele_closure_tables.py new file mode 100644 index 000000000..49c502d05 --- /dev/null +++ b/alembic/versions/a1b2c3d4e5f6_add_vrs_allele_closure_tables.py @@ -0,0 +1,151 @@ +"""Add mapping_records, alleles, and mapping_record_alleles tables + +Revision ID: a1b2c3d4e5f6 +Revises: 398067c53257 +Create Date: 2026-05-29 + +New parallel tables for the Better Reverse Translation epic (#746). +The existing mapped_variants table is left untouched (frozen serving). +""" + +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "a1b2c3d4e5f6" +down_revision = "398067c53257" +branch_labels = None +depends_on = None + +VALID_ASSAY_LEVELS = "('genomic', 'cdna', 'protein')" +VALID_ALIGNMENT_LEVELS = "('protein', 'cdna', 'genomic')" + + +def upgrade() -> None: + op.create_table( + "alleles", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("vrs_digest", sa.String(), nullable=False), + sa.Column("level", sa.String(length=16), nullable=False), + sa.Column("transcript", sa.String(), nullable=False), + sa.Column("hgvs_g", sa.String(), nullable=True), + sa.Column("hgvs_c", sa.String(), nullable=True), + sa.Column("hgvs_p", sa.String(), nullable=True), + sa.Column("clingen_allele_id", sa.String(), nullable=True), + sa.Column("post_mapped", postgresql.JSONB(), nullable=True), + sa.Column("created_at", sa.Date(), nullable=False, server_default=sa.text("CURRENT_DATE")), + sa.Column( + "updated_at", + sa.Date(), + nullable=False, + server_default=sa.text("CURRENT_DATE"), + onupdate=sa.text("CURRENT_DATE"), + ), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("vrs_digest", name="uq_alleles_vrs_digest"), + ) + op.create_index("ix_alleles_vrs_digest", "alleles", ["vrs_digest"]) + op.create_index("ix_alleles_level", "alleles", ["level"]) + op.create_index("ix_alleles_clingen_allele_id", "alleles", ["clingen_allele_id"]) + + op.create_table( + "mapping_records", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("variant_id", sa.Integer(), nullable=False), + sa.Column("vrs_digest", sa.String(), nullable=True), + sa.Column("pre_mapped", postgresql.JSONB(), nullable=True), + sa.Column("assay_level", sa.String(length=16), nullable=False), + sa.Column("hgvs_assay_level", sa.String(), nullable=True), + sa.Column("mapping_api_version", sa.String(), nullable=False), + sa.Column("mapped_date", sa.Date(), nullable=False), + sa.Column("vrs_version", sa.String(), nullable=True), + sa.Column("current", sa.Boolean(), nullable=False), + sa.Column("alignment_level", sa.String(length=16), nullable=True), + sa.Column("at_mismatched_locus", sa.Boolean(), nullable=True), + sa.Column("near_gap", sa.Boolean(), nullable=True), + sa.Column("target_gene_mapping_id", sa.Integer(), nullable=True), + sa.Column("created_at", sa.Date(), nullable=False, server_default=sa.text("CURRENT_DATE")), + sa.Column( + "updated_at", + sa.Date(), + nullable=False, + server_default=sa.text("CURRENT_DATE"), + onupdate=sa.text("CURRENT_DATE"), + ), + sa.ForeignKeyConstraint( + ["variant_id"], + ["variants.id"], + name="fk_mapping_records_variant_id", + ), + sa.ForeignKeyConstraint( + ["target_gene_mapping_id"], + ["target_gene_mappings.id"], + name="fk_mapping_records_target_gene_mapping_id", + ), + sa.PrimaryKeyConstraint("id"), + sa.CheckConstraint( + f"assay_level IN {VALID_ASSAY_LEVELS}", + name="ck_mapping_records_assay_level_valid", + ), + ) + op.create_index("ix_mapping_records_variant_id", "mapping_records", ["variant_id"]) + op.create_index("ix_mapping_records_vrs_digest", "mapping_records", ["vrs_digest"]) + op.create_index( + "ix_mapping_records_target_gene_mapping_id", + "mapping_records", + ["target_gene_mapping_id"], + ) + + op.create_table( + "mapping_record_alleles", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("mapping_record_id", sa.Integer(), nullable=False), + sa.Column("allele_id", sa.Integer(), nullable=False), + sa.Column( + "is_authoritative", + sa.Boolean(), + nullable=False, + server_default=sa.text("false"), + ), + sa.ForeignKeyConstraint( + ["mapping_record_id"], + ["mapping_records.id"], + name="fk_mapping_record_alleles_mapping_record_id", + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["allele_id"], + ["alleles.id"], + name="fk_mapping_record_alleles_allele_id", + ondelete="RESTRICT", + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + "ix_mapping_record_alleles_mapping_record_id", + "mapping_record_alleles", + ["mapping_record_id"], + ) + op.create_index( + "ix_mapping_record_alleles_allele_id", + "mapping_record_alleles", + ["allele_id"], + ) + + +def downgrade() -> None: + op.drop_index("ix_mapping_record_alleles_allele_id", table_name="mapping_record_alleles") + op.drop_index("ix_mapping_record_alleles_mapping_record_id", table_name="mapping_record_alleles") + op.drop_table("mapping_record_alleles") + + op.drop_index("ix_mapping_records_target_gene_mapping_id", table_name="mapping_records") + op.drop_index("ix_mapping_records_vrs_digest", table_name="mapping_records") + op.drop_index("ix_mapping_records_variant_id", table_name="mapping_records") + op.drop_table("mapping_records") + + op.drop_index("ix_alleles_clingen_allele_id", table_name="alleles") + op.drop_index("ix_alleles_level", table_name="alleles") + op.drop_index("ix_alleles_vrs_digest", table_name="alleles") + op.drop_table("alleles") diff --git a/alembic/versions/b8e1f0a2c4d7_drop_alleles_transcript_column.py b/alembic/versions/b8e1f0a2c4d7_drop_alleles_transcript_column.py new file mode 100644 index 000000000..2705d9a3c --- /dev/null +++ b/alembic/versions/b8e1f0a2c4d7_drop_alleles_transcript_column.py @@ -0,0 +1,32 @@ +"""drop alleles.transcript column + +Revision ID: b8e1f0a2c4d7 +Revises: f4d2a9c1b7e3 +Create Date: 2026-06-05 + +The `transcript` column duplicated data already present in the HGVS columns — it was +always extract_accession(hgvs_g/hgvs_c/hgvs_p). It is now a derived hybrid_property on +the Allele model (split_part(coalesce(hgvs_g, hgvs_c, hgvs_p), ':', 1)), so the stored +column is removed to keep a single source of truth and avoid drift. +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "b8e1f0a2c4d7" +down_revision = "f4d2a9c1b7e3" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.drop_column("alleles", "transcript") + + +def downgrade() -> None: + # Re-add the column and backfill it from the HGVS columns (the same derivation the + # hybrid_property uses) so the restored NOT NULL column is consistent. + op.add_column("alleles", sa.Column("transcript", sa.String(), nullable=True)) + op.execute("UPDATE alleles SET transcript = split_part(coalesce(hgvs_g, hgvs_c, hgvs_p), ':', 1)") + op.alter_column("alleles", "transcript", nullable=False) diff --git a/alembic/versions/c3d5e7f9a1b2_temporal_mapping_record_alleles.py b/alembic/versions/c3d5e7f9a1b2_temporal_mapping_record_alleles.py new file mode 100644 index 000000000..26d3d6dbf --- /dev/null +++ b/alembic/versions/c3d5e7f9a1b2_temporal_mapping_record_alleles.py @@ -0,0 +1,48 @@ +"""add valid-time versioning to mapping_record_alleles + +Revision ID: c3d5e7f9a1b2 +Revises: b8e1f0a2c4d7 +Create Date: 2026-06-05 + +Make the link table valid-time versioned (TemporalLink): a link is live while valid_to is +NULL, and superseding it closes valid_to instead of deleting, so reverse translation can be +re-run independently while prior derivations remain queryable point-in-time. The partial +unique index enforces a single live link per (mapping_record, allele). + +Assumes no pre-existing duplicate live links — true for these parallel tables, which are +new-only writes and not yet serving. If this ever runs against data with duplicates, the +unique index creation will fail and the duplicates must be retired first. +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "c3d5e7f9a1b2" +down_revision = "b8e1f0a2c4d7" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column( + "mapping_record_alleles", + sa.Column("valid_from", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + ) + op.add_column( + "mapping_record_alleles", + sa.Column("valid_to", sa.DateTime(timezone=True), nullable=True), + ) + op.create_index( + "uq_mapping_record_alleles_live", + "mapping_record_alleles", + ["mapping_record_id", "allele_id"], + unique=True, + postgresql_where=sa.text("valid_to IS NULL"), + ) + + +def downgrade() -> None: + op.drop_index("uq_mapping_record_alleles_live", table_name="mapping_record_alleles") + op.drop_column("mapping_record_alleles", "valid_to") + op.drop_column("mapping_record_alleles", "valid_from") diff --git a/alembic/versions/d4e6f8a0b2c3_temporal_mapping_records.py b/alembic/versions/d4e6f8a0b2c3_temporal_mapping_records.py new file mode 100644 index 000000000..603c171a5 --- /dev/null +++ b/alembic/versions/d4e6f8a0b2c3_temporal_mapping_records.py @@ -0,0 +1,76 @@ +"""move mapping_records onto valid-time versioning + +Revision ID: d4e6f8a0b2c3 +Revises: c3d5e7f9a1b2 +Create Date: 2026-06-05 + +Replace the stored `current` flag and the `created_at`/`updated_at` audit dates with valid-time +columns (ValidTime mixin): a mapping record is live while valid_to is NULL, and a re-map retires +the prior version (closing valid_to) instead of flipping a boolean. `current` becomes derived +(valid_to IS NULL). `mapped_date` (the date the mapping was performed) is domain data and stays. + +The partial unique index promotes to the database the "one live mapping record per variant" +invariant the mapping job previously enforced only in app code. + +Backfills from the columns being dropped, so existing rows keep their validity. Assumes no +duplicate live records per variant (true for these pre-cutover parallel tables; otherwise the +unique index creation fails and the duplicates must be retired first). +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "d4e6f8a0b2c3" +down_revision = "c3d5e7f9a1b2" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column( + "mapping_records", + sa.Column("valid_from", sa.DateTime(timezone=True), nullable=True, server_default=sa.func.now()), + ) + op.add_column( + "mapping_records", + sa.Column("valid_to", sa.DateTime(timezone=True), nullable=True), + ) + + # Backfill validity from the columns being replaced: a record's life began at created_at, and + # a non-current record was retired at updated_at (the only in-place update it ever took). + op.execute("UPDATE mapping_records SET valid_from = created_at::timestamptz") + op.execute("UPDATE mapping_records SET valid_to = updated_at::timestamptz WHERE current = false") + + op.alter_column("mapping_records", "valid_from", nullable=False) + + op.drop_column("mapping_records", "current") + op.drop_column("mapping_records", "created_at") + op.drop_column("mapping_records", "updated_at") + + op.create_index( + "uq_mapping_records_current", + "mapping_records", + ["variant_id"], + unique=True, + postgresql_where=sa.text("valid_to IS NULL"), + ) + + +def downgrade() -> None: + op.drop_index("uq_mapping_records_current", table_name="mapping_records") + + op.add_column("mapping_records", sa.Column("current", sa.Boolean(), nullable=True)) + op.add_column("mapping_records", sa.Column("created_at", sa.Date(), nullable=True)) + op.add_column("mapping_records", sa.Column("updated_at", sa.Date(), nullable=True)) + + op.execute("UPDATE mapping_records SET current = (valid_to IS NULL)") + op.execute("UPDATE mapping_records SET created_at = valid_from::date") + op.execute("UPDATE mapping_records SET updated_at = coalesce(valid_to, valid_from)::date") + + op.alter_column("mapping_records", "current", nullable=False) + op.alter_column("mapping_records", "created_at", nullable=False) + op.alter_column("mapping_records", "updated_at", nullable=False) + + op.drop_column("mapping_records", "valid_to") + op.drop_column("mapping_records", "valid_from") diff --git a/alembic/versions/f4d2a9c1b7e3_add_cross_level_translation_annotation_type.py b/alembic/versions/f4d2a9c1b7e3_add_cross_level_translation_annotation_type.py new file mode 100644 index 000000000..790659d08 --- /dev/null +++ b/alembic/versions/f4d2a9c1b7e3_add_cross_level_translation_annotation_type.py @@ -0,0 +1,49 @@ +"""add cross_level_translation annotation type + +Revision ID: f4d2a9c1b7e3 +Revises: a1b2c3d4e5f6 +Create Date: 2026-06-02 + +Extends ck_variant_annotation_type_valid to allow the 'cross_level_translation' +annotation type. The VRS mapping worker writes one such row per variant to record +whether cross-level translation (filling the levels the assay did not map) +succeeded, was skipped (multivariant / no transcript), or failed. +""" + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "f4d2a9c1b7e3" +down_revision = "a1b2c3d4e5f6" +branch_labels = None +depends_on = None + +_TYPES_OLD = ( + "'vrs_mapping', 'clingen_allele_id', 'mapped_hgvs', 'variant_translation', " + "'gnomad_allele_frequency', 'clinvar_control', 'vep_functional_consequence', " + "'ldh_submission'" +) +_TYPES_NEW = "'vrs_mapping', 'cross_level_translation', " + ( + "'clingen_allele_id', 'mapped_hgvs', 'variant_translation', " + "'gnomad_allele_frequency', 'clinvar_control', 'vep_functional_consequence', " + "'ldh_submission'" +) + + +def upgrade() -> None: + op.drop_constraint("ck_variant_annotation_type_valid", "variant_annotation_status", type_="check") + op.create_check_constraint( + "ck_variant_annotation_type_valid", + "variant_annotation_status", + f"annotation_type IN ({_TYPES_NEW})", + ) + + +def downgrade() -> None: + op.execute("DELETE FROM variant_annotation_status WHERE annotation_type = 'cross_level_translation'") + op.drop_constraint("ck_variant_annotation_type_valid", "variant_annotation_status", type_="check") + op.create_check_constraint( + "ck_variant_annotation_type_valid", + "variant_annotation_status", + f"annotation_type IN ({_TYPES_OLD})", + ) diff --git a/src/mavedb/models/__init__.py b/src/mavedb/models/__init__.py index 2f0d65b48..35c1ba0a5 100644 --- a/src/mavedb/models/__init__.py +++ b/src/mavedb/models/__init__.py @@ -1,6 +1,7 @@ __all__ = [ "access_key", "acmg_classification", + "allele", "collection", "clinical_control", "controlled_keyword", @@ -16,6 +17,8 @@ "legacy_keyword", "license", "mapped_variant", + "mapping_record", + "mapping_record_allele", "pipeline", "publication_identifier", "published_variant", diff --git a/src/mavedb/models/allele.py b/src/mavedb/models/allele.py new file mode 100644 index 000000000..54ce07e50 --- /dev/null +++ b/src/mavedb/models/allele.py @@ -0,0 +1,59 @@ +from datetime import date +from typing import TYPE_CHECKING, Any, Optional + +from sqlalchemy import Column, Date, Index, Integer, String, UniqueConstraint, func +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.ext.hybrid import hybrid_property +from sqlalchemy.orm import Mapped, relationship + +from mavedb.db.base import Base +from mavedb.lib.hgvs import extract_accession + +if TYPE_CHECKING: + from .mapping_record_allele import MappingRecordAllele + + +class Allele(Base): + __tablename__ = "alleles" + + id: Mapped[int] = Column(Integer, primary_key=True) + + vrs_digest = Column(String, nullable=False) + level = Column(String(length=16), nullable=False) + + hgvs_g = Column(String, nullable=True) + hgvs_c = Column(String, nullable=True) + hgvs_p = Column(String, nullable=True) + + clingen_allele_id = Column(String, nullable=True) + post_mapped: Optional[Any] = Column(JSONB(none_as_null=True), nullable=True) + + created_at = Column(Date, nullable=False, default=date.today) + updated_at = Column(Date, nullable=False, default=date.today, onupdate=date.today) + + @hybrid_property + def transcript(self) -> str: + """Reference accession of the populated HGVS column (derived, not stored). + + Exactly one of hgvs_g/hgvs_c/hgvs_p is populated per allele, and the transcript is that + string's accession. Derived rather than stored so it cannot drift from the HGVS column + it duplicates. + """ + return extract_accession(self.hgvs_g or self.hgvs_c or self.hgvs_p or "") + + @transcript.inplace.expression + @classmethod + def _transcript_expression(cls): + return func.split_part(func.coalesce(cls.hgvs_g, cls.hgvs_c, cls.hgvs_p), ":", 1) + + mapping_record_links: Mapped[list["MappingRecordAllele"]] = relationship( + "MappingRecordAllele", + back_populates="allele", + ) + + __table_args__ = ( + UniqueConstraint("vrs_digest", name="uq_alleles_vrs_digest"), + Index("ix_alleles_vrs_digest", "vrs_digest"), + Index("ix_alleles_level", "level"), + Index("ix_alleles_clingen_allele_id", "clingen_allele_id"), + ) diff --git a/src/mavedb/models/enums/annotation_type.py b/src/mavedb/models/enums/annotation_type.py index b1595347b..a3bbd1693 100644 --- a/src/mavedb/models/enums/annotation_type.py +++ b/src/mavedb/models/enums/annotation_type.py @@ -3,6 +3,7 @@ class AnnotationType(str, Enum): VRS_MAPPING = "vrs_mapping" + CROSS_LEVEL_TRANSLATION = "cross_level_translation" CLINGEN_ALLELE_ID = "clingen_allele_id" MAPPED_HGVS = "mapped_hgvs" VARIANT_TRANSLATION = "variant_translation" diff --git a/src/mavedb/models/mapping_record.py b/src/mavedb/models/mapping_record.py new file mode 100644 index 000000000..132a69b8b --- /dev/null +++ b/src/mavedb/models/mapping_record.py @@ -0,0 +1,96 @@ +from datetime import date +from typing import TYPE_CHECKING, Any, Optional + +from sqlalchemy import Boolean, Column, Date, Enum, ForeignKey, Index, Integer, String, func, text +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.ext.hybrid import hybrid_property +from sqlalchemy.orm import Mapped, relationship + +from mavedb.db.base import Base +from mavedb.db.mixins import ValidTime +from mavedb.lib.hgvs import extract_accession +from mavedb.models.enums.annotation_layer import AnnotationLayer + +if TYPE_CHECKING: + from .mapping_record_allele import MappingRecordAllele + from .target_gene_mapping import TargetGeneMapping + from .variant import Variant + + +class MappingRecord(ValidTime, Base): + __tablename__ = "mapping_records" + + id: Mapped[int] = Column(Integer, primary_key=True) + + variant_id = Column(Integer, ForeignKey("variants.id"), nullable=False) + variant: Mapped["Variant"] = relationship("Variant", back_populates="mapping_records") + + # Digest of the pre-mapped (assayed-level) VRS representation, indexed for + # cross-score-set dedup lookups. + vrs_digest = Column(String, nullable=True) + pre_mapped: Optional[Any] = Column(JSONB(none_as_null=True), nullable=True) + + # Level at which the variant was *assayed* — distinct from alignment_level (QC). + assay_level = Column(String(length=16), nullable=False) + hgvs_assay_level = Column(String, nullable=True) + + @hybrid_property + def transcript(self) -> str: + """Reference accession of the assay-level HGVS (derived, not stored). + + Derived rather than stored so it cannot drift from hgvs_assay_level, which already + carries the accession. + """ + return extract_accession(self.hgvs_assay_level or "") + + @transcript.inplace.expression + @classmethod + def _transcript_expression(cls): + return func.split_part(cls.hgvs_assay_level, ":", 1) + + mapping_api_version = Column(String, nullable=False) + # Domain data: the date the mapping was performed (from the dcd-mapping run), surfaced to + # users — distinct from valid_from, which is when this version became the live record. + mapped_date = Column(Date, nullable=False, default=date.today) + vrs_version = Column(String, nullable=True) + + # valid_from / valid_to and the derived `current` come from ValidTime: a re-map retires the + # prior live record (closes its valid_to) and inserts a new live one, so mapping history is + # retained and point-in-time queries are a single predicate. + + # Per-mapping QC fields from dcd-mapping. + alignment_level = Column( + Enum(AnnotationLayer, create_constraint=True, length=16, native_enum=False, validate_strings=True), + nullable=True, + ) + at_mismatched_locus = Column(Boolean, nullable=True) + near_gap = Column(Boolean, nullable=True) + + target_gene_mapping_id = Column(Integer, ForeignKey("target_gene_mappings.id"), index=True, nullable=True) + target_gene_mapping: Mapped[Optional["TargetGeneMapping"]] = relationship( + "TargetGeneMapping", back_populates="mapping_records" + ) + + allele_links: Mapped[list["MappingRecordAllele"]] = relationship( + "MappingRecordAllele", + back_populates="mapping_record", + cascade="all, delete-orphan", + ) + + # Retiring a record retires its live allele links too (both the authoritative link and any + # derived links a reverse-translation run attached). See ValidTime.__retire_cascade__. + __retire_cascade__ = ("allele_links",) + + __table_args__ = ( + Index("ix_mapping_records_vrs_digest", "vrs_digest"), + Index("ix_mapping_records_variant_id", "variant_id"), + # At most one live mapping record per variant — promotes to the database the invariant the + # mapping job enforces in app code (it retires the prior live record before inserting a new + # one). Superseded versions remain with a closed valid_to for point-in-time queries. + Index( + "uq_mapping_records_current", + "variant_id", + unique=True, + postgresql_where=text("valid_to IS NULL"), + ), + ) diff --git a/src/mavedb/models/mapping_record_allele.py b/src/mavedb/models/mapping_record_allele.py new file mode 100644 index 000000000..cda8bf0f3 --- /dev/null +++ b/src/mavedb/models/mapping_record_allele.py @@ -0,0 +1,60 @@ +from typing import TYPE_CHECKING + +from sqlalchemy import Boolean, Column, ForeignKey, Index, Integer, text +from sqlalchemy.orm import Mapped, relationship + +from mavedb.db.base import Base +from mavedb.db.mixins import ValidTime + +if TYPE_CHECKING: + from .allele import Allele + from .mapping_record import MappingRecord + + +class MappingRecordAllele(ValidTime, Base): + __tablename__ = "mapping_record_alleles" + + id: Mapped[int] = Column(Integer, primary_key=True) + mapping_record_id: Mapped[int] = Column( + Integer, + ForeignKey("mapping_records.id", ondelete="CASCADE"), + nullable=False, + ) + allele_id: Mapped[int] = Column( + Integer, + ForeignKey("alleles.id", ondelete="RESTRICT"), + nullable=False, + ) + # Provenance: ``True`` when this allele was the assay's actual measurement + # for this mapping record (sourced from vrs_map, with real alignment QC), + # ``False`` when it was derived by the cross-level or same-level + # synonymous translator. The flag lives on the link rather than on + # ``Allele`` because the same VRS allele can be authoritative for one + # mapping record and translator-derived for another. + is_authoritative = Column(Boolean, nullable=False, default=False) + + mapping_record: Mapped["MappingRecord"] = relationship("MappingRecord", back_populates="allele_links") + allele: Mapped["Allele"] = relationship("Allele", back_populates="mapping_record_links") + + __table_args__ = ( + Index( + "ix_mapping_record_alleles_mapping_record_id", + "mapping_record_id", + ), + Index( + "ix_mapping_record_alleles_allele_id", + "allele_id", + ), + # At most one live link per (mapping_record, allele). The job retires (closes valid_to) + # rather than deletes, so superseded links remain for point-in-time queries; only the + # live row participates in this constraint. A derived link is never written when an + # authoritative one already exists for the pair, so is_authoritative is intentionally + # absent from the key — a record links a given allele once. + Index( + "uq_mapping_record_alleles_live", + "mapping_record_id", + "allele_id", + unique=True, + postgresql_where=text("valid_to IS NULL"), + ), + ) diff --git a/src/mavedb/models/target_gene_mapping.py b/src/mavedb/models/target_gene_mapping.py index 417f249d4..52761e34b 100644 --- a/src/mavedb/models/target_gene_mapping.py +++ b/src/mavedb/models/target_gene_mapping.py @@ -23,15 +23,16 @@ if TYPE_CHECKING: from mavedb.models.mapped_variant import MappedVariant + from mavedb.models.mapping_record import MappingRecord from mavedb.models.target_gene import TargetGene class TargetGeneMapping(Base): __tablename__ = "target_gene_mappings" - id = Column(Integer, primary_key=True) + id: Mapped[int] = Column(Integer, primary_key=True) - target_gene_id = Column(Integer, ForeignKey("target_genes.id"), nullable=False, index=True) + target_gene_id: Mapped[int] = Column(Integer, ForeignKey("target_genes.id"), nullable=False, index=True) target_gene: Mapped["TargetGene"] = relationship("TargetGene", back_populates="target_gene_mappings") alignment_level = Column( @@ -80,4 +81,9 @@ class TargetGeneMapping(Base): back_populates="target_gene_mapping", ) + mapping_records: Mapped[list["MappingRecord"]] = relationship( + "MappingRecord", + back_populates="target_gene_mapping", + ) + __table_args__ = (Index("ix_target_gene_mappings_target_alignment", "target_gene_id", "alignment_level"),) diff --git a/src/mavedb/models/variant.py b/src/mavedb/models/variant.py index 59b6e729c..dd46c430c 100644 --- a/src/mavedb/models/variant.py +++ b/src/mavedb/models/variant.py @@ -9,13 +9,14 @@ if TYPE_CHECKING: from .mapped_variant import MappedVariant + from .mapping_record import MappingRecord from .score_set import ScoreSet class Variant(Base): __tablename__ = "variants" - id = Column(Integer, primary_key=True) + id: Mapped[int] = Column(Integer, primary_key=True) urn = Column(String(64), index=True, nullable=True, unique=True) data = Column(JSONB, nullable=False) @@ -35,5 +36,9 @@ class Variant(Base): back_populates="variant", cascade="all, delete-orphan" ) + mapping_records: Mapped[List["MappingRecord"]] = relationship( + back_populates="variant", cascade="all, delete-orphan" + ) + # Bidirectional relationship with ScoreCalibrationFunctionalClassification is left # purposefully undefined for performance reasons. diff --git a/src/mavedb/models/variant_annotation_status.py b/src/mavedb/models/variant_annotation_status.py index f39c47a64..357d6a6e6 100644 --- a/src/mavedb/models/variant_annotation_status.py +++ b/src/mavedb/models/variant_annotation_status.py @@ -93,7 +93,7 @@ class VariantAnnotationStatus(Base): # FK index for job_run_id — needed for CASCADE deletes on job_runs Index("ix_variant_annotation_status_job_run_id", "job_run_id"), CheckConstraint( - "annotation_type IN ('vrs_mapping', 'clingen_allele_id', 'mapped_hgvs', 'variant_translation', 'gnomad_allele_frequency', 'clinvar_control', 'vep_functional_consequence', 'ldh_submission')", + "annotation_type IN ('vrs_mapping', 'cross_level_translation', 'clingen_allele_id', 'mapped_hgvs', 'variant_translation', 'gnomad_allele_frequency', 'clinvar_control', 'vep_functional_consequence', 'ldh_submission')", name="ck_variant_annotation_type_valid", ), CheckConstraint( From fb0232f5c09add1c52af6d5cec26e4fdf37a0334 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Sat, 6 Jun 2026 18:55:21 -0700 Subject: [PATCH 012/252] chore(types): add mypy stubs for ga4gh and hgvs translation APIs Add type stubs for the VRS and hgvs APIs used by the reverse translation work so callers can be type-checked without casts. - ga4gh.core: type ga4gh_identify and re-export PrevVrsVersion - ga4gh.vrs: stub the data proxy, AlleleTranslator, and normalize; translate_from returns Any so callers annotate the concrete VRS subtype they expect without a redundant cast - hgvs.assemblymapper: stub AssemblyMapper with the g_to_t/c_to_g/c_to_p conversions used for cross-level translation --- mypy_stubs/ga4gh/core/__init__.pyi | 9 ++++++++- mypy_stubs/ga4gh/vrs/dataproxy.pyi | 8 ++++++++ mypy_stubs/ga4gh/vrs/extras/__init__.pyi | 0 mypy_stubs/ga4gh/vrs/extras/translator.pyi | 19 +++++++++++++++++++ mypy_stubs/ga4gh/vrs/normalize.pyi | 5 +++++ mypy_stubs/hgvs/assemblymapper.pyi | 20 ++++++++++++++++++++ 6 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 mypy_stubs/ga4gh/vrs/dataproxy.pyi create mode 100644 mypy_stubs/ga4gh/vrs/extras/__init__.pyi create mode 100644 mypy_stubs/ga4gh/vrs/extras/translator.pyi create mode 100644 mypy_stubs/ga4gh/vrs/normalize.pyi create mode 100644 mypy_stubs/hgvs/assemblymapper.pyi diff --git a/mypy_stubs/ga4gh/core/__init__.pyi b/mypy_stubs/ga4gh/core/__init__.pyi index f5814d7cf..8044c175f 100644 --- a/mypy_stubs/ga4gh/core/__init__.pyi +++ b/mypy_stubs/ga4gh/core/__init__.pyi @@ -1,3 +1,10 @@ from . import models as models +from .identifiers import PrevVrsVersion as PrevVrsVersion -__all__ = ["models"] +def ga4gh_identify( + vro: object, + in_place: str = ..., + as_version: PrevVrsVersion | None = ..., +) -> str | None: ... + +__all__ = ["PrevVrsVersion", "ga4gh_identify", "models"] diff --git a/mypy_stubs/ga4gh/vrs/dataproxy.pyi b/mypy_stubs/ga4gh/vrs/dataproxy.pyi new file mode 100644 index 000000000..25cabfda9 --- /dev/null +++ b/mypy_stubs/ga4gh/vrs/dataproxy.pyi @@ -0,0 +1,8 @@ +from abc import ABC + +class _DataProxy(ABC): + def get_metadata(self, identifier: str) -> dict: ... + def get_sequence(self, identifier: str, start: int | None = ..., end: int | None = ...) -> str: ... + +class SeqRepoDataProxy(_DataProxy): + def __init__(self, sr: object) -> None: ... diff --git a/mypy_stubs/ga4gh/vrs/extras/__init__.pyi b/mypy_stubs/ga4gh/vrs/extras/__init__.pyi new file mode 100644 index 000000000..e69de29bb diff --git a/mypy_stubs/ga4gh/vrs/extras/translator.pyi b/mypy_stubs/ga4gh/vrs/extras/translator.pyi new file mode 100644 index 000000000..d3bb6aeb4 --- /dev/null +++ b/mypy_stubs/ga4gh/vrs/extras/translator.pyi @@ -0,0 +1,19 @@ +from typing import Any + +from ga4gh.vrs.dataproxy import _DataProxy + +class _Translator: + default_assembly_name: str + data_proxy: _DataProxy + identify: bool + def __init__( + self, + data_proxy: _DataProxy, + default_assembly_name: str = ..., + identify: bool = ..., + ) -> None: ... + # Returns a VRS variation (Allele/CisPhasedBlock/...); typed loosely so callers + # can annotate the concrete subtype they expect without a redundant cast. + def translate_from(self, var: str, fmt: str | None = ..., **kwargs: Any) -> Any: ... + +class AlleleTranslator(_Translator): ... diff --git a/mypy_stubs/ga4gh/vrs/normalize.pyi b/mypy_stubs/ga4gh/vrs/normalize.pyi new file mode 100644 index 000000000..1aea15b23 --- /dev/null +++ b/mypy_stubs/ga4gh/vrs/normalize.pyi @@ -0,0 +1,5 @@ +from typing import Any + +from ga4gh.vrs.dataproxy import _DataProxy + +def normalize(vo: Any, data_proxy: _DataProxy | None = ..., **kwargs: Any) -> Any: ... diff --git a/mypy_stubs/hgvs/assemblymapper.pyi b/mypy_stubs/hgvs/assemblymapper.pyi new file mode 100644 index 000000000..798878312 --- /dev/null +++ b/mypy_stubs/hgvs/assemblymapper.pyi @@ -0,0 +1,20 @@ +from typing import Any + +import hgvs.sequencevariant + +class AssemblyMapper: + def __init__( + self, + hdp: Any, + assembly_name: str = ..., + alt_aln_method: str = ..., + normalize: bool = ..., + prevalidation_level: str | None = ..., + in_par_assume: str = ..., + replace_reference: bool = ..., + *args: Any, + **kwargs: Any, + ) -> None: ... + def g_to_t(self, var_g: Any, tx_ac: str) -> hgvs.sequencevariant.SequenceVariant: ... + def c_to_g(self, var_c: Any) -> hgvs.sequencevariant.SequenceVariant: ... + def c_to_p(self, var_c: Any, translation_table: Any = ...) -> hgvs.sequencevariant.SequenceVariant: ... From bcddaddf52e3f3c723a4fa1a246e49f2e27434d5 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Sat, 6 Jun 2026 18:55:56 -0700 Subject: [PATCH 013/252] build(deps): add variant-annotation as editable sibling dependency Wire the variant-annotation package into the server extra as a local editable (PEP 660) dependency so the API can drive the reverse translation pipeline. - declare variant-annotation as a develop path dependency in pyproject.toml and the server extra; lock pulls in its transitive deps (variant-translation, biopython, openpyxl, et-xmlfile) - mount ../variant-annotation into the dev and worker containers and prepend it to PYTHONPATH so the editable install resolves - pass --no-directory to poetry install in the Dockerfile so the build does not try to install the not-yet-copied editable sibling - ignore_missing_imports for variant_annotation.* in mypy: its editable import hook is unfollowable and it ships no py.typed, so treat it as untyped rather than maintaining drift-prone local stubs - add a Makefile with dev/test/lint/format targets --- Dockerfile | 2 +- Makefile | 23 +++++++ docker-compose-dev.yml | 4 ++ poetry.lock | 133 ++++++++++++++++++++++++++++++++++++++++- pyproject.toml | 10 +++- 5 files changed, 168 insertions(+), 4 deletions(-) create mode 100644 Makefile diff --git a/Dockerfile b/Dockerfile index 422e2bf83..bc5a52c45 100644 --- a/Dockerfile +++ b/Dockerfile @@ -80,7 +80,7 @@ RUN curl -sSL https://install.python-poetry.org | python3 - COPY poetry.lock pyproject.toml ./ # installs runtime dependencies to $VIRTUAL_ENV -RUN poetry install --no-root --extras server +RUN poetry install --no-root --extras server --no-directory COPY alembic /code/alembic COPY alembic.ini /code/alembic.ini COPY src /code/src diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..16f8319bc --- /dev/null +++ b/Makefile @@ -0,0 +1,23 @@ +VENV := .venv/bin + +.DEFAULT_GOAL := help + +.PHONY: help +help: + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf " %-12s %s\n", $$1, $$2}' + +.PHONY: dev +dev: ## Install deps including local editable variant-annotation + poetry install --extras server + +.PHONY: test +test: ## Run the test suite + $(VENV)/pytest tests/ + +.PHONY: lint +lint: ## Check code with ruff + $(VENV)/ruff check src/ tests/ + +.PHONY: format +format: ## Format code with ruff + $(VENV)/ruff format src/ tests/ diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 9835cff4e..ac4bcff58 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -19,6 +19,7 @@ services: REDIS_PORT: 6379 REDIS_SSL: "false" LOG_CONFIG: dev + PYTHONPATH: "/variant-annotation:/code/src" ports: - "8002:8000" ulimits: @@ -27,6 +28,7 @@ services: hard: 65536 volumes: - .:/code + - ../variant-annotation:/variant-annotation - mavedb-seqrepo-dev:/usr/local/share/seqrepo worker: @@ -44,12 +46,14 @@ services: REDIS_PORT: 6379 REDIS_SSL: "false" LOG_CONFIG: dev + PYTHONPATH: "/variant-annotation:/code/src" ulimits: nofile: soft: 65536 hard: 65536 volumes: - .:/code + - ../variant-annotation:/variant-annotation - mavedb-seqrepo-dev:/usr/local/share/seqrepo depends_on: - db diff --git a/poetry.lock b/poetry.lock index 037bda653..10a9a3f08 100644 --- a/poetry.lock +++ b/poetry.lock @@ -269,6 +269,51 @@ dev = ["bandit (>=1.7,<2.0)", "build (>=0.8,<1.0)", "flake8 (>=4.0,<5.0)", "ipyt docs = ["mkdocs"] tests = ["pytest (>=7.1,<8.0)", "pytest-cov (>=4.1,<5.0)", "pytest-optional-tests", "tox (>=3.25,<4.0)", "vcrpy"] +[[package]] +name = "biopython" +version = "1.87" +description = "Freely available tools for computational molecular biology." +optional = true +python-versions = ">=3.10" +groups = ["main"] +markers = "extra == \"server\"" +files = [ + {file = "biopython-1.87-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:35f13796188412f135acbed196bd6cfedc1257199d50b4883e289bbd96320efc"}, + {file = "biopython-1.87-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:63978e1b3ae040c52369bc1bd3f4c668171f5f1f0808798eccd74b575cce455d"}, + {file = "biopython-1.87-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1e951f4862ffc1dccc28e1c25245059fa653d86028a88f5dfe1b7875875f3a4f"}, + {file = "biopython-1.87-cp310-cp310-win32.whl", hash = "sha256:86596b05f39afbc5984ee6239c93fd75f6a97fffb9a630d74b45652c24aab964"}, + {file = "biopython-1.87-cp310-cp310-win_amd64.whl", hash = "sha256:efc16dd8a9312eb655fa590821495b0d8ca25d19f7b4ef4fa4da9e71d59d33e5"}, + {file = "biopython-1.87-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:58e36efa7eaa8813cffe440af4824afa20cc3c21b1b179a95cc0fb15c4b83c01"}, + {file = "biopython-1.87-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ccf00d15e698656796ab14ff3eb175e282da7a08eedd36a29b6cacec5a33e97f"}, + {file = "biopython-1.87-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bab39ec4108fb2542a9f1012db8269426fd6e86ed84ebc1b0bd11134ef443dd3"}, + {file = "biopython-1.87-cp311-cp311-win32.whl", hash = "sha256:126e18ad44e959a1984560562fa4f295c075ce2e610e8ddbc9ec6f52e09f70d8"}, + {file = "biopython-1.87-cp311-cp311-win_amd64.whl", hash = "sha256:d4ff5369ffb7a966bcf921cae6c8a6eab5070b5df1c9f2ae8c18ad40fdcb7ec5"}, + {file = "biopython-1.87-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:77ccc634621904d4a8fa0a43b5e0f093fa9df8c9577ed3858af648bb3528f51e"}, + {file = "biopython-1.87-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a3428155c3e0abbed7aad5ff08e034d435b84dfe560c8ec58e7d43abda4b6a43"}, + {file = "biopython-1.87-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:65ba69ef0273e983a9036c2a228142bc34266179a5f03660fc281d332d718630"}, + {file = "biopython-1.87-cp312-cp312-win32.whl", hash = "sha256:b077777fd2c555434bdcee58743f6f860aa80e1e005d9671913aa73823c6a773"}, + {file = "biopython-1.87-cp312-cp312-win_amd64.whl", hash = "sha256:856e3d64f1f27db493474ff84916ed8572731a525e001c7d0d8f41a0fd187000"}, + {file = "biopython-1.87-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e05ef5d632c319ab3ef77705c74061190d0792b07e1f2b9eee867401b2758e7e"}, + {file = "biopython-1.87-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:772539297fa16a78f38651c793f53f8c11bd18317b111982e72cf30a6e57512a"}, + {file = "biopython-1.87-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6d221b2e08e7e89713fdbfb15c8ea6744e908d59f672cd2b6fcf9ed47910d05e"}, + {file = "biopython-1.87-cp313-cp313-win32.whl", hash = "sha256:fab1b12f6bc4646b7f56b4c390ecff685f02b5b29e3a0c10477195bb49fe62f8"}, + {file = "biopython-1.87-cp313-cp313-win_amd64.whl", hash = "sha256:01ee30203bd4b2145cdfe2878499e549a7087f897a6f4d1ebd9de30790123140"}, + {file = "biopython-1.87-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:db73fe16aa2b20677ac86d1997612acb0aa1a3720bc899f65d2bce5583208e1b"}, + {file = "biopython-1.87-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:89ffe272517478691439a59cccd3cc2929fc8f6bfb8cbc8cc5acc103660395a7"}, + {file = "biopython-1.87-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ff28a6f31630b3c9f52903478a2ed9dd894b07c1998e40eaeefbeefac20f2d0f"}, + {file = "biopython-1.87-cp314-cp314-win32.whl", hash = "sha256:d740c75d4bc94f9dff51719a0deda37e5e885f06ee6dfbb5e9a21bbe9de35a9c"}, + {file = "biopython-1.87-cp314-cp314-win_amd64.whl", hash = "sha256:98e397096336a49804b6aaaeac8c47ad82e3e4430862f0cde37be73037f1017e"}, + {file = "biopython-1.87-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:e4878a9b56775480154c686f81e98f6d907b44d87605bdc2f53538ccdfde9624"}, + {file = "biopython-1.87-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6fcec2e3602ed52ced701f8f7851952383f84dbc4caeb4d202d088170e86b6d"}, + {file = "biopython-1.87-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c8da2d44a4b912c7550a051a5ff4bb72a61decc9c4b19ea92cba4c02fffb143c"}, + {file = "biopython-1.87-cp314-cp314t-win32.whl", hash = "sha256:3670d76759c6cb53ba617f9823d3a438c1aa5415abef6addd29cb81d61d7b312"}, + {file = "biopython-1.87-cp314-cp314t-win_amd64.whl", hash = "sha256:331c4151608a1d8406eff0d3c52a0ff1fa3e82604fc85f11c696c562919fb161"}, + {file = "biopython-1.87.tar.gz", hash = "sha256:8456c803459b679a9712422e5a7fd9809f2f089bf69bb085f3b077946ac9bdbf"}, +] + +[package.dependencies] +numpy = "*" + [[package]] name = "bioutils" version = "0.6.1" @@ -1432,6 +1477,19 @@ files = [ dnspython = ">=2.0.0" idna = ">=2.0.0" +[[package]] +name = "et-xmlfile" +version = "2.0.0" +description = "An implementation of lxml.xmlfile for the standard library" +optional = true +python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"server\"" +files = [ + {file = "et_xmlfile-2.0.0-py3-none-any.whl", hash = "sha256:7a91720bc756843502c3b7504c77b8fe44217c85c537d85037f0f536151b2caa"}, + {file = "et_xmlfile-2.0.0.tar.gz", hash = "sha256:dab3f4764309081ce75662649be815c4c9081e88f0837825f90fd28317d4da54"}, +] + [[package]] name = "eutils" version = "0.6.0" @@ -2893,6 +2951,22 @@ files = [ {file = "numpy-1.26.4.tar.gz", hash = "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010"}, ] +[[package]] +name = "openpyxl" +version = "3.1.5" +description = "A Python library to read/write Excel 2010 xlsx/xlsm files" +optional = true +python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"server\"" +files = [ + {file = "openpyxl-3.1.5-py2.py3-none-any.whl", hash = "sha256:5282c12b107bffeef825f4617dc029afaf41d0ea60823bbb665ef3079dc79de2"}, + {file = "openpyxl-3.1.5.tar.gz", hash = "sha256:cf0e3cf56142039133628b5acffe8ef0c12bc902d2aadd3e0fe5878dc08d1050"}, +] + +[package.dependencies] +et-xmlfile = "*" + [[package]] name = "orcid" version = "1.0.3" @@ -4796,6 +4870,61 @@ dev = ["Cython (>=3.0,<4.0)", "setuptools (>=60)"] docs = ["Sphinx (>=4.1.2,<4.2.0)", "sphinx_rtd_theme (>=0.5.2,<0.6.0)", "sphinxcontrib-asyncio (>=0.3.0,<0.4.0)"] test = ["aiohttp (>=3.10.5)", "flake8 (>=6.1,<7.0)", "mypy (>=0.800)", "psutil", "pyOpenSSL (>=25.3.0,<25.4.0)", "pycodestyle (>=2.11.0,<2.12.0)"] +[[package]] +name = "variant-annotation" +version = "0.1.0" +description = "Genetic variant mapping and annotation pipeline" +optional = true +python-versions = ">=3.11" +groups = ["main"] +markers = "extra == \"server\"" +files = [] +develop = true + +[package.dependencies] +biopython = "*" +boto3 = "*" +click = "*" +openpyxl = "*" +python-dotenv = "*" +redis = "*" +requests = "*" +variant-translation = "1.1.0b1" + +[package.extras] +dev = ["ruff"] +gnomad = ["hail"] +mapping = ["dcd-mapping @ git+https://github.com/VariantEffect/dcd_mapping2.git@v2026.1.2"] +publish = ["build", "twine"] +tests = ["pytest"] + +[package.source] +type = "directory" +url = "../variant-annotation" + +[[package]] +name = "variant-translation" +version = "1.1.0b1" +description = "Reverse-translate protein HGVS variants." +optional = true +python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"server\"" +files = [ + {file = "variant_translation-1.1.0b1-py3-none-any.whl", hash = "sha256:cdf693ab564b767c89e7b463b1345e801d2ab4a5f312631584bf79d178ba6122"}, + {file = "variant_translation-1.1.0b1.tar.gz", hash = "sha256:5151d02ccb5c651c7ba47398d75355f2bd19a36717faa39dc2a390cfb4c7968f"}, +] + +[package.dependencies] +click = "*" +hgvs = "*" +python-dotenv = "*" +requests = "*" + +[package.extras] +dev = ["pytest", "ruff"] +publish = ["build", "twine"] + [[package]] name = "virtualenv" version = "21.3.1" @@ -5098,9 +5227,9 @@ test = ["big-O", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more_it type = ["pytest-mypy"] [extras] -server = ["aiocache", "alembic", "alembic-utils", "arq", "authlib", "biocommons", "boto3", "cdot", "cryptography", "fastapi", "hgvs", "orcid", "psycopg2", "pyathena", "python-jose", "python-multipart", "requests", "slack-sdk", "starlette", "starlette-context", "uvicorn", "watchtower"] +server = ["aiocache", "alembic", "alembic-utils", "arq", "authlib", "biocommons", "boto3", "cdot", "cryptography", "fastapi", "hgvs", "orcid", "psycopg2", "pyathena", "python-jose", "python-multipart", "requests", "slack-sdk", "starlette", "starlette-context", "uvicorn", "variant-annotation", "watchtower"] [metadata] lock-version = "2.1" python-versions = "^3.11" -content-hash = "64ca4755b1a5c8930882a7ad3b25feee434f42ffb02ba2841f38c86de56cc948" +content-hash = "25445affad8eb09fb30e96f152dd0f066eb6f6a4d13e61023fba5357002f62ba" diff --git a/pyproject.toml b/pyproject.toml index 7bc2556d0..427c88996 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,6 +63,7 @@ starlette = { version = "~0.49.0", optional = true } starlette-context = { version = "^0.3.6", optional = true } slack-sdk = { version = "~3.21.3", optional = true } uvicorn = { extras = ["standard"], version = "*", optional = true } +variant-annotation = { path = "../variant-annotation", develop = true, optional = true } watchtower = { version = "~3.2.0", optional = true } asyncclick = "^8.3.0.7" filelock = "^3.29.0" @@ -93,7 +94,7 @@ SQLAlchemy = { extras = ["mypy"], version = "~2.0.0" } [tool.poetry.extras] -server = ["aiocache", "alembic", "alembic-utils", "arq", "authlib", "biocommons", "boto3", "cdot", "cryptography", "fastapi", "hgvs", "orcid", "psycopg2", "python-jose", "python-multipart", "pyathena", "requests", "starlette", "starlette-context", "slack-sdk", "uvicorn", "watchtower"] +server = ["aiocache", "alembic", "alembic-utils", "arq", "authlib", "biocommons", "boto3", "cdot", "cryptography", "fastapi", "hgvs", "orcid", "psycopg2", "python-jose", "python-multipart", "pyathena", "requests", "starlette", "starlette-context", "slack-sdk", "uvicorn", "variant-annotation", "watchtower"] [tool.mypy] @@ -104,6 +105,13 @@ plugins = [ ] mypy_path = "mypy_stubs" +# variant-annotation is installed as a PEP 660 editable sibling package whose import +# hook mypy cannot follow, and it ships no py.typed marker. Treat it as untyped rather +# than maintaining drift-prone local stubs for an actively-developed internal library. +[[tool.mypy.overrides]] +module = "variant_annotation.*" +ignore_missing_imports = true + [tool.pytest.ini_options] addopts = "-v --import-mode=importlib" asyncio_mode = 'strict' From ec36115003b893e9496d9eb2a7f2c384c71366bd Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Sat, 6 Jun 2026 18:57:09 -0700 Subject: [PATCH 014/252] feat(worker): write mapping job output to the allele closure tables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rework the variant mapping job to persist its results into the new MappingRecord / Allele / MappingRecordAllele schema instead of MappedVariant, building on the closure tables for the Better Reverse Translation epic. - create one MappingRecord per variant (including failed variants, with null VRS data); successfully post-mapped variants additionally get-or-create an authoritative Allele and link it via an is_authoritative MappingRecordAllele - supersede a variant's prior live record through ValidTime supersede_with — retiring it and its allele links and inserting the new record under one timestamp — instead of flipping a current flag, so the old/new handoff is gap-free - require a TargetGeneMapping for every mapped score, raising on a miss rather than tolerating a null link, since dcd-mapping guarantees one - combine cis-phased members when deriving hgvs_assay_level - update the mapping job tests to assert against MappingRecord and the authoritative-allele link, including the gap-free supersession and cascade-retire of the prior link TODO#765: mapping is not idempotent, so each run always creates a new authoritative link. --- .../worker/jobs/variant_processing/mapping.py | 93 ++++-- .../jobs/variant_processing/test_mapping.py | 276 ++++++++++-------- 2 files changed, 229 insertions(+), 140 deletions(-) diff --git a/src/mavedb/worker/jobs/variant_processing/mapping.py b/src/mavedb/worker/jobs/variant_processing/mapping.py index efc8226b3..d8b02cebb 100644 --- a/src/mavedb/worker/jobs/variant_processing/mapping.py +++ b/src/mavedb/worker/jobs/variant_processing/mapping.py @@ -24,12 +24,15 @@ from mavedb.lib.logging.context import format_raised_exception_info_as_dict from mavedb.lib.mapping import EXCLUDED_PREMAPPED_ANNOTATION_KEYS from mavedb.lib.types.workflow import JobExecutionOutcome +from mavedb.lib.variant_translations import get_or_create_allele from mavedb.lib.variants import get_hgvs_from_post_mapped +from mavedb.models.allele import Allele as AlleleDbModel from mavedb.models.enums.annotation_layer import AnnotationLayer from mavedb.models.enums.annotation_type import AnnotationType from mavedb.models.enums.job_pipeline import AnnotationFailureCategory, AnnotationStatus, FailureCategory from mavedb.models.enums.mapping_state import MappingState -from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.score_set import ScoreSet from mavedb.models.target_gene_mapping import TargetGeneMapping from mavedb.models.user import User @@ -130,7 +133,7 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan # Per-(target, alignment_level) QC records produced by the dcd-mapping API. # All records share the same tool_version because they come from a single run; - # we use that as the global ``mapping_api_version`` carried on each MappedVariant. + # we use that as the global ``mapping_api_version`` carried on each MappingRecord. target_mappings_payload = mapping_results.get("target_mappings") or [] tool_version = next( (tm.get("tool_version") for tm in target_mappings_payload if tm.get("tool_version")), @@ -218,7 +221,7 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan continue target_gene_mapping = TargetGeneMapping( - target_gene_id=target_gene.id, + target_gene=target_gene, alignment_level=AnnotationLayer.from_wire(level_value), preferred=bool(tm.get("preferred", False)), reference_assembly=tm.get("reference_assembly"), @@ -268,18 +271,16 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan job_manager.save_to_context({"processing_variant": variant.id}) logger.debug(f"Processing variant {variant.id}.", extra=job_manager.logging_context()) - # there should only be one current mapped variant per variant id, so update old mapped variant to current = false + # Only allow one live MappingRecord per variant. The prior live record (if any) is + # superseded by the new version below via supersede_with, which retires it (cascading to + # its allele links) and inserts the new record under one timestamp. existing_mapped_variant = ( - job_manager.db.query(MappedVariant) - .filter(MappedVariant.variant_id == variant.id, MappedVariant.current.is_(True)) + job_manager.db.query(MappingRecord) + .filter(MappingRecord.variant_id == variant.id, MappingRecord.current) .one_or_none() ) - if existing_mapped_variant: job_manager.save_to_context({"existing_mapped_variant": existing_mapped_variant.id}) - existing_mapped_variant.current = False - job_manager.db.add(existing_mapped_variant) - logger.debug(msg="Set existing mapped variant to current = false.", extra=job_manager.logging_context()) annotation_was_successful = mapped_score.get("pre_mapped") and mapped_score.get("post_mapped") if annotation_was_successful: @@ -294,27 +295,47 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan score_alignment_level = mapped_score.get("alignment_level") if not score_target or not score_alignment_level: raise NonexistentMappingResultsError( - f"ScoreAnnotation for variant {variant_urn!r} is missing " - f"target_gene_identifier or alignment_level." + f"ScoreAnnotation for variant {variant_urn!r} is missing target_gene_identifier or alignment_level." ) + pre_mapped_allele: dict = mapped_score.get("pre_mapped") or {} + post_mapped_allele: dict = mapped_score.get("post_mapped") or {} + annotation_layer = AnnotationLayer.from_wire(score_alignment_level) + assay_level_hgvs = get_hgvs_from_post_mapped(post_mapped_allele, combine_cis=True) + + # dcd-mapping guarantees every mapped score is attributable to a TargetGeneMapping + # via (target_gene_identifier, alignment_level) -- including failed variants, which + # it re-attributes to the target's preferred layer. A miss implies a malformed payload. target_gene_mapping_row = target_gene_mapping_by_key.get((score_target, score_alignment_level)) - mapped_variant = MappedVariant( - pre_mapped=mapped_score.get("pre_mapped", null()), - post_mapped=mapped_score.get("post_mapped", null()), - hgvs_assay_level=get_hgvs_from_post_mapped(mapped_score.get("post_mapped", {})), + if target_gene_mapping_row is None: + raise NonexistentMappingResultsError( + f"ScoreAnnotation for variant {variant_urn!r} has no TargetGeneMapping for " + f"(target={score_target!r}, alignment_level={score_alignment_level!r})." + ) + + mapping_record = MappingRecord( variant_id=variant.id, - modification_date=date.today(), + vrs_digest=pre_mapped_allele.get("id"), + pre_mapped=pre_mapped_allele or None, + assay_level=annotation_layer, + hgvs_assay_level=assay_level_hgvs, mapped_date=mapping_results["mapped_date"], - vrs_version=mapped_score.get("vrs_version", null()), + vrs_version=mapped_score.get("vrs_version", None), mapping_api_version=tool_version, - error_message=mapped_score.get("error_message", null()), - target_gene_mapping_id=target_gene_mapping_row.id if target_gene_mapping_row else None, - alignment_level=AnnotationLayer.from_wire(score_alignment_level), + target_gene_mapping_id=target_gene_mapping_row.id, + alignment_level=annotation_layer, at_mismatched_locus=mapped_score.get("at_mismatched_locus"), near_gap=mapped_score.get("near_gap"), - current=True, ) + if existing_mapped_variant: + # Retire the prior record (cascading to its allele links) and insert this one under a + # single timestamp, so the prior valid_to equals the new valid_from with no gap. + existing_mapped_variant.supersede_with(job_manager.db, mapping_record) + logger.debug( + msg="Superseded prior mapping record and its allele links.", extra=job_manager.logging_context() + ) + else: + job_manager.db.add(mapping_record) annotation_manager.add_annotation( variant_id=variant.id, # type: ignore @@ -327,14 +348,38 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan annotation_data={ "error_message": mapped_score.get("error_message", null()), "annotation_metadata": { - "mapped_assay_level_hgvs": get_hgvs_from_post_mapped(mapped_score.get("post_mapped", {})), + "mapped_assay_level_hgvs": assay_level_hgvs, }, }, current=True, ) - job_manager.db.add(mapped_variant) + # Only variants with a post-mapped representation yield an authoritative Allele. Failed variants + # get a MappingRecord (with null VRS data) but no linked allele. + if post_mapped_allele: + allele_draft = AlleleDbModel( + vrs_digest=post_mapped_allele["id"], + level=annotation_layer, + hgvs_g=assay_level_hgvs if annotation_layer == AnnotationLayer.genomic else None, + hgvs_c=assay_level_hgvs if annotation_layer == AnnotationLayer.cdna else None, + hgvs_p=assay_level_hgvs if annotation_layer == AnnotationLayer.protein else None, + post_mapped=post_mapped_allele, + ) + authoritative_allele = get_or_create_allele(job_manager.db, allele_draft) + job_manager.db.flush() + + # TODO#765: Mapping is not idempotent, so we must always create a new link. + job_manager.db.add( + MappingRecordAllele( + mapping_record_id=mapping_record.id, + allele_id=authoritative_allele.id, + is_authoritative=True, + ) + ) + logger.debug(msg="Linked mapped variant to authoritative allele.", extra=job_manager.logging_context()) + logger.debug(msg="Added new mapped variant to session.", extra=job_manager.logging_context()) + job_manager.db.flush() annotation_manager.flush() diff --git a/tests/worker/jobs/variant_processing/test_mapping.py b/tests/worker/jobs/variant_processing/test_mapping.py index dc2ac843e..b28f61564 100644 --- a/tests/worker/jobs/variant_processing/test_mapping.py +++ b/tests/worker/jobs/variant_processing/test_mapping.py @@ -5,15 +5,19 @@ pytest.importorskip("arq") from asyncio.unix_events import _UnixSelectorEventLoop +from datetime import date from unittest.mock import MagicMock, patch from sqlalchemy.exc import NoResultFound from mavedb.lib.mapping import EXCLUDED_PREMAPPED_ANNOTATION_KEYS from mavedb.lib.types.workflow import JobExecutionOutcome +from mavedb.models.allele import Allele +from mavedb.models.enums.annotation_layer import AnnotationLayer from mavedb.models.enums.job_pipeline import JobStatus, PipelineStatus from mavedb.models.enums.mapping_state import MappingState -from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.variant import Variant from mavedb.models.variant_annotation_status import VariantAnnotationStatus from mavedb.worker.jobs.variant_processing.mapping import map_variants_for_score_set @@ -24,6 +28,25 @@ pytestmark = pytest.mark.usefixtures("patch_db_session_ctxmgr") +def authoritative_allele_for(session, mapping_record): + """Return the authoritative (post-mapped) Allele linked to a mapping record, or None. + + Under the allele data model the post-mapped VRS representation lives on an + ``Allele`` joined to the ``MappingRecord`` through ``MappingRecordAllele``. + A successfully post-mapped variant has exactly one authoritative link; a + variant that failed to post-map has a mapping record but no such link. + """ + return ( + session.query(Allele) + .join(MappingRecordAllele, MappingRecordAllele.allele_id == Allele.id) + .filter( + MappingRecordAllele.mapping_record_id == mapping_record.id, + MappingRecordAllele.is_authoritative.is_(True), + ) + .one_or_none() + ) + + @pytest.mark.unit @pytest.mark.asyncio class TestMapVariantsForScoreSetUnit: @@ -90,7 +113,10 @@ async def test_map_variants_for_score_set_no_mapped_scores( _UnixSelectorEventLoop, "run_in_executor", return_value=self.dummy_mapping_output( - {"mapped_scores": [], "error_message": "No variants were mapped for this score set"} + { + "mapped_scores": [], + "error_message": "No variants were mapped for this score set", + } ), ), ): @@ -134,7 +160,10 @@ async def test_map_variants_for_score_set_no_reference_data( _UnixSelectorEventLoop, "run_in_executor", return_value=self.dummy_mapping_output( - {"mapped_scores": [MagicMock()], "error_message": "Reference metadata missing from mapping results"} + { + "mapped_scores": [MagicMock()], + "error_message": "Reference metadata missing from mapping results", + } ), ), ): @@ -318,9 +347,9 @@ async def dummy_mapping_job(): for target in sample_score_set.target_genes: assert target.mapped_hgnc_name is None - # Verify that a mapped variant was created - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 1 + # Verify that a mapping record was created + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 1 # Verify that annotation statuses were created and correct annotation_statuses = ( @@ -435,9 +464,9 @@ async def dummy_mapping_job(): else: assert target.post_mapped_metadata.get("protein") is None - # Verify that a mapped variant was created - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 1 + # Verify that a mapping record was created + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 1 # Verify that annotation statuses were created and correct annotation_statuses = ( @@ -501,13 +530,12 @@ async def dummy_mapping_job(): assert sample_score_set.mapping_state == MappingState.failed assert sample_score_set.mapping_errors["error_message"] == "All variants failed to map." - # Verify that one mapped variant was created. Although no successful mapping, an entry is still created. - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 1 + # Verify that one mapping record was created. Although no successful mapping, an entry is still created. + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 1 - # Verify that the mapped variant has no post-mapped data - mapped_variant = mapped_variants[0] - assert mapped_variant.post_mapped == {} + # Verify that the mapping record has no authoritative (post-mapped) allele + assert authoritative_allele_for(session, mapping_records[0]) is None # Verify that annotation statuses were created and correct annotation_statuses = ( @@ -584,19 +612,14 @@ async def dummy_mapping_job(): # Although only one variant was successfully mapped, verify that an entity was created # for each variant in the score set - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 2 + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 2 - # Verify that only one variant has post-mapped data - mapped_variant_with_post_data = ( - session.query(MappedVariant).filter(MappedVariant.post_mapped != {}).one_or_none() - ) - assert mapped_variant_with_post_data is not None - - mapped_variant_without_post_data = ( - session.query(MappedVariant).filter(MappedVariant.post_mapped == {}).one_or_none() - ) - assert mapped_variant_without_post_data is not None + # Verify that exactly one mapping record has post-mapped (authoritative) allele data + records_with_post_data = [r for r in mapping_records if authoritative_allele_for(session, r) is not None] + records_without_post_data = [r for r in mapping_records if authoritative_allele_for(session, r) is None] + assert len(records_with_post_data) == 1 + assert len(records_without_post_data) == 1 # Verify that annotation statuses were created and correct annotation_status_success = ( @@ -678,17 +701,17 @@ async def dummy_mapping_job(): assert sample_score_set.mapping_state == MappingState.complete assert sample_score_set.mapping_errors is None - # Verify that mapped variants were created - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 2 + # Verify that mapping records were created + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 2 # Verify that both variants have post-mapped data. I'm comfortable assuming the # data is correct given our layer permutation tests above. for urn in ["variant:1", "variant:2"]: - mapped_variant = session.query(MappedVariant).filter(MappedVariant.variant.has(urn=urn)).one_or_none() - assert mapped_variant is not None - assert mapped_variant.post_mapped != {} - assert mapped_variant.hgvs_assay_level is not None + mapping_record = session.query(MappingRecord).filter(MappingRecord.variant.has(urn=urn)).one_or_none() + assert mapping_record is not None + assert authoritative_allele_for(session, mapping_record) is not None + assert mapping_record.hgvs_assay_level is not None # Verify that annotation statuses were created and correct annotation_statuses = ( @@ -733,13 +756,23 @@ async def dummy_mapping_job(): ) session.add(variant) session.commit() - mapped_variant = MappedVariant( + mapping_record = MappingRecord( variant_id=variant.id, - current=True, - mapped_date="2023-01-01T00:00:00Z", + assay_level=AnnotationLayer.genomic, + mapped_date=date(2023, 1, 1), mapping_api_version="v1.0.0", ) - session.add(mapped_variant) + session.add(mapping_record) + session.commit() + # Link the prior record to an authoritative allele. Re-mapping must retire this link too, + # not just the record — a live link dangling off a retired record breaks the temporal model. + prior_allele = Allele(vrs_digest="ga4gh:VA.prior", level=AnnotationLayer.genomic.value) + session.add(prior_allele) + session.commit() + prior_link = MappingRecordAllele( + mapping_record_id=mapping_record.id, allele_id=prior_allele.id, is_authoritative=True + ) + session.add(prior_link) session.commit() variant_annotation_status = VariantAnnotationStatus( variant_id=variant.id, current=True, annotation_type="vrs_mapping", status="success" @@ -766,31 +799,42 @@ async def dummy_mapping_job(): assert sample_score_set.mapping_state == MappingState.complete assert sample_score_set.mapping_errors is None - # Verify the existing mapped variant was marked as non-current - non_current_mapped_variant = ( - session.query(MappedVariant) - .filter(MappedVariant.id == mapped_variant.id, MappedVariant.current.is_(False)) + # Verify the existing mapping record was retired (valid_to closed) + non_current_mapping_record = ( + session.query(MappingRecord) + .filter(MappingRecord.id == mapping_record.id, MappingRecord.valid_to.isnot(None)) .one_or_none() ) - assert non_current_mapped_variant is not None + assert non_current_mapping_record is not None - # Verify a new mapped variant entry was created - new_mapped_variant = ( - session.query(MappedVariant) - .filter(MappedVariant.variant_id == variant.id, MappedVariant.current.is_(True)) + # Verify the prior record's authoritative allele link was retired alongside the record, + # so no live link dangles off the retired record. + session.refresh(prior_link) + assert prior_link.valid_to is not None + + # Verify a new, live mapping record entry was created + new_mapping_record = ( + session.query(MappingRecord) + .filter(MappingRecord.variant_id == variant.id, MappingRecord.current) .one_or_none() ) - assert new_mapped_variant is not None + assert new_mapping_record is not None + + # Gap-free handoff: the prior record closed at exactly the new record's valid_from (one + # supersession timestamp), and the cascade closed the prior link at that same instant — so + # no point-in-time query lands in a hole between the old and new record or their links. + assert non_current_mapping_record.valid_to == new_mapping_record.valid_from + assert prior_link.valid_to == non_current_mapping_record.valid_to - # Verify that the new mapped variant has updated mapping data - assert new_mapped_variant.mapped_date != "2023-01-01T00:00:00Z" - assert new_mapped_variant.mapping_api_version != "v1.0.0" + # Verify that the new mapping record has updated mapping data + assert new_mapping_record.mapped_date != date(2023, 1, 1) + assert new_mapping_record.mapping_api_version != "v1.0.0" # Verify the non-current annotation status still exists old_annotation_status = ( session.query(VariantAnnotationStatus) .filter( - VariantAnnotationStatus.variant_id == non_current_mapped_variant.variant_id, + VariantAnnotationStatus.variant_id == non_current_mapping_record.variant_id, VariantAnnotationStatus.current.is_(False), ) .one_or_none() @@ -862,9 +906,9 @@ async def dummy_mapping_job(): assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - # Verify that mapped variants were created - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 4 + # Verify that mapping records were created + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 4 # Verify score set mapping state assert sample_score_set.mapping_state == MappingState.complete @@ -875,11 +919,11 @@ async def dummy_mapping_job(): assert target.mapped_hgnc_name is not None assert target.post_mapped_metadata is not None - # Verify that each variant has a corresponding mapped variant + # Verify that each variant has a corresponding mapping record variants = ( session.query(Variant) - .join(MappedVariant, MappedVariant.variant_id == Variant.id) - .filter(Variant.score_set_id == sample_score_set.id, MappedVariant.current.is_(True)) + .join(MappingRecord, MappingRecord.variant_id == Variant.id) + .filter(Variant.score_set_id == sample_score_set.id, MappingRecord.current.is_(True)) .all() ) assert len(variants) == 4 @@ -953,9 +997,9 @@ async def dummy_mapping_job(): assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - # Verify that mapped variants were created - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 4 + # Verify that mapping records were created + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 4 # Verify score set mapping state assert sample_score_set.mapping_state == MappingState.complete @@ -966,11 +1010,11 @@ async def dummy_mapping_job(): assert target.mapped_hgnc_name is not None assert target.post_mapped_metadata is not None - # Verify that each variant has a corresponding mapped variant + # Verify that each variant has a corresponding mapping record variants = ( session.query(Variant) - .join(MappedVariant, MappedVariant.variant_id == Variant.id) - .filter(Variant.score_set_id == sample_score_set.id, MappedVariant.current.is_(True)) + .join(MappingRecord, MappingRecord.variant_id == Variant.id) + .filter(Variant.score_set_id == sample_score_set.id, MappingRecord.current.is_(True)) .all() ) assert len(variants) == 4 @@ -1053,9 +1097,9 @@ async def dummy_mapping_job(): in sample_score_set.mapping_errors["error_message"] ) - # Verify that no mapped variants were created - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 0 + # Verify that no mapping records were created + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 0 # Verify that no annotation statuses were created annotation_statuses = session.query(VariantAnnotationStatus).all() @@ -1130,9 +1174,9 @@ async def dummy_mapping_job(): # Error message originates from our mock mapping construction function assert "test error: no mapped scores" in sample_score_set.mapping_errors["error_message"] - # Verify that no mapped variants were created - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 0 + # Verify that no mapping records were created + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 0 # Verify that no annotation statuses were created annotation_statuses = session.query(VariantAnnotationStatus).all() @@ -1206,9 +1250,9 @@ async def dummy_mapping_job(): assert sample_score_set.mapping_errors is not None assert "Reference metadata missing from mapping results" in sample_score_set.mapping_errors["error_message"] - # Verify that no mapped variants were created - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 0 + # Verify that no mapping records were created + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 0 # Verify that no annotation statuses were created annotation_statuses = session.query(VariantAnnotationStatus).all() @@ -1246,20 +1290,20 @@ async def test_map_variants_for_score_set_updates_current_mapped_variants( sample_independent_variant_creation_run, ) - # Associate mapped variants with all variants just created in the score set + # Associate mapping records with all variants just created in the score set variants = session.query(Variant).filter(Variant.score_set_id == sample_score_set.id).all() for variant in variants: - mapped_variant = MappedVariant( + mapping_record = MappingRecord( variant_id=variant.id, - current=True, - mapped_date="2023-01-01T00:00:00Z", + assay_level=AnnotationLayer.genomic, + mapped_date=date(2023, 1, 1), mapping_api_version="v1.0.0", ) annotation_status = VariantAnnotationStatus( variant_id=variant.id, current=True, annotation_type="vrs_mapping", status="success" ) session.add(annotation_status) - session.add(mapped_variant) + session.add(mapping_record) session.commit() async def dummy_mapping_job(): @@ -1295,27 +1339,27 @@ async def dummy_mapping_job(): assert sample_score_set.mapping_state == MappingState.complete assert sample_score_set.mapping_errors is None - # Verify that mapped variants were marked as non-current and new entries created - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == len(variants) * 2 # Each variant has two mapped entries now + # Verify that mapping records were marked as non-current and new entries created + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == len(variants) * 2 # Each variant has two mapping records now for variant in variants: - non_current_mapped_variant = ( - session.query(MappedVariant) - .filter(MappedVariant.variant_id == variant.id, MappedVariant.current.is_(False)) + non_current_mapping_record = ( + session.query(MappingRecord) + .filter(MappingRecord.variant_id == variant.id, MappingRecord.valid_to.isnot(None)) .one_or_none() ) - assert non_current_mapped_variant is not None + assert non_current_mapping_record is not None - new_mapped_variant = ( - session.query(MappedVariant) - .filter(MappedVariant.variant_id == variant.id, MappedVariant.current.is_(True)) + new_mapping_record = ( + session.query(MappingRecord) + .filter(MappingRecord.variant_id == variant.id, MappingRecord.current) .one_or_none() ) - assert new_mapped_variant is not None + assert new_mapping_record is not None - # Verify that the new mapped variant has updated mapping data - assert new_mapped_variant.mapped_date != "2023-01-01T00:00:00Z" - assert new_mapped_variant.mapping_api_version != "v1.0.0" + # Verify that the new mapping record has updated mapping data + assert new_mapping_record.mapped_date != date(2023, 1, 1) + assert new_mapping_record.mapping_api_version != "v1.0.0" # Verify that annotation statuses where marked as non-current and new entries created annotation_statuses = session.query(VariantAnnotationStatus).all() @@ -1389,9 +1433,9 @@ async def dummy_mapping_job(): assert sample_score_set.mapping_errors is not None assert "test error: no mapped scores" in sample_score_set.mapping_errors["error_message"] - # Verify that no mapped variants were created - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 0 + # Verify that no mapping records were created + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 0 # Verify that no annotation statuses were created annotation_statuses = session.query(VariantAnnotationStatus).all() @@ -1448,9 +1492,9 @@ async def dummy_mapping_job(): in sample_score_set.mapping_errors["error_message"] ) - # Verify that no mapped variants were created - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 0 + # Verify that no mapping records were created + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 0 # Verify that no annotation statuses were created annotation_statuses = session.query(VariantAnnotationStatus).all() @@ -1518,19 +1562,19 @@ async def dummy_mapping_job(): await arq_worker.async_run() await arq_worker.run_check() - # Verify that mapped variants were created - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 4 + # Verify that mapping records were created + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 4 # Verify score set mapping state assert sample_score_set.mapping_state == MappingState.complete assert sample_score_set.mapping_errors is None - # Verify that each variant has a corresponding mapped variant + # Verify that each variant has a corresponding mapping record variants = ( session.query(Variant) - .join(MappedVariant, MappedVariant.variant_id == Variant.id) - .filter(Variant.score_set_id == sample_score_set.id, MappedVariant.current.is_(True)) + .join(MappingRecord, MappingRecord.variant_id == Variant.id) + .filter(Variant.score_set_id == sample_score_set.id, MappingRecord.current.is_(True)) .all() ) assert len(variants) == 4 @@ -1606,19 +1650,19 @@ async def dummy_mapping_job(): await arq_worker.async_run() await arq_worker.run_check() - # Verify that mapped variants were created - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 4 + # Verify that mapping records were created + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 4 # Verify score set mapping state assert sample_score_set.mapping_state == MappingState.complete assert sample_score_set.mapping_errors is None - # Verify that each variant has a corresponding mapped variant + # Verify that each variant has a corresponding mapping record variants = ( session.query(Variant) - .join(MappedVariant, MappedVariant.variant_id == Variant.id) - .filter(Variant.score_set_id == sample_score_set.id, MappedVariant.current.is_(True)) + .join(MappingRecord, MappingRecord.variant_id == Variant.id) + .filter(Variant.score_set_id == sample_score_set.id, MappingRecord.current.is_(True)) .all() ) assert len(variants) == 4 @@ -1690,9 +1734,9 @@ async def dummy_mapping_job(): in sample_score_set.mapping_errors["error_message"] ) - # Verify that no mapped variants were created - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 0 + # Verify that no mapping records were created + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 0 # Verify that no annotation statuses were created annotation_statuses = session.query(VariantAnnotationStatus).all() @@ -1744,9 +1788,9 @@ async def dummy_mapping_job(): in sample_score_set.mapping_errors["error_message"] ) - # Verify that no mapped variants were created - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 0 + # Verify that no mapping records were created + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 0 # Verify that no annotation statuses were created annotation_statuses = session.query(VariantAnnotationStatus).all() From b70611f5028089884b4771a174cec5fd968fef82 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Sat, 6 Jun 2026 18:57:29 -0700 Subject: [PATCH 015/252] fixup into cisphased hgvs commit --- tests/lib/test_variants.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/lib/test_variants.py b/tests/lib/test_variants.py index ca9c2b0b3..9cffaa793 100644 --- a/tests/lib/test_variants.py +++ b/tests/lib/test_variants.py @@ -46,6 +46,18 @@ def test_get_hgvs_from_post_mapped_cis_phased_block(): assert result is None +def test_get_hgvs_from_post_mapped_cis_phased_block_combine_cis(): + # combine_cis collapses the cis-phased members into one bracketed expression. + result = get_hgvs_from_post_mapped(TEST_VALID_POST_MAPPED_VRS_CIS_PHASED_BLOCK, combine_cis=True) + assert result == "NM_003345:p.[Asp5Phe;Asp5Phe]" + + +def test_get_hgvs_from_post_mapped_single_allele_combine_cis_is_unbracketed(): + # A single-variant post-mapped allele is unaffected by combine_cis. + result = get_hgvs_from_post_mapped(TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X, combine_cis=True) + assert result == TEST_HGVS_IDENTIFIER + + def test_get_hgvs_from_post_mapped_single_allele_vrs_1(): with pytest.raises(ValueError): get_hgvs_from_post_mapped(TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS1_X) From c6a06e665076d56e6d1dc23665ea7365bfd0f663 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Sat, 6 Jun 2026 18:58:35 -0700 Subject: [PATCH 016/252] feat(worker): provide a SeqRepo data proxy in the worker context Add a seqrepo_data_proxy to the worker context in both the standalone context and the on_job_start hook, so jobs performing VRS translation have a SeqRepo-backed data proxy available alongside the cdot HGVS data provider. Mirror it in the mock worker context used by tests. --- src/mavedb/worker/settings/lifecycle.py | 4 +++- tests/worker/conftest_optional.py | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/mavedb/worker/settings/lifecycle.py b/src/mavedb/worker/settings/lifecycle.py index 54a0b4c76..5be66bd2d 100644 --- a/src/mavedb/worker/settings/lifecycle.py +++ b/src/mavedb/worker/settings/lifecycle.py @@ -9,7 +9,7 @@ from concurrent import futures -from mavedb.data_providers.services import cdot_rest +from mavedb.data_providers.services import cdot_rest, seqrepo_data_proxy def standalone_ctx(): @@ -18,6 +18,7 @@ def standalone_ctx(): ctx["pool"] = futures.ProcessPoolExecutor() ctx["redis"] = None # Redis connection can be set up here if needed. ctx["hdp"] = cdot_rest() + ctx["seqrepo"] = seqrepo_data_proxy() ctx["state"] = {} # Additional context setup can be added here as needed. @@ -37,6 +38,7 @@ async def shutdown(ctx): async def on_job_start(ctx): ctx["hdp"] = cdot_rest() + ctx["seqrepo"] = seqrepo_data_proxy() ctx["state"] = {} diff --git a/tests/worker/conftest_optional.py b/tests/worker/conftest_optional.py index 0f1d2e95f..ccd4208a0 100644 --- a/tests/worker/conftest_optional.py +++ b/tests/worker/conftest_optional.py @@ -4,6 +4,7 @@ import pytest from arq import ArqRedis from cdot.hgvs.dataproviders import RESTDataProvider +from ga4gh.vrs.dataproxy import SeqRepoDataProxy from sqlalchemy.orm import Session from mavedb.worker.lib.managers.job_manager import JobManager @@ -52,6 +53,7 @@ def mock_worker_ctx(): """Create a mock worker context dictionary for testing.""" mock_redis = Mock(spec=ArqRedis) mock_hdp = Mock(spec=RESTDataProvider) + mock_seqrepo = Mock(spec=SeqRepoDataProxy) mock_pool = Mock(spec=ProcessPoolExecutor) # Don't mock the session itself to allow real DB interactions in tests @@ -60,5 +62,6 @@ def mock_worker_ctx(): return { "redis": mock_redis, "hdp": mock_hdp, + "seqrepo": mock_seqrepo, "pool": mock_pool, } From a8e16363080b87a284bdbfa3ac3e81d21f28bdf5 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Sat, 6 Jun 2026 18:58:41 -0700 Subject: [PATCH 017/252] fixup into mapper changes --- tests/helpers/util/setup/worker.py | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/tests/helpers/util/setup/worker.py b/tests/helpers/util/setup/worker.py index cb6949b4e..815dd1a90 100644 --- a/tests/helpers/util/setup/worker.py +++ b/tests/helpers/util/setup/worker.py @@ -4,13 +4,13 @@ from sqlalchemy import select +from mavedb.models.enums.job_pipeline import JobStatus from mavedb.models.score_set import ScoreSet as ScoreSetDbModel from mavedb.models.variant import Variant from mavedb.worker.jobs import ( create_variants_for_score_set, map_variants_for_score_set, ) -from mavedb.models.enums.job_pipeline import JobStatus from mavedb.worker.lib.managers.job_manager import JobManager from tests.helpers.constants import ( TEST_CODING_LAYER, @@ -23,6 +23,16 @@ ) +# Placeholder reference accessions for qualifying target-relative HGVS in mock +# post-mapped output, keyed by the HGVS kind prefix (``c.``/``n.``/``g.``/``p.``). +_PLACEHOLDER_ACCESSIONS = { + "c.": "NM_999999.1", + "n.": "NR_999999.1", + "g.": "NC_999999.1", + "p.": "NP_999999.1", +} + + async def create_variants_in_score_set( session, mock_s3_client, score_df, count_df, mock_worker_ctx, variant_creation_run ): @@ -128,6 +138,10 @@ async def construct_mock_mapping_output( "target_gene_identifier": target.name, "alignment_level": layer, "preferred": idx == 0, + # The accession this level was aligned against — a transcript (NM_) + # only at the coding level. Reverse translation resolves its coding + # transcript hint from the cdna entry's reference_accession. + "reference_accession": _PLACEHOLDER_ACCESSIONS[f"{layer}."], "tool_name": "dcd-mapping", "tool_version": "pytest.0.0", "tool_parameters": {}, @@ -155,10 +169,15 @@ async def construct_mock_mapping_output( "alignment_level": default_alignment_level, } - # Don't alter HGVS strings in post mapped output. This makes it considerably - # easier to assert correctness in tests. + # Reuse the variant's own HGVS in post-mapped output to keep assertions simple. + # Real mapper post-mapped HGVS is always accession-qualified (it is mapped onto + # a reference sequence), so prefix a placeholder accession when the source HGVS + # is target-relative (a bare ``c.``/``n.``/``g.``/``p.`` with no accession). if with_post_mapped: - mapped_score["post_mapped"]["expressions"][0]["value"] = variant.hgvs_nt or variant.hgvs_pro + hgvs = variant.hgvs_nt or variant.hgvs_pro + if hgvs and ":" not in hgvs: + hgvs = f"{_PLACEHOLDER_ACCESSIONS.get(hgvs[:2], 'NM_999999.1')}:{hgvs}" + mapped_score["post_mapped"]["expressions"][0]["value"] = hgvs # Skip every other variant if not with_all_variants if not with_all_variants and idx % 2 == 0: From 3bd97176ea7d96e0c607c15a27a5398afcfc0858 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Sat, 6 Jun 2026 18:59:45 -0700 Subject: [PATCH 018/252] feat(worker): add reverse translation job for cross-level HGVS alleles MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a reverse_translate_variants_for_score_set worker job that builds the cross-level HGVS equivalence class for every mapped variant in a score set, persisting the candidates as non-authoritative alleles. - for each current authoritative MappingRecord, collapse the assay-level HGVS to its protein consequence and expand to all coding/genomic candidates via a single batched construct_equivalent_variants call from the variant-annotation library, run off-loop in a thread - resolve each record's coding (NM_) transcript from its target gene's cdna alignment, falling back to a batched UTA NP_→NM_ lookup for protein-level mappings; records with no coding transcript are skipped - translate each candidate to VRS and write it as a get-or-created Allele linked non-authoritatively to the record, deduping by vrs_digest and never relinking the record's authoritative allele - supersede the prior live derived links with the new set in one gap-free operation via ValidTime.supersede_live_where - record per-variant cross_level_translation annotation status (success / failed / skipped), retaining per-candidate translation errors as metadata; the job fails only when every variant fails - add WorkerCoordinateTranslator and NullTranscriptSource adapters for the library's translation ports, deferring AssemblyMapper init so its network calls do not fire under mocked unit tests - get_or_create_allele helper, job registration, and a pipeline dependency placing reverse translation after mapping and before CAR submission TODO#765: a re-run supersedes the whole derived set wholesale because re-mapping re-mints the records; idempotent records would let unchanged derived links stay live. --- src/mavedb/lib/variant_translations.py | 20 + src/mavedb/lib/workflow/definitions.py | 12 +- src/mavedb/worker/jobs/__init__.py | 2 + src/mavedb/worker/jobs/registry.py | 9 + .../jobs/variant_processing/__init__.py | 2 + .../variant_processing/reverse_translation.py | 477 +++++++++ src/mavedb/worker/lib/translation_ports.py | 59 ++ tests/worker/jobs/conftest.py | 32 + .../test_reverse_translation.py | 919 ++++++++++++++++++ 9 files changed, 1531 insertions(+), 1 deletion(-) create mode 100644 src/mavedb/worker/jobs/variant_processing/reverse_translation.py create mode 100644 src/mavedb/worker/lib/translation_ports.py create mode 100644 tests/worker/jobs/variant_processing/test_reverse_translation.py diff --git a/src/mavedb/lib/variant_translations.py b/src/mavedb/lib/variant_translations.py index c24c5ed95..701cc17d1 100644 --- a/src/mavedb/lib/variant_translations.py +++ b/src/mavedb/lib/variant_translations.py @@ -7,10 +7,12 @@ from typing import cast +from sqlalchemy import select from sqlalchemy.dialects.postgresql import insert from sqlalchemy.engine import CursorResult from sqlalchemy.orm import Session +from mavedb.models.allele import Allele from mavedb.models.variant_translation import VariantTranslation @@ -39,3 +41,21 @@ def upsert_variant_translations(db: Session, translations: list[tuple[str, str]] created = result.rowcount existing = len(unique) - created return created, existing + + +def get_or_create_allele(db: Session, allele_draft: Allele) -> Allele: + """Return the existing Allele matching ``allele_draft``'s vrs_digest, else add the draft. + + This is a get-or-create, not an upsert: a matching row is returned untouched, and on + a miss the draft is added to the session (not flushed). The draft is never used to + update an existing row. + + NOTE: This function does not persist the returned Allele to the database; the caller + is responsible for committing the session. + """ + existing = db.scalars(select(Allele).where(Allele.vrs_digest == allele_draft.vrs_digest)).one_or_none() + if existing is not None: + return existing + + db.add(allele_draft) + return allele_draft diff --git a/src/mavedb/lib/workflow/definitions.py b/src/mavedb/lib/workflow/definitions.py index 056ff5309..3d3b70511 100644 --- a/src/mavedb/lib/workflow/definitions.py +++ b/src/mavedb/lib/workflow/definitions.py @@ -20,6 +20,16 @@ def annotation_pipeline_job_definitions( [(upstream_mapping_key, DependencyType.SUCCESS_REQUIRED)] if upstream_mapping_key else [] ) return [ + { + "key": "reverse_translate_variants_for_score_set", + "function": "reverse_translate_variants_for_score_set", + "type": JobType.MAPPED_VARIANT_ANNOTATION, + "params": { + "correlation_id": None, # Required param to be filled in at runtime + "score_set_id": None, # Required param to be filled in at runtime + }, + "dependencies": mapping_dep, + }, { "key": "submit_score_set_mappings_to_car", "function": "submit_score_set_mappings_to_car", @@ -29,7 +39,7 @@ def annotation_pipeline_job_definitions( "score_set_id": None, # Required param to be filled in at runtime "updater_id": None, # Required param to be filled in at runtime }, - "dependencies": mapping_dep, + "dependencies": [("reverse_translate_variants_for_score_set", DependencyType.SUCCESS_REQUIRED)], }, { "key": "warm_clingen_cache", diff --git a/src/mavedb/worker/jobs/__init__.py b/src/mavedb/worker/jobs/__init__.py index e421bbad2..488398117 100644 --- a/src/mavedb/worker/jobs/__init__.py +++ b/src/mavedb/worker/jobs/__init__.py @@ -33,11 +33,13 @@ from mavedb.worker.jobs.variant_processing.mapping import ( map_variants_for_score_set, ) +from mavedb.worker.jobs.variant_processing.reverse_translation import reverse_translate_variants_for_score_set __all__ = [ # Variant processing jobs "create_variants_for_score_set", "map_variants_for_score_set", + "reverse_translate_variants_for_score_set", # External service integration jobs "submit_score_set_mappings_to_car", "submit_score_set_mappings_to_ldh", diff --git a/src/mavedb/worker/jobs/registry.py b/src/mavedb/worker/jobs/registry.py index ba9754a05..c445098ef 100644 --- a/src/mavedb/worker/jobs/registry.py +++ b/src/mavedb/worker/jobs/registry.py @@ -32,6 +32,7 @@ from mavedb.worker.jobs.variant_processing import ( create_variants_for_score_set, map_variants_for_score_set, + reverse_translate_variants_for_score_set, ) # All job functions for ARQ worker @@ -39,6 +40,7 @@ # Variant processing jobs create_variants_for_score_set, map_variants_for_score_set, + reverse_translate_variants_for_score_set, # External service jobs submit_score_set_mappings_to_car, submit_score_set_mappings_to_ldh, @@ -102,6 +104,13 @@ "key": "map_variants_for_score_set", "type": JobType.VARIANT_MAPPING, }, + reverse_translate_variants_for_score_set: { + "dependencies": [], + "params": {"score_set_id": None, "correlation_id": None}, + "function": "reverse_translate_variants_for_score_set", + "key": "reverse_translate_variants_for_score_set", + "type": JobType.VARIANT_MAPPING, + }, submit_score_set_mappings_to_car: { "dependencies": [], "params": {"score_set_id": None, "correlation_id": None}, diff --git a/src/mavedb/worker/jobs/variant_processing/__init__.py b/src/mavedb/worker/jobs/variant_processing/__init__.py index a6df09753..240de5e28 100644 --- a/src/mavedb/worker/jobs/variant_processing/__init__.py +++ b/src/mavedb/worker/jobs/variant_processing/__init__.py @@ -10,8 +10,10 @@ from .mapping import ( map_variants_for_score_set, ) +from .reverse_translation import reverse_translate_variants_for_score_set __all__ = [ "create_variants_for_score_set", "map_variants_for_score_set", + "reverse_translate_variants_for_score_set", ] diff --git a/src/mavedb/worker/jobs/variant_processing/reverse_translation.py b/src/mavedb/worker/jobs/variant_processing/reverse_translation.py new file mode 100644 index 000000000..61ca1d883 --- /dev/null +++ b/src/mavedb/worker/jobs/variant_processing/reverse_translation.py @@ -0,0 +1,477 @@ +"""Reverse translation worker job — builds cross-level HGVS equivalence classes. + +For each mapped variant in a score set, calls construct_equivalent_variants from +the variant-annotation library to produce all coding and genomic HGVS candidates +encoding the same protein consequence. The candidates are written as non-authoritative +Allele rows linked to the existing MappingRecord via MappingRecordAllele. +""" + +import asyncio +import contextlib +import dataclasses +import functools +import logging +import os +from typing import Any, NamedTuple, Sequence + +from ga4gh.vrs.extras.translator import AlleleTranslator +from sqlalchemy import select +from variant_annotation.lib.accessions import looks_like_refseq_protein_accession +from variant_annotation.lib.clients.uta import UtaClient, connect_uta +from variant_annotation.lib.translation import construct_equivalent_variants +from variant_annotation.lib.translation.types import TranslationConfig, VariantInput, WtCodonMode + +from mavedb.lib.annotation_status_manager import AnnotationStatusManager +from mavedb.lib.hgvs import extract_accession +from mavedb.lib.types.workflow import JobExecutionOutcome +from mavedb.lib.variant_translations import get_or_create_allele +from mavedb.lib.vrs_utils import translate_hgvs_to_variation +from mavedb.models.allele import Allele as AlleleDbModel +from mavedb.models.enums.annotation_layer import AnnotationLayer +from mavedb.models.enums.annotation_type import AnnotationType +from mavedb.models.enums.job_pipeline import AnnotationFailureCategory, AnnotationStatus, FailureCategory +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.score_set import ScoreSet +from mavedb.models.target_gene import TargetGene +from mavedb.models.target_gene_mapping import TargetGeneMapping +from mavedb.models.variant import Variant +from mavedb.worker.jobs.utils.setup import validate_job_params +from mavedb.worker.lib.decorators.pipeline_management import with_pipeline_management +from mavedb.worker.lib.managers.job_manager import JobManager +from mavedb.worker.lib.translation_ports import NullTranscriptSource, WorkerCoordinateTranslator + +logger = logging.getLogger(__name__) + +# Job defaults when no `translation_config` override is supplied: enumerate the full coding +# equivalence class — every synonymous codon, indels included. wt_codon_mode "all" requires +# include_indels=True (TranslationConfig enforces this). +_DEFAULT_TRANSLATION_CONFIG: dict[str, Any] = { + "include_indels": True, + "wt_codon_mode": WtCodonMode.ALL, +} + + +class _TranscriptResolution(NamedTuple): + """A mapping record paired with what we know about its coding transcript before the + batched UTA lookup: the gene-level cdna transcript if the mapper supplied one, and the + protein accession still awaiting NP_→NM_ resolution otherwise. Exactly one of the two is + set for a resolvable record; both are None when neither path applies (record is skipped).""" + + rec: MappingRecord + variant: Variant + gene_transcript: str | None + protein_accession: str | None + + +def _coding_transcripts_for_proteins(protein_accessions: set[str]) -> dict[str, str]: + """Resolve each RefSeq protein accession (NP_/XP_…) to its preferred coding transcript + via UTA's associated_accessions table. + + Protein-level mappings carry no coding transcript — the mapper emits reference_accession + only for cdna alignments — so reverse-translating them relies on the NP_→NM_ association + UTA records. Resolution and the multi-transcript preference order both live in the + variant-annotation library's UtaClient; we own only the connection lifecycle here so the + long-lived worker process does not leak UTA connections across jobs. + """ + if not protein_accessions: + return {} + + uta_db_url = (os.environ.get("UTA_DB_URL") or "").strip() + if not uta_db_url: + raise RuntimeError("UTA_DB_URL must be set to resolve protein→transcript associations.") + + with contextlib.closing(connect_uta(uta_db_url)) as conn: + client = UtaClient(conn) + return { + pro_ac: transcript + for pro_ac in sorted(protein_accessions) + if (transcript := client.transcript_for_protein(pro_ac)) is not None + } + + +def _build_translation_config(overrides: dict[str, Any] | None) -> TranslationConfig: + """Build the variant-annotation TranslationConfig from optional job-param overrides. + + Each key in `overrides` is a TranslationConfig field (that dataclass is the source of truth + for the available knobs) and wins over the job defaults in _DEFAULT_TRANSLATION_CONFIG; + absent or None means use the defaults. wt_codon_mode is coerced to the enum so a JSON string + value ("all"/"unambiguous"/"none") works. + + Raises ValueError with an actionable message — listing the offending value and the valid + options — for an unknown field, an invalid wt_codon_mode, or an invalid combination (e.g. a + wt_codon_mode other than "none" without include_indels). The allowed-field set is derived + from TranslationConfig itself, so it never drifts from the library. + """ + config_kwargs: dict[str, Any] = {**_DEFAULT_TRANSLATION_CONFIG, **(overrides or {})} + + allowed_fields = {field.name for field in dataclasses.fields(TranslationConfig)} + unknown_fields = set(config_kwargs) - allowed_fields + if unknown_fields: + raise ValueError( + f"Unknown translation_config option(s): {', '.join(sorted(unknown_fields))}. " + f"Valid options: {', '.join(sorted(allowed_fields))}." + ) + + raw_mode = config_kwargs.get("wt_codon_mode") + if raw_mode is not None: + try: + config_kwargs["wt_codon_mode"] = WtCodonMode(raw_mode) + except ValueError: + valid_modes = ", ".join(repr(mode.value) for mode in WtCodonMode) + raise ValueError( + f"Invalid translation_config wt_codon_mode {raw_mode!r}. Valid values: {valid_modes}." + ) from None + + try: + return TranslationConfig(**config_kwargs) + except ValueError as exc: + raise ValueError(f"Invalid translation_config: {exc}") from exc + + +@with_pipeline_management +async def reverse_translate_variants_for_score_set( + ctx: dict, job_id: int, job_manager: JobManager +) -> JobExecutionOutcome: + """Build the cross-level HGVS equivalence class for every mapped variant in the score set. + + Reads current MappingRecords that carry an hgvs_assay_level string, collapses each + to its ProteinConsequence, and expands to all coding/genomic HGVS candidates via a + single batched subprocess call to the variant-annotation library. Each candidate is + written as a non-authoritative Allele linked to the MappingRecord. + + Required job_params: + - score_set_id (int): ID of the ScoreSet to process + - correlation_id (str): Correlation ID for tracking + + Optional job_params: + - translation_config (dict): Overrides for any variant-annotation TranslationConfig + field (e.g. include_indels, wt_codon_mode, max_indel_size). Omitted keys fall back to + the job defaults in _DEFAULT_TRANSLATION_CONFIG (full codon equivalence class, indels + included). + """ + job = job_manager.get_job() + validate_job_params(["score_set_id", "correlation_id"], job) + + score_set_id: int = job.job_params["score_set_id"] # type: ignore[index] + correlation_id: str = job.job_params["correlation_id"] # type: ignore[index] + translation_config = _build_translation_config(job.job_params.get("translation_config")) # type: ignore[union-attr] + score_set = job_manager.db.scalars(select(ScoreSet).where(ScoreSet.id == score_set_id)).one() + + job_manager.save_to_context( + { + "application": "mavedb-worker", + "function": "reverse_translate_variants_for_score_set", + "resource": score_set.urn, + "correlation_id": correlation_id, + } + ) + job_manager.update_progress(0, 100, "Starting reverse translation job.") + logger.info(msg="Started reverse translation job.", extra=job_manager.logging_context()) + + # Build {target_gene_id -> NM_ transcript} from the cdna TargetGeneMappings the mapper + # emits. Reverse translation must run against the cdna (NM_) transcript. + cdna_transcript_by_gene: dict[int, str | None] = dict( + job_manager.db.execute( + select(TargetGeneMapping.target_gene_id, TargetGeneMapping.reference_accession) + .join(TargetGene, TargetGene.id == TargetGeneMapping.target_gene_id) + .where(TargetGene.score_set_id == score_set_id) + .where(TargetGeneMapping.alignment_level == AnnotationLayer.cdna) + .where(TargetGeneMapping.reference_accession.isnot(None)) + ) + .tuples() + .all() + ) + + # Load current, authoritative, and successfully-mapped MappingRecords along with their + # target gene (for the coding-transcript lookup) and parent Variant. + rows: Sequence[tuple[MappingRecord, Variant, int]] = ( + job_manager.db.execute( + select(MappingRecord, Variant, TargetGeneMapping.target_gene_id) + .join(MappingRecordAllele, MappingRecord.id == MappingRecordAllele.mapping_record_id) + .join(Variant, MappingRecord.variant_id == Variant.id) + .outerjoin(TargetGeneMapping, MappingRecord.target_gene_mapping_id == TargetGeneMapping.id) + .where(Variant.score_set_id == score_set_id) + .where(MappingRecord.current) + .where(MappingRecordAllele.is_authoritative.is_(True)) + .where(MappingRecordAllele.current) + ) + .tuples() + .all() + ) + + if not rows: + logger.warning( + msg="No current and authoritative mapping records found for this score set.", + extra=job_manager.logging_context(), + ) + job_manager.db.flush() + return JobExecutionOutcome.succeeded(data={"translated": 0, "failed": 0, "skipped": 0, "alleles_created": 0}) + + # Resolve each record's coding transcript. Genomic/cdna mappings share their target + # gene's cdna alignment reference; protein mappings have none (the mapper emits + # reference_accession only for cdna alignments), so gather their protein accessions and + # resolve NP_→NM_ from UTA in a single batch query below. + transcript_resolutions: list[_TranscriptResolution] = [] + protein_accessions: set[str] = set() + for rec, variant, target_gene_id in rows: + coding_accession = cdna_transcript_by_gene.get(target_gene_id) + protein_accession = None + if not coding_accession and rec.hgvs_assay_level is not None: + raw_accession = extract_accession(rec.hgvs_assay_level) + if looks_like_refseq_protein_accession(raw_accession): + protein_accession = raw_accession + protein_accessions.add(raw_accession) + + transcript_resolutions.append(_TranscriptResolution(rec, variant, coding_accession, protein_accession)) + + transcript_by_protein = _coding_transcripts_for_proteins(protein_accessions) + + # Build VariantInputs, supplying the resolved coding transcript for every input + # regardless of its own assay level (p./c./g. all collapse to a ProteinConsequence + # anchored on that transcript). Object identity on each VariantInput is preserved + # through the call so we can correlate TranslationResult.input back to its MappingRecord. + # + # A record with no coding transcript (e.g. a regulatory element aligned only at the + # genomic level, or a protein with no UTA association) has no protein consequence to + # reverse-translate; it is skipped and recorded as SKIPPED rather than counted as a failure. + variant_inputs: list[Any] = [] + variant_input_map: dict[int, tuple[MappingRecord, Variant]] = {} + skipped_variants: list[tuple[MappingRecord, Variant]] = [] + for p in transcript_resolutions: + transcript = p.gene_transcript or ( + transcript_by_protein.get(p.protein_accession) if p.protein_accession else None + ) + if not transcript or not p.rec.hgvs_assay_level: + skipped_variants.append((p.rec, p.variant)) + continue + + inp = VariantInput(hgvs=p.rec.hgvs_assay_level, transcript=transcript) + variant_inputs.append(inp) + variant_input_map[id(inp)] = (p.rec, p.variant) + + total = len(variant_inputs) + job_manager.save_to_context({"total_variants": total, "skipped_variants": len(skipped_variants)}) + job_manager.update_progress( + 10, + 100, + f"Prepared {total} variants for reverse translation ({len(skipped_variants)} skipped, no coding transcript).", + ) + logger.info( + msg=f"Running reverse translation for {total} variants ({len(skipped_variants)} skipped).", + extra=job_manager.logging_context(), + ) + + # construct_equivalent_variants is I/O-bound (blocks on a subprocess); run in the + # default thread pool rather than the process pool to avoid pickling the port objects. + coordinates = WorkerCoordinateTranslator(ctx["hdp"]) + transcripts = NullTranscriptSource() + loop = asyncio.get_running_loop() + job_manager.update_progress(20, 100, "Running reverse translation subprocess.") + results, errors = await loop.run_in_executor( + None, + functools.partial( + construct_equivalent_variants, + variant_inputs, + transcripts=transcripts, + coordinates=coordinates, + config=translation_config, + ), + ) + + job_manager.update_progress(70, 100, "Writing translated alleles to database.") + logger.info( + msg=f"Translation complete: {len(results)} succeeded, {len(errors)} failed.", + extra=job_manager.logging_context(), + ) + + translated = 0 + failed = 0 + alleles_created = 0 + annotation_manager = AnnotationStatusManager(job_manager.db, job_run_id=job_manager.job_id) + allele_translator = AlleleTranslator(ctx["seqrepo"]) + + current_record_ids = ( + select(MappingRecord.id) + .join(Variant, MappingRecord.variant_id == Variant.id) + .where(Variant.score_set_id == score_set_id) + .where(MappingRecord.current.is_(True)) + ) + + # Collect new MappingRecord -> Allele links and defer linkage until all candidates are processed. + # This allows us to supersede the prior live derived links with the new set in one atomic operation at the end, + # so the retire and insert share a timestamp and there is no gap between the old and new sets of live links. + new_links: list[MappingRecordAllele] = [] + + # The live authoritative (record, allele) pairs. A derived candidate that equals a record's + # authoritative allele is not linked again. + authoritative_pairs: set[tuple[int, int]] = { + (record_id, allele_id) + for record_id, allele_id in job_manager.db.execute( + select(MappingRecordAllele.mapping_record_id, MappingRecordAllele.allele_id) + .where(MappingRecordAllele.is_authoritative.is_(True)) + .where(MappingRecordAllele.current) + .where(MappingRecordAllele.mapping_record_id.in_(current_record_ids)) + ) + .tuples() + .all() + if record_id is not None and allele_id is not None + } + + for result in results: + rec, variant = variant_input_map[id(result.input)] + candidate_count = 0 + + # Equivalence generation may surface the same VRS object more than once. + # Dedup by vrs_digest per mapping record to avoid duplicate links. + seen_digests: set[str] = set() + failed_candidates: list[dict[str, str]] = [] + candidates: list[tuple[str, AnnotationLayer, str]] = [ + (hgvs_g, AnnotationLayer.genomic, "hgvs_g") for hgvs_g in result.hgvs_g_candidates + ] + [(hgvs_c, AnnotationLayer.cdna, "hgvs_c") for hgvs_c in result.hgvs_c_candidates] + + for hgvs, level, hgvs_field in candidates: + # A candidate the equivalence class produced may be a form ga4gh cannot + # translate (an intronic projection, an unsupported edge case, a malformed + # bracketed expression). A variant with at least one translatable candidate still + # advances the pipeline; only a variant where every candidate fails is + # recorded as failed (with the per-candidate errors retained as metadata). + try: + variation = translate_hgvs_to_variation(hgvs, allele_translator) + except Exception as e: + logger.warning( + msg=f"Failed to translate candidate HGVS to VRS: {hgvs} ({e})", + extra=job_manager.logging_context(), + ) + failed_candidates.append({"hgvs": hgvs, "level": level.value, "error": str(e)}) + continue + + if variation.id in seen_digests: + continue + + seen_digests.add(variation.id) + draft_allele = AlleleDbModel( + vrs_digest=variation.id, + post_mapped=variation.model_dump(), + level=level, + **{hgvs_field: hgvs}, # type: ignore[arg-type] + ) + allele = get_or_create_allele(job_manager.db, draft_allele) + job_manager.db.flush() + + if (rec.id, allele.id) in authoritative_pairs: + continue # already linked + + new_links.append( + MappingRecordAllele( + mapping_record_id=rec.id, + allele_id=allele.id, + is_authoritative=False, + ) + ) + candidate_count += 1 + + alleles_created += candidate_count + annotation_metadata = { + "hgvs_input": result.input.hgvs, + "hgvs_c_candidates": result.hgvs_c_candidates, + "hgvs_g_candidates": result.hgvs_g_candidates, + "alleles_created": candidate_count, + "failed_candidates": failed_candidates, + } + + # No translatable candidates and failures mean the variant failed reverse translation. No + # failures and no candidates is a success with no alleles created. + if candidate_count == 0 and failed_candidates: + failed += 1 + annotation_manager.add_annotation( + variant_id=variant.id, + annotation_type=AnnotationType.CROSS_LEVEL_TRANSLATION, + status=AnnotationStatus.FAILED, + failure_category=AnnotationFailureCategory.UNKNOWN, + annotation_data={ + "error_message": "All candidate HGVS failed VRS translation.", + "annotation_metadata": annotation_metadata, + }, + ) + else: + translated += 1 + annotation_manager.add_annotation( + variant_id=variant.id, + annotation_type=AnnotationType.CROSS_LEVEL_TRANSLATION, + status=AnnotationStatus.SUCCESS, + annotation_data={"annotation_metadata": annotation_metadata}, + ) + + # Supersede the prior live derived links with the new set in one gap-free operation. + # TODO#765: a re-run supersedes the whole derived set wholesale because re-mapping re-mints the + # records, so we cannot tell which derivations are unchanged; idempotent mapping records would let + # unchanged derived links stay live instead of being retired and recreated. + MappingRecordAllele.supersede_live_where( + job_manager.db, + new_links, + MappingRecordAllele.is_authoritative.is_(False), + MappingRecordAllele.mapping_record_id.in_(current_record_ids), + ) + + for error in errors: + _rec, variant = variant_input_map[id(error.input)] + failed += 1 + annotation_manager.add_annotation( + variant_id=variant.id, + annotation_type=AnnotationType.CROSS_LEVEL_TRANSLATION, + status=AnnotationStatus.FAILED, + failure_category=AnnotationFailureCategory.UNKNOWN, + annotation_data={ + "error_message": error.error, + "annotation_metadata": {"hgvs_input": error.input.hgvs}, + }, + ) + + skipped = len(skipped_variants) + for rec, variant in skipped_variants: + annotation_manager.add_annotation( + variant_id=variant.id, + annotation_type=AnnotationType.CROSS_LEVEL_TRANSLATION, + status=AnnotationStatus.SKIPPED, + annotation_data={ + "annotation_metadata": { + "hgvs_input": rec.hgvs_assay_level, + "reason": "No coding transcript for target gene; no protein consequence to reverse-translate.", + } + }, + ) + + annotation_manager.flush() + + job_manager.save_to_context( + { + "translated": translated, + "failed": failed, + "skipped": skipped, + "alleles_created": alleles_created, + } + ) + logger.info( + msg=( + f"Reverse translation complete: {translated} translated, {failed} failed, " + f"{skipped} skipped, {alleles_created} alleles created." + ), + extra=job_manager.logging_context(), + ) + job_manager.db.flush() + + if translated == 0 and failed > 0: + logger.error( + msg="All variant reverse translations failed.", + extra=job_manager.logging_context(), + ) + return JobExecutionOutcome.failed( + reason="All variant reverse translations failed.", + data={"translated": 0, "failed": failed, "skipped": skipped, "alleles_created": 0}, + failure_category=FailureCategory.DATA_ERROR, + ) + + return JobExecutionOutcome.succeeded( + data={"translated": translated, "failed": failed, "skipped": skipped, "alleles_created": alleles_created} + ) diff --git a/src/mavedb/worker/lib/translation_ports.py b/src/mavedb/worker/lib/translation_ports.py new file mode 100644 index 000000000..467d2f93a --- /dev/null +++ b/src/mavedb/worker/lib/translation_ports.py @@ -0,0 +1,59 @@ +"""Worker-side adapters for the variant_annotation translation ports. + +construct_equivalent_variants depends on two protocols — CoordinateTranslator +and TranscriptSource. These are the worker's concrete implementations, so jobs +can pass them in without importing the port definitions directly. + +WorkerCoordinateTranslator defers AssemblyMapper initialization until the first +call — the hgvs library makes network calls on mapper construction that must not +fire in unit-test contexts where construct_equivalent_variants is mocked. +""" + +from __future__ import annotations + +from typing import Any + + +class WorkerCoordinateTranslator: + """CoordinateTranslator backed by the worker's HGVS data provider.""" + + def __init__(self, hdp: Any) -> None: + self._hdp = hdp + self._parser: Any = None + self._mapper: Any = None + + def _ensure_initialized(self) -> None: + if self._mapper is None: + import hgvs.assemblymapper + import hgvs.parser + + self._parser = hgvs.parser.Parser() + self._mapper = hgvs.assemblymapper.AssemblyMapper( + self._hdp, + assembly_name="GRCh38", + alt_aln_method="splign", + ) + + def c_to_p(self, c_hgvs: str) -> str: + self._ensure_initialized() + return str(self._mapper.c_to_p(self._parser.parse(c_hgvs))) + + def g_to_c(self, g_hgvs: str, transcript: str) -> str: + self._ensure_initialized() + return str(self._mapper.g_to_t(self._parser.parse(g_hgvs), transcript)) + + def c_to_g(self, c_hgvs: str) -> str: + self._ensure_initialized() + return str(self._mapper.c_to_g(self._parser.parse(c_hgvs))) + + +class NullTranscriptSource: + """Null TranscriptSource — the reverse-translation job resolves every coding transcript + itself (including the UTA NP_→NM_ lookup) and always supplies it via VariantInput.transcript, + so the library never needs to resolve one.""" + + def transcript_for_protein(self, protein_accession: str) -> str | None: + return None + + def codon_at(self, transcript: str, aa_position: int) -> str | None: + return None diff --git a/tests/worker/jobs/conftest.py b/tests/worker/jobs/conftest.py index de835a56d..eac380862 100644 --- a/tests/worker/jobs/conftest.py +++ b/tests/worker/jobs/conftest.py @@ -44,6 +44,38 @@ def map_variants_sample_params(with_populated_domain_data, sample_score_set, sam } +@pytest.fixture +def reverse_translate_variants_sample_params(with_populated_domain_data, sample_score_set): + """Provide sample parameters for reverse_translate_variants_for_score_set job.""" + + return { + "score_set_id": sample_score_set.id, + "correlation_id": "sample-reverse-translation-correlation-id", + } + + +@pytest.fixture +def sample_independent_reverse_translation_run(reverse_translate_variants_sample_params): + """Create a JobRun instance for the reverse_translate_variants_for_score_set job.""" + + return JobRun( + urn="test:reverse_translate_variants_for_score_set", + job_type="reverse_translate_variants_for_score_set", + job_function="reverse_translate_variants_for_score_set", + max_retries=3, + retry_count=0, + job_params=reverse_translate_variants_sample_params, + ) + + +@pytest.fixture +def with_reverse_translation_run(session, sample_independent_reverse_translation_run): + """Add a reverse_translate_variants_for_score_set job run to the session.""" + + session.add(sample_independent_reverse_translation_run) + session.commit() + + @pytest.fixture def link_gnomad_variants_sample_params(with_populated_domain_data, sample_score_set): """Provide sample parameters for create_variants_for_score_set job.""" diff --git a/tests/worker/jobs/variant_processing/test_reverse_translation.py b/tests/worker/jobs/variant_processing/test_reverse_translation.py new file mode 100644 index 000000000..c7e434845 --- /dev/null +++ b/tests/worker/jobs/variant_processing/test_reverse_translation.py @@ -0,0 +1,919 @@ +# ruff: noqa: E402 + +import pytest + +pytest.importorskip("arq") + +from asyncio.unix_events import _UnixSelectorEventLoop +from unittest.mock import patch + +from variant_annotation.lib.translation.types import TranslationError, TranslationResult, WtCodonMode + +from mavedb.lib.types.workflow import JobExecutionOutcome +from mavedb.models.allele import Allele +from mavedb.models.enums.annotation_layer import AnnotationLayer +from mavedb.models.enums.job_pipeline import JobStatus +from mavedb.models.job_run import JobRun +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.variant import Variant +from mavedb.models.variant_annotation_status import VariantAnnotationStatus +from mavedb.worker.jobs.variant_processing.mapping import map_variants_for_score_set +from mavedb.worker.jobs.variant_processing.reverse_translation import ( + _build_translation_config, + reverse_translate_variants_for_score_set, +) +from mavedb.worker.lib.managers.job_manager import JobManager +from tests.helpers.constants import TEST_GA4GH_IDENTIFIER +from tests.helpers.util.setup.worker import construct_mock_mapping_output + +pytestmark = pytest.mark.usefixtures("patch_db_session_ctxmgr") + +# Module under test, used as the patch root for the external annotation-package calls. +RT_MODULE = "mavedb.worker.jobs.variant_processing.reverse_translation" + + +class FakeVrsVariation: + """Stand-in for a ga4gh.vrs variation returned by translate_hgvs_to_variation. + + The reverse translation job only reads ``.id`` (for the VRS digest / dedup key) + and ``.model_dump()`` (persisted into the ``post_mapped`` JSONB column), so the + fake exposes exactly those. ``vrs_type`` lets a test assert whether a candidate + became a single Allele or a CisPhasedBlock. + """ + + def __init__(self, vrs_id: str, vrs_type: str = "Allele"): + self.id = vrs_id + self.vrs_type = vrs_type + + def model_dump(self) -> dict: + return {"type": self.vrs_type, "id": self.id} + + +def fake_construct(results_by_hgvs: dict, errors_by_hgvs: dict | None = None): + """Build a stand-in for ``construct_equivalent_variants``. + + The job correlates each TranslationResult/TranslationError back to its originating + mapping record via object identity on ``result.input``, so the fake must echo the + exact VariantInput instances it was handed. Candidates are keyed by the input's + assay-level HGVS string. + """ + errors_by_hgvs = errors_by_hgvs or {} + + def _construct(inputs, *, transcripts, coordinates, config): + results, errors = [], [] + for inp in inputs: + if inp.hgvs in errors_by_hgvs: + errors.append(TranslationError(input=inp, error=errors_by_hgvs[inp.hgvs])) + elif inp.hgvs in results_by_hgvs: + c_candidates, g_candidates = results_by_hgvs[inp.hgvs] + results.append( + TranslationResult( + input=inp, + hgvs_c_candidates=list(c_candidates), + hgvs_g_candidates=list(g_candidates), + ) + ) + else: + errors.append(TranslationError(input=inp, error=f"no stub result for {inp.hgvs!r}")) + return results, errors + + return _construct + + +def fake_translate(id_by_hgvs: dict, type_by_hgvs: dict | None = None, errors_by_hgvs: dict | None = None): + """Build a stand-in for ``translate_hgvs_to_variation`` mapping candidate HGVS -> VRS id. + + Unknown HGVS get a deterministic id derived from the string so distinct candidates + stay distinct; supplying explicit ids lets a test force two candidates to collapse. + ``type_by_hgvs`` lets a test mark a bracketed candidate as a ``CisPhasedBlock`` (the + real translate_hgvs_to_variation returns one for cis-phased multivariants). + ``errors_by_hgvs`` makes a candidate raise, standing in for an HGVS form ga4gh cannot + translate to VRS. + """ + type_by_hgvs = type_by_hgvs or {} + errors_by_hgvs = errors_by_hgvs or {} + + def _translate(hgvs: str, translator=None) -> FakeVrsVariation: + if hgvs in errors_by_hgvs: + raise ValueError(errors_by_hgvs[hgvs]) + return FakeVrsVariation( + id_by_hgvs.get(hgvs, f"ga4gh:VA.{hgvs}"), + type_by_hgvs.get(hgvs, "Allele"), + ) + + return _translate + + +async def _map_variants(session, mock_worker_ctx, mapping_run, score_set, with_layers=frozenset({"g", "c", "p"})): + """Run the (mocked) mapping job so the score set's variants gain authoritative + MappingRecords/Alleles — the real inputs the reverse translation job reads.""" + + async def dummy_mapping_job(): + return await construct_mock_mapping_output(session=session, score_set=score_set, with_layers=with_layers) + + with patch.object(_UnixSelectorEventLoop, "run_in_executor", return_value=dummy_mapping_job()): + result = await map_variants_for_score_set( + mock_worker_ctx, + mapping_run.id, + JobManager(session, mock_worker_ctx["redis"], mapping_run.id), + ) + + assert result.status == JobStatus.SUCCEEDED + session.commit() + + +async def _reverse_translate(session, mock_worker_ctx, rt_run): + return await reverse_translate_variants_for_score_set( + mock_worker_ctx, + rt_run.id, + JobManager(session, mock_worker_ctx["redis"], rt_run.id), + ) + + +def _cross_level_statuses(session, score_set_id, status=None): + query = ( + session.query(VariantAnnotationStatus) + .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) + .filter( + Variant.score_set_id == score_set_id, + VariantAnnotationStatus.annotation_type == "cross_level_translation", + ) + ) + if status is not None: + query = query.filter(VariantAnnotationStatus.status == status) + return query.all() + + +def _non_authoritative_links(session): + return session.query(MappingRecordAllele).filter(MappingRecordAllele.is_authoritative.is_(False)).all() + + +@pytest.mark.unit +class TestBuildTranslationConfig: + """Unit tests for _build_translation_config — the optional translation_config job param.""" + + def test_none_uses_job_defaults(self): + config = _build_translation_config(None) + assert config.include_indels is True + assert config.wt_codon_mode == WtCodonMode.ALL + + def test_empty_dict_uses_job_defaults(self): + config = _build_translation_config({}) + assert config.include_indels is True + assert config.wt_codon_mode == WtCodonMode.ALL + + def test_overrides_win_and_string_wt_codon_mode_is_coerced(self): + config = _build_translation_config({"wt_codon_mode": "unambiguous", "max_indel_size": 7}) + assert config.wt_codon_mode == WtCodonMode.UNAMBIGUOUS + assert config.max_indel_size == 7 + assert config.include_indels is True # untouched default preserved + + def test_can_disable_indels_with_none_mode(self): + config = _build_translation_config({"wt_codon_mode": "none", "include_indels": False}) + assert config.wt_codon_mode == WtCodonMode.NONE + assert config.include_indels is False + + def test_rejects_invalid_wt_codon_mode_with_actionable_message(self): + with pytest.raises(ValueError, match=r"wt_codon_mode 'bogus'.*Valid values"): + _build_translation_config({"wt_codon_mode": "bogus"}) + + def test_rejects_wt_codon_mode_without_indels(self): + # TranslationConfig enforces: a wt_codon_mode other than "none" requires include_indels. + with pytest.raises(ValueError, match="translation_config"): + _build_translation_config({"wt_codon_mode": "all", "include_indels": False}) + + def test_rejects_unknown_field_and_lists_valid_options(self): + with pytest.raises( + ValueError, match=r"Unknown translation_config option\(s\): not_a_real_field.*Valid options" + ): + _build_translation_config({"not_a_real_field": 1}) + + +@pytest.mark.unit +@pytest.mark.asyncio +class TestReverseTranslateVariantsForScoreSetUnit: + """Unit tests for the reverse_translate_variants_for_score_set job.""" + + async def test_no_mapping_records_is_a_noop( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """With no current, post-mapped mapping records there is nothing to translate.""" + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert isinstance(result, JobExecutionOutcome) + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 0, "failed": 0, "skipped": 0, "alleles_created": 0} + + # No candidate alleles, links, or annotations were produced. + assert session.query(Allele).count() == 0 + assert _non_authoritative_links(session) == [] + assert _cross_level_statuses(session, sample_score_set.id) == [] + + async def test_creates_genomic_and_coding_candidate_alleles( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """A mapped variant expands into one non-authoritative allele per equivalence-class + candidate, each linked to the originating mapping record.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + assay_hgvs = "NM_000000.1:c.1A>G" + g_candidate = "NC_000001.11:g.1000A>G" + c_candidate = "NM_000001.1:c.5A>G" + construct = fake_construct({assay_hgvs: ([c_candidate], [g_candidate])}) + translate = fake_translate({g_candidate: "ga4gh:VA.genomic", c_candidate: "ga4gh:VA.coding"}) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 2} + + # Two non-authoritative links were created, both for our one mapping record. + non_auth_links = _non_authoritative_links(session) + assert len(non_auth_links) == 2 + + genomic_allele = session.query(Allele).filter(Allele.vrs_digest == "ga4gh:VA.genomic").one() + assert genomic_allele.level == AnnotationLayer.genomic.value + assert genomic_allele.hgvs_g == g_candidate + assert genomic_allele.hgvs_c is None + assert genomic_allele.transcript == "NC_000001.11" + assert genomic_allele.post_mapped == {"type": "Allele", "id": "ga4gh:VA.genomic"} + + coding_allele = session.query(Allele).filter(Allele.vrs_digest == "ga4gh:VA.coding").one() + assert coding_allele.level == AnnotationLayer.cdna.value + assert coding_allele.hgvs_c == c_candidate + assert coding_allele.hgvs_g is None + assert coding_allele.transcript == "NM_000001.1" + + # The candidate alleles are linked to the same mapping record as the authoritative allele. + mapping_record = session.query(MappingRecord).filter(MappingRecord.variant_id == variant.id).one() + assert {link.mapping_record_id for link in non_auth_links} == {mapping_record.id} + + statuses = _cross_level_statuses(session, sample_score_set.id) + assert len(statuses) == 1 + assert statuses[0].status == "success" + + async def test_transcript_is_queryable_via_derived_expression( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """Allele.transcript is a derived hybrid_property — filtering on it in SQL resolves to + the accession of the populated HGVS column, with no stored transcript column.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + assay_hgvs = "NM_000000.1:c.1A>G" + g_candidate = "NC_000001.11:g.1000A>G" + c_candidate = "NM_000001.1:c.5A>G" + construct = fake_construct({assay_hgvs: ([c_candidate], [g_candidate])}) + translate = fake_translate({g_candidate: "ga4gh:VA.genomic", c_candidate: "ga4gh:VA.coding"}) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + # The SQL expression derives the accession from the populated HGVS column. + genomic = session.query(Allele).filter(Allele.transcript == "NC_000001.11").all() + assert [a.vrs_digest for a in genomic] == ["ga4gh:VA.genomic"] + coding = session.query(Allele).filter(Allele.transcript == "NM_000001.1").all() + assert [a.vrs_digest for a in coding] == ["ga4gh:VA.coding"] + + async def test_cis_phased_multivariant_candidate_is_persisted_as_a_block( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """A bracketed genomic candidate (non-adjacent codon components) is translated into a + CisPhasedBlock and persisted like any other candidate allele, keyed by its CPB digest.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + assay_hgvs = "NM_000000.1:c.1A>G" + block_candidate = "NC_000001.11:g.[1000A>G;1002T>C]" + construct = fake_construct({assay_hgvs: ([], [block_candidate])}) + translate = fake_translate( + {block_candidate: "ga4gh:CPB.block"}, + type_by_hgvs={block_candidate: "CisPhasedBlock"}, + ) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 1} + + block_allele = session.query(Allele).filter(Allele.vrs_digest == "ga4gh:CPB.block").one() + assert block_allele.level == AnnotationLayer.genomic.value + # The full bracketed expression is stored verbatim; its accession anchors the row. + assert block_allele.hgvs_g == block_candidate + assert block_allele.transcript == "NC_000001.11" + assert block_allele.post_mapped == {"type": "CisPhasedBlock", "id": "ga4gh:CPB.block"} + + assert len(_non_authoritative_links(session)) == 1 + + async def test_duplicate_candidate_digests_are_deduped_per_record( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """When a coding and genomic candidate resolve to the same VRS digest, only one + allele/link is written for that mapping record (per the library's dedup contract).""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + assay_hgvs = "NM_000000.1:c.1A>G" + g_candidate = "NC_000001.11:g.1000A>G" + c_candidate = "NM_000001.1:c.5A>G" + construct = fake_construct({assay_hgvs: ([c_candidate], [g_candidate])}) + # Both candidates collapse to the same VRS digest. + translate = fake_translate({g_candidate: "ga4gh:VA.shared", c_candidate: "ga4gh:VA.shared"}) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 1} + assert len(_non_authoritative_links(session)) == 1 + assert session.query(Allele).filter(Allele.vrs_digest == "ga4gh:VA.shared").count() == 1 + + async def test_candidate_equal_to_authoritative_allele_is_not_relinked( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """A derived candidate whose digest equals the record's authoritative (assay-measured) + allele is not linked again — the record already links that allele authoritatively, and a + derived duplicate would surface the measured variant twice. No new link and no new allele; + an empty derived set is still a success with nothing to add.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + # Exactly one allele exists after mapping: the authoritative post-mapped allele. + assert session.query(Allele).count() == 1 + authoritative_allele = session.query(Allele).one() + assert authoritative_allele.vrs_digest == TEST_GA4GH_IDENTIFIER + + assay_hgvs = "NM_000000.1:c.1A>G" + c_candidate = "NM_000001.1:c.5A>G" + construct = fake_construct({assay_hgvs: ([c_candidate], [])}) + # The candidate resolves to the same digest as the authoritative allele. + translate = fake_translate({c_candidate: TEST_GA4GH_IDENTIFIER}) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 0} + + # The authoritative allele was reused, not duplicated. + assert session.query(Allele).count() == 1 + # No derived link was added: the candidate equals the record's authoritative allele. + assert _non_authoritative_links(session) == [] + + async def test_independent_rerun_retires_prior_links_and_keeps_current_stable( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """An independent second run is idempotent: it retires the prior derived links (closing + valid_to) and writes fresh live ones, so the set of *current* non-authoritative links is + stable while the superseded links are retained as history — the partial unique index on + live links is never violated and no alleles are duplicated.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + assay_hgvs = "NM_000000.1:c.1A>G" + g_candidate = "NC_000001.11:g.1000A>G" + c_candidate = "NM_000001.1:c.5A>G" + construct = fake_construct({assay_hgvs: ([c_candidate], [g_candidate])}) + translate = fake_translate({g_candidate: "ga4gh:VA.genomic", c_candidate: "ga4gh:VA.coding"}) + + # First run: two live derived links. + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + first = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert first.status == JobStatus.SUCCEEDED + first_links = _non_authoritative_links(session) + assert len(first_links) == 2 + assert all(link.valid_to is None for link in first_links) + + # Second, independent run with identical inputs. + rerun = JobRun( + urn="test:reverse_translate_variants_for_score_set:rerun", + job_type="reverse_translate_variants_for_score_set", + job_function="reverse_translate_variants_for_score_set", + max_retries=3, + retry_count=0, + job_params=dict(sample_independent_reverse_translation_run.job_params), + ) + session.add(rerun) + session.commit() + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + second = await _reverse_translate(session, mock_worker_ctx, rerun) + + assert second.status == JobStatus.SUCCEEDED + + links = _non_authoritative_links(session) + live = [link for link in links if link.valid_to is None] + retired = [link for link in links if link.valid_to is not None] + + # The live set is stable (still exactly the two candidates) and the prior run's links are + # retained as closed history rather than deleted. + assert len(live) == 2 + assert len(retired) == 2 + + # Gap-free handoff: the prior links closed at exactly the instant the new links opened, under + # one supersession timestamp — so a point-in-time query never lands in a hole between runs, + # even though the translation loop ran (and could have committed) between retire and insert. + assert {link.valid_to for link in retired} == {link.valid_from for link in live} + assert len({link.valid_from for link in live}) == 1 + + # The two candidate alleles are reused across runs, not duplicated. + assert session.query(Allele).filter(Allele.vrs_digest.in_(["ga4gh:VA.genomic", "ga4gh:VA.coding"])).count() == 2 + + # The cross-level translation status is versioned the same way: prior retired, one current. + statuses = _cross_level_statuses(session, sample_score_set.id) + assert len(statuses) == 2 + assert len([s for s in statuses if s.current]) == 1 + + async def test_all_failures_fail_the_job( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """If every variant's translation errors, the job fails and records FAILED annotations.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + assay_hgvs = "NM_000000.1:c.1A>G" + construct = fake_construct({}, errors_by_hgvs={assay_hgvs: "forward translation failed"}) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", fake_translate({})), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.FAILED + assert result.data == {"translated": 0, "failed": 1, "skipped": 0, "alleles_created": 0} + + assert _non_authoritative_links(session) == [] + failed_statuses = _cross_level_statuses(session, sample_score_set.id, status="failed") + assert len(failed_statuses) == 1 + assert failed_statuses[0].error_message == "forward translation failed" + # Library/input-level failures still carry the assay-level HGVS as metadata. + assert failed_statuses[0].annotation_metadata == {"hgvs_input": assay_hgvs} + + async def test_partial_success_succeeds_with_mixed_annotations( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """One variant translating and another erroring yields a SUCCEEDED job with both a + success and a failure annotation.""" + variant_ok = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + variant_err = Variant( + score_set_id=sample_score_set.id, + urn="variant:2", + hgvs_nt="NM_000000.1:c.2G>T", + hgvs_pro="NP_000000.1:p.Val2Leu", + data={}, + ) + session.add_all([variant_ok, variant_err]) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + g_candidate = "NC_000001.11:g.1000A>G" + construct = fake_construct( + {"NM_000000.1:c.1A>G": ([], [g_candidate])}, + errors_by_hgvs={"NM_000000.1:c.2G>T": "no consequence"}, + ) + translate = fake_translate({g_candidate: "ga4gh:VA.genomic"}) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 1, "failed": 1, "skipped": 0, "alleles_created": 1} + + assert len(_cross_level_statuses(session, sample_score_set.id, status="success")) == 1 + assert len(_cross_level_statuses(session, sample_score_set.id, status="failed")) == 1 + assert len(_non_authoritative_links(session)) == 1 + + async def test_partial_candidate_translation_failure_keeps_success_with_metadata( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """One candidate translating and another failing VRS translation leaves the variant a + SUCCESS (one allele created) while retaining the dropped candidate in metadata.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + assay_hgvs = "NM_000000.1:c.1A>G" + good_candidate = "NC_000001.11:g.1000A>G" + bad_candidate = "NC_000001.11:g.999A>T" + construct = fake_construct({assay_hgvs: ([], [good_candidate, bad_candidate])}) + translate = fake_translate( + {good_candidate: "ga4gh:VA.genomic"}, + errors_by_hgvs={bad_candidate: "untranslatable form"}, + ) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 1} + assert len(_non_authoritative_links(session)) == 1 + + statuses = _cross_level_statuses(session, sample_score_set.id, status="success") + assert len(statuses) == 1 + failed_candidates = statuses[0].annotation_metadata["failed_candidates"] + assert failed_candidates == [{"hgvs": bad_candidate, "level": "genomic", "error": "untranslatable form"}] + + async def test_all_candidates_failing_translation_marks_variant_failed( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """When every candidate fails VRS translation, the variant is FAILED, no allele is + created, and the per-candidate errors are retained in metadata.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + assay_hgvs = "NM_000000.1:c.1A>G" + bad_candidate = "NC_000001.11:g.999A>T" + construct = fake_construct({assay_hgvs: ([], [bad_candidate])}) + translate = fake_translate({}, errors_by_hgvs={bad_candidate: "untranslatable form"}) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.FAILED + assert result.data == {"translated": 0, "failed": 1, "skipped": 0, "alleles_created": 0} + assert _non_authoritative_links(session) == [] + + failed_statuses = _cross_level_statuses(session, sample_score_set.id, status="failed") + assert len(failed_statuses) == 1 + assert failed_statuses[0].error_message == "All candidate HGVS failed VRS translation." + metadata = failed_statuses[0].annotation_metadata + assert metadata["hgvs_input"] == assay_hgvs + assert metadata["failed_candidates"] == [ + {"hgvs": bad_candidate, "level": "genomic", "error": "untranslatable form"} + ] + + async def test_no_coding_transcript_is_skipped_not_failed( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """A target gene aligned only at the genomic level has no coding transcript, so its + variants are SKIPPED (no protein consequence) rather than attempted and failed.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NC_000001.11:g.1000A>G", + data={}, + ) + session.add(variant) + session.commit() + # Genomic-only mapping: no cdna TargetGeneMapping, so no coding transcript to anchor on. + await _map_variants( + session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set, with_layers={"g"} + ) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", fake_construct({})), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", fake_translate({})), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 0, "failed": 0, "skipped": 1, "alleles_created": 0} + + # No translation attempted: no candidate alleles or links, and the variant is + # recorded as SKIPPED rather than FAILED. + assert _non_authoritative_links(session) == [] + assert _cross_level_statuses(session, sample_score_set.id, status="failed") == [] + assert len(_cross_level_statuses(session, sample_score_set.id, status="skipped")) == 1 + + async def test_translation_config_param_overrides_job_defaults( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """A translation_config job param overrides the job's TranslationConfig defaults, and the + resulting config is passed through to construct_equivalent_variants.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + assay_hgvs = "NM_000000.1:c.1A>G" + g_candidate = "NC_000001.11:g.1000A>G" + delegate = fake_construct({assay_hgvs: ([], [g_candidate])}) + translate = fake_translate({g_candidate: "ga4gh:VA.genomic"}) + + captured: dict = {} + + def capturing_construct(inputs, *, transcripts, coordinates, config): + captured["config"] = config + return delegate(inputs, transcripts=transcripts, coordinates=coordinates, config=config) + + run = sample_independent_reverse_translation_run + run.job_params = {**run.job_params, "translation_config": {"wt_codon_mode": "unambiguous", "max_indel_size": 7}} + session.add(run) + session.commit() + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", capturing_construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + result = await _reverse_translate(session, mock_worker_ctx, run) + + assert result.status == JobStatus.SUCCEEDED + config = captured["config"] + assert config.wt_codon_mode == WtCodonMode.UNAMBIGUOUS + assert config.max_indel_size == 7 + # Defaults the param did not override are preserved. + assert config.include_indels is True + + async def test_translation_config_defaults_when_param_absent( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """With no translation_config param the job falls back to its sensible defaults + (full codon equivalence class, indels included).""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + assay_hgvs = "NM_000000.1:c.1A>G" + g_candidate = "NC_000001.11:g.1000A>G" + delegate = fake_construct({assay_hgvs: ([], [g_candidate])}) + translate = fake_translate({g_candidate: "ga4gh:VA.genomic"}) + + captured: dict = {} + + def capturing_construct(inputs, *, transcripts, coordinates, config): + captured["config"] = config + return delegate(inputs, transcripts=transcripts, coordinates=coordinates, config=config) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", capturing_construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + config = captured["config"] + assert config.include_indels is True + assert config.wt_codon_mode == WtCodonMode.ALL + + async def test_protein_level_transcript_resolved_via_uta( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """A protein-only mapping has no cdna alignment transcript, so its coding transcript + is resolved NP_→NM_ via UTA and supplied as the VariantInput hint.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + # Protein-only mapping: no cdna TargetGeneMapping, so the transcript must come from UTA. + await _map_variants( + session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set, with_layers={"p"} + ) + + assay_hgvs = "NP_000000.1:p.Met1Val" + g_candidate = "NC_000001.11:g.1000A>G" + translate = fake_translate({g_candidate: "ga4gh:VA.genomic"}) + + # Capture the transcript hint each VariantInput was built with, then delegate to the + # normal stub for the translation result. + captured_hints: dict[str, str | None] = {} + delegate = fake_construct({assay_hgvs: ([], [g_candidate])}) + + def capturing_construct(inputs, *, transcripts, coordinates, config): + captured_hints.update({inp.hgvs: inp.transcript for inp in inputs}) + return delegate(inputs, transcripts=transcripts, coordinates=coordinates, config=config) + + with ( + patch(f"{RT_MODULE}._coding_transcripts_for_proteins", lambda accessions: {"NP_000000.1": "NM_000111.1"}), + patch(f"{RT_MODULE}.construct_equivalent_variants", capturing_construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 1} + # The NP_ accession was resolved to its NM_ transcript and passed as the hint. + assert captured_hints == {assay_hgvs: "NM_000111.1"} From 7371d44b7b0cb84cb7ffc967fe06957c11d3b6e9 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Mon, 8 Jun 2026 12:16:08 -0700 Subject: [PATCH 019/252] fix(reverse-translation): anchor cdna transcript lookup to mapping run date Previously the cdna-transcript lookup was keyed by target_gene_id alone, which meant a re-mapped score set could bind a stale NM_ transcript from an earlier run rather than the one the current run emitted. - Key cdna_transcript_by_run on (target_gene_id, mapped_date); within a key the highest-id row wins, so a same-run replacement takes precedence. - Carry mapped_date through the MappingRecord query so each record anchors to its own run's cdna row. - Add _TranscriptResolutionSkipReason to distinguish recoverable skips (protein-coding target, transcript unresolved) from correct skips (non-coding/regulatory target, no protein consequence). Emit skip_category in annotation_metadata. - Add target_gene_id to _TranscriptResolution so skip classification can look up the target's TargetCategory. - Add Mapped[] type annotations to TargetGene.id and .category columns. - New tests: genomic-accession coding target RT, latest-cdna-row-within- run selection, stale-cdna-row isolation, skip classification (coding recoverable vs. regulatory correct), and cdna TGM persistence in the mapping job. Refs mavedb-api#763 --- src/mavedb/models/target_gene.py | 4 +- .../variant_processing/reverse_translation.py | 117 ++++++-- .../jobs/variant_processing/test_mapping.py | 45 ++++ .../test_reverse_translation.py | 250 ++++++++++++++++++ 4 files changed, 397 insertions(+), 19 deletions(-) diff --git a/src/mavedb/models/target_gene.py b/src/mavedb/models/target_gene.py index 671a4380f..5419c3478 100644 --- a/src/mavedb/models/target_gene.py +++ b/src/mavedb/models/target_gene.py @@ -23,10 +23,10 @@ class TargetGene(Base): __tablename__ = "target_genes" - id = Column(Integer, primary_key=True) + id: Mapped[int] = Column(Integer, primary_key=True) name = Column(String, nullable=False) - category = Column( + category: Mapped[TargetCategory] = Column( Enum(TargetCategory, create_constraint=True, length=32, native_enum=False, validate_strings=True), nullable=False, ) diff --git a/src/mavedb/worker/jobs/variant_processing/reverse_translation.py b/src/mavedb/worker/jobs/variant_processing/reverse_translation.py index 61ca1d883..c5f14bf68 100644 --- a/src/mavedb/worker/jobs/variant_processing/reverse_translation.py +++ b/src/mavedb/worker/jobs/variant_processing/reverse_translation.py @@ -12,6 +12,8 @@ import functools import logging import os +from datetime import date +from enum import Enum from typing import Any, NamedTuple, Sequence from ga4gh.vrs.extras.translator import AlleleTranslator @@ -30,6 +32,7 @@ from mavedb.models.enums.annotation_layer import AnnotationLayer from mavedb.models.enums.annotation_type import AnnotationType from mavedb.models.enums.job_pipeline import AnnotationFailureCategory, AnnotationStatus, FailureCategory +from mavedb.models.enums.target_category import TargetCategory from mavedb.models.mapping_record import MappingRecord from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.score_set import ScoreSet @@ -62,6 +65,40 @@ class _TranscriptResolution(NamedTuple): variant: Variant gene_transcript: str | None protein_accession: str | None + target_gene_id: int | None + + +class _TranscriptResolutionSkipReason(Enum): + NO_ASSAY_LEVEL_HGVS = "no_assay_level_hgvs" + TRANSCRIPT_UNRESOLVED = "transcript_unresolved" + NO_CODING_TRANSCRIPT = "no_coding_transcript" + + @classmethod + def classify( + cls, resolution: _TranscriptResolution, category: TargetCategory | None + ) -> tuple["_TranscriptResolutionSkipReason", str]: + """Classify why a record was skipped, distinguishing recoverable from correct skips. + + A protein-coding target with no resolvable transcript is *recoverable* (it should now be + rare -- the mapper selects a MANE transcript for coding genomic-accession targets); a + regulatory / non-coding target has no protein consequence and is a *correct* skip. + """ + if not resolution.rec.hgvs_assay_level: + return ( + cls.NO_ASSAY_LEVEL_HGVS, + "No assay-level HGVS available to reverse-translate.", + ) + if category == TargetCategory.protein_coding: + return ( + cls.TRANSCRIPT_UNRESOLVED, + "Protein-coding target but no coding transcript could be resolved " + "(no cdna TargetGeneMapping and no NP_->NM_ association). Recoverable: " + "re-map or check transcript selection.", + ) + return ( + cls.NO_CODING_TRANSCRIPT, + "Non-coding/regulatory target has no protein consequence to reverse-translate.", + ) def _coding_transcripts_for_proteins(protein_accessions: set[str]) -> dict[str, str]: @@ -169,25 +206,55 @@ async def reverse_translate_variants_for_score_set( job_manager.update_progress(0, 100, "Starting reverse translation job.") logger.info(msg="Started reverse translation job.", extra=job_manager.logging_context()) - # Build {target_gene_id -> NM_ transcript} from the cdna TargetGeneMappings the mapper - # emits. Reverse translation must run against the cdna (NM_) transcript. - cdna_transcript_by_gene: dict[int, str | None] = dict( - job_manager.db.execute( - select(TargetGeneMapping.target_gene_id, TargetGeneMapping.reference_accession) + # Build {(target_gene_id, run date) -> NM_ transcript} from the cdna TargetGeneMappings + # the mapper emits. Reverse translation must run against the cdna (NM_) transcript. + # + # TargetGeneMapping is not valid-time-versioned and re-mapping cannot delete prior rows + # (retired MappingRecords still FK them), so a re-mapped target accumulates several cdna + # rows. There is no run anchor on the row, so we approximate one with mapped_date and key + # the lookup by (target gene, run date): each record resolves only the cdna transcript + # from *its own* run, matched below against the run date of the genomic TargetGeneMapping + # the record points to. All of a run's TargetGeneMappings share one mapped_date (stamped + # once per job), so a record and its run's cdna row match even if the job spans midnight. + # Within a single key, order by id so the newest row wins. + # + # TODO#763: mapped_date is day-granular, so two re-maps on the *same calendar day* collide + # on the key. If a later same-day re-map emits no cdna row (e.g. transcript selection hit + # a transient Ensembl/gene-normalizer failure, or the target genuinely lost its coding + # transcript) but an earlier same-day one did, RT can still bind the earlier (stale) + # transcript. This keying narrows the window from "any prior run ever" to "a same-day + # re-map" -- the bug now requires re-runs in quick succession -- but only an actual run + # anchor / versioned home for reference identity closes it fully. + cdna_transcript_by_run: dict[tuple[int, date | None], str | None] = { + (target_gene_id, mapped_date): reference_accession + for target_gene_id, mapped_date, reference_accession in job_manager.db.execute( + select( + TargetGeneMapping.target_gene_id, + TargetGeneMapping.mapped_date, + TargetGeneMapping.reference_accession, + ) .join(TargetGene, TargetGene.id == TargetGeneMapping.target_gene_id) .where(TargetGene.score_set_id == score_set_id) .where(TargetGeneMapping.alignment_level == AnnotationLayer.cdna) .where(TargetGeneMapping.reference_accession.isnot(None)) + .order_by(TargetGeneMapping.id) ) .tuples() .all() - ) + } # Load current, authoritative, and successfully-mapped MappingRecords along with their # target gene (for the coding-transcript lookup) and parent Variant. - rows: Sequence[tuple[MappingRecord, Variant, int]] = ( + # The joined TargetGeneMapping is the record's own (genomic) mapping row, so its + # mapped_date is this record's run date -- used to anchor the cdna lookup to the run. + rows: Sequence[tuple[MappingRecord, Variant, int, date | None]] = ( job_manager.db.execute( - select(MappingRecord, Variant, TargetGeneMapping.target_gene_id) + select( + MappingRecord, + Variant, + TargetGeneMapping.target_gene_id, + TargetGeneMapping.mapped_date, + ) .join(MappingRecordAllele, MappingRecord.id == MappingRecordAllele.mapping_record_id) .join(Variant, MappingRecord.variant_id == Variant.id) .outerjoin(TargetGeneMapping, MappingRecord.target_gene_mapping_id == TargetGeneMapping.id) @@ -214,8 +281,8 @@ async def reverse_translate_variants_for_score_set( # resolve NP_→NM_ from UTA in a single batch query below. transcript_resolutions: list[_TranscriptResolution] = [] protein_accessions: set[str] = set() - for rec, variant, target_gene_id in rows: - coding_accession = cdna_transcript_by_gene.get(target_gene_id) + for rec, variant, target_gene_id, mapped_date in rows: + coding_accession = cdna_transcript_by_run.get((target_gene_id, mapped_date)) protein_accession = None if not coding_accession and rec.hgvs_assay_level is not None: raw_accession = extract_accession(rec.hgvs_assay_level) @@ -223,10 +290,23 @@ async def reverse_translate_variants_for_score_set( protein_accession = raw_accession protein_accessions.add(raw_accession) - transcript_resolutions.append(_TranscriptResolution(rec, variant, coding_accession, protein_accession)) + transcript_resolutions.append( + _TranscriptResolution(rec, variant, coding_accession, protein_accession, target_gene_id) + ) transcript_by_protein = _coding_transcripts_for_proteins(protein_accessions) + # Target gene category per gene, so a skip can be classified honestly: a coding target + # with no resolvable transcript is recoverable (should now be rare), while a regulatory / + # non-coding target has no protein consequence and is a correct skip. + target_category_by_gene: dict[int, TargetCategory] = dict( + job_manager.db.execute( + select(TargetGene.id, TargetGene.category).where(TargetGene.score_set_id == score_set_id) + ) + .tuples() + .all() + ) + # Build VariantInputs, supplying the resolved coding transcript for every input # regardless of its own assay level (p./c./g. all collapse to a ProteinConsequence # anchored on that transcript). Object identity on each VariantInput is preserved @@ -237,13 +317,13 @@ async def reverse_translate_variants_for_score_set( # reverse-translate; it is skipped and recorded as SKIPPED rather than counted as a failure. variant_inputs: list[Any] = [] variant_input_map: dict[int, tuple[MappingRecord, Variant]] = {} - skipped_variants: list[tuple[MappingRecord, Variant]] = [] + skipped_variants: list[_TranscriptResolution] = [] for p in transcript_resolutions: transcript = p.gene_transcript or ( transcript_by_protein.get(p.protein_accession) if p.protein_accession else None ) if not transcript or not p.rec.hgvs_assay_level: - skipped_variants.append((p.rec, p.variant)) + skipped_variants.append(p) continue inp = VariantInput(hgvs=p.rec.hgvs_assay_level, transcript=transcript) @@ -429,15 +509,18 @@ async def reverse_translate_variants_for_score_set( ) skipped = len(skipped_variants) - for rec, variant in skipped_variants: + for p in skipped_variants: + category = target_category_by_gene.get(p.target_gene_id) if p.target_gene_id is not None else None + skip_category, reason = _TranscriptResolutionSkipReason.classify(p, category) annotation_manager.add_annotation( - variant_id=variant.id, + variant_id=p.variant.id, annotation_type=AnnotationType.CROSS_LEVEL_TRANSLATION, status=AnnotationStatus.SKIPPED, annotation_data={ "annotation_metadata": { - "hgvs_input": rec.hgvs_assay_level, - "reason": "No coding transcript for target gene; no protein consequence to reverse-translate.", + "hgvs_input": p.rec.hgvs_assay_level, + "skip_category": skip_category.value, + "reason": reason, } }, ) diff --git a/tests/worker/jobs/variant_processing/test_mapping.py b/tests/worker/jobs/variant_processing/test_mapping.py index b28f61564..74c9acb1c 100644 --- a/tests/worker/jobs/variant_processing/test_mapping.py +++ b/tests/worker/jobs/variant_processing/test_mapping.py @@ -18,6 +18,7 @@ from mavedb.models.enums.mapping_state import MappingState from mavedb.models.mapping_record import MappingRecord from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.target_gene_mapping import TargetGeneMapping from mavedb.models.variant import Variant from mavedb.models.variant_annotation_status import VariantAnnotationStatus from mavedb.worker.jobs.variant_processing.mapping import map_variants_for_score_set @@ -479,6 +480,50 @@ async def dummy_mapping_job(): assert annotation_statuses[0].annotation_type == "vrs_mapping" assert annotation_statuses[0].status == "success" + async def test_persists_cdna_target_gene_mapping_with_reference_accession_and_null_qc( + self, + session, + with_independent_processing_runs, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_score_set, + ): + """An identity cdna TargetGeneMapping (a cdna layer with no per-variant scores + joining it) persists with its reference_accession (NM_) and null QC/counts -- the + artifact reverse translation consumes to resolve the projection transcript.""" + variant = Variant( + score_set_id=sample_score_set.id, + hgvs_nt="NC_000001.11:g.1000A>G", + data={}, + ) + session.add(variant) + session.commit() + + # Genomic assay layer + an identity cdna layer (no cdna per-variant scores). + async def dummy_mapping_job(): + return await construct_mock_mapping_output( + session=session, score_set=sample_score_set, with_layers={"g", "c"} + ) + + with patch.object(_UnixSelectorEventLoop, "run_in_executor", return_value=dummy_mapping_job()): + result = await map_variants_for_score_set( + mock_worker_ctx, + sample_independent_variant_mapping_run.id, + JobManager(session, mock_worker_ctx["redis"], sample_independent_variant_mapping_run.id), + ) + assert result.status == JobStatus.SUCCEEDED + + cdna_tgms = ( + session.query(TargetGeneMapping).filter(TargetGeneMapping.alignment_level == AnnotationLayer.cdna).all() + ) + assert len(cdna_tgms) == len(sample_score_set.target_genes) + tgm = cdna_tgms[0] + assert tgm.reference_accession == "NM_999999.1" + # Identity row: no mapped_variant joins it, so QC/counts are null. + assert tgm.total_variants is None + assert tgm.variants_mapped_cleanly is None + assert tgm.percent_identity is None + async def test_map_variants_for_score_set_success_no_successful_mapping( self, session, diff --git a/tests/worker/jobs/variant_processing/test_reverse_translation.py b/tests/worker/jobs/variant_processing/test_reverse_translation.py index c7e434845..459ec53db 100644 --- a/tests/worker/jobs/variant_processing/test_reverse_translation.py +++ b/tests/worker/jobs/variant_processing/test_reverse_translation.py @@ -5,6 +5,7 @@ pytest.importorskip("arq") from asyncio.unix_events import _UnixSelectorEventLoop +from datetime import timedelta from unittest.mock import patch from variant_annotation.lib.translation.types import TranslationError, TranslationResult, WtCodonMode @@ -13,9 +14,11 @@ from mavedb.models.allele import Allele from mavedb.models.enums.annotation_layer import AnnotationLayer from mavedb.models.enums.job_pipeline import JobStatus +from mavedb.models.enums.target_category import TargetCategory from mavedb.models.job_run import JobRun from mavedb.models.mapping_record import MappingRecord from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.target_gene_mapping import TargetGeneMapping from mavedb.models.variant import Variant from mavedb.models.variant_annotation_status import VariantAnnotationStatus from mavedb.worker.jobs.variant_processing.mapping import map_variants_for_score_set @@ -771,6 +774,253 @@ async def test_no_coding_transcript_is_skipped_not_failed( assert _cross_level_statuses(session, sample_score_set.id, status="failed") == [] assert len(_cross_level_statuses(session, sample_score_set.id, status="skipped")) == 1 + async def test_genomic_accession_coding_target_is_reverse_translated( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """A genomic-accession (NC_:g.) coding variant -- previously skipped for want of a + coding transcript -- is reverse-translated once the mapper emits a cdna + TargetGeneMapping, whose reference_accession anchors the projection.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NC_000001.11:g.1000A>G", + data={}, + ) + session.add(variant) + session.commit() + # Genomic assay layer + cdna identity TargetGeneMapping (carrying the NM_); the + # genomic variant projects onto that transcript for reverse translation. + await _map_variants( + session, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_score_set, + with_layers={"g", "c"}, + ) + + assay_hgvs = "NC_000001.11:g.1000A>G" + g_candidate = "NC_000001.11:g.1000A>G" + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", fake_construct({assay_hgvs: ([], [g_candidate])})), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", fake_translate({g_candidate: "ga4gh:VA.genomic"})), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 1} + assert _cross_level_statuses(session, sample_score_set.id, status="skipped") == [] + + def _run_mapped_date(self, session, target_gene_id): + """The mapped_date the (mock) mapping run stamped on its TargetGeneMappings.""" + return ( + session.query(TargetGeneMapping) + .filter( + TargetGeneMapping.target_gene_id == target_gene_id, + TargetGeneMapping.alignment_level == AnnotationLayer.genomic, + ) + .first() + .mapped_date + ) + + async def test_uses_latest_cdna_row_within_the_run( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """When a run has more than one cdna TargetGeneMapping for a target (same run date), + RT reverse-translates against the newest (highest id), not an arbitrary one.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NC_000001.11:g.1000A>G", + data={}, + ) + session.add(variant) + session.commit() + # Mapping emits a cdna TargetGeneMapping (NM_999999.1) stamped with the run date. + await _map_variants( + session, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_score_set, + with_layers={"g", "c"}, + ) + target_gene = sample_score_set.target_genes[0] + run_date = self._run_mapped_date(session, target_gene.id) + # A newer cdna row (higher id) for the same target and same run date. + session.add( + TargetGeneMapping( + target_gene_id=target_gene.id, + alignment_level=AnnotationLayer.cdna, + reference_accession="NM_111111.1", + preferred=False, + tool_version="pytest.0.0", + mapped_date=run_date, + ) + ) + session.commit() + + assay_hgvs = "NC_000001.11:g.1000A>G" + g_candidate = "NC_000001.11:g.1000A>G" + delegate = fake_construct({assay_hgvs: ([], [g_candidate])}) + captured: dict = {} + + def capturing_construct(inputs, *, transcripts, coordinates, config): + captured["transcripts"] = [inp.transcript for inp in inputs] + return delegate(inputs, transcripts=transcripts, coordinates=coordinates, config=config) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", capturing_construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", fake_translate({g_candidate: "ga4gh:VA.g"})), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + # The newest cdna row within the run wins, not the first-mapped one. + assert captured["transcripts"] == ["NM_111111.1"] + + async def test_ignores_stale_cdna_row_from_a_different_run( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """A cdna row left behind by a *different* run (different run date) is not used: + the current run emitted no cdna row, so the variant is skipped (transcript + unresolved) rather than reverse-translated against the stale transcript -- the + mapped_date anchor narrows the accumulation edge case (mavedb-api#763).""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NC_000001.11:g.1000A>G", + data={}, + ) + session.add(variant) + session.commit() + # Current run maps genomic only -- no cdna row for this run's date. + await _map_variants( + session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set, with_layers={"g"} + ) + target_gene = sample_score_set.target_genes[0] + run_date = self._run_mapped_date(session, target_gene.id) + # A leftover cdna row from a *prior* run (an earlier date) must not be picked up. + session.add( + TargetGeneMapping( + target_gene_id=target_gene.id, + alignment_level=AnnotationLayer.cdna, + reference_accession="NM_888888.1", + preferred=False, + tool_version="pytest.0.0", + mapped_date=run_date - timedelta(days=1), + ) + ) + session.commit() + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", fake_construct({})), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", fake_translate({})), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.data == {"translated": 0, "failed": 0, "skipped": 1, "alleles_created": 0} + skipped = _cross_level_statuses(session, sample_score_set.id, status="skipped") + assert len(skipped) == 1 + assert skipped[0].annotation_metadata["skip_category"] == "transcript_unresolved" + + async def test_coding_target_skip_is_classified_recoverable( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """A protein-coding target with no resolvable coding transcript is a *recoverable* + skip -- classified ``transcript_unresolved`` so it is findable, distinct from a + regulatory target's correct skip.""" + assert sample_score_set.target_genes[0].category == TargetCategory.protein_coding + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NC_000001.11:g.1000A>G", + data={}, + ) + session.add(variant) + session.commit() + # Genomic-only mapping: no cdna TargetGeneMapping, so no coding transcript resolves. + await _map_variants( + session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set, with_layers={"g"} + ) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", fake_construct({})), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", fake_translate({})), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.data == {"translated": 0, "failed": 0, "skipped": 1, "alleles_created": 0} + skipped = _cross_level_statuses(session, sample_score_set.id, status="skipped") + assert len(skipped) == 1 + assert skipped[0].annotation_metadata["skip_category"] == "transcript_unresolved" + + async def test_regulatory_target_skip_is_classified_correct( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """A non-coding/regulatory target has no protein consequence: its skip is *correct*, + classified ``no_coding_transcript`` rather than the recoverable category.""" + target = sample_score_set.target_genes[0] + target.category = TargetCategory.regulatory + session.add(target) + session.commit() + + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NC_000001.11:g.1000A>G", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants( + session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set, with_layers={"g"} + ) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", fake_construct({})), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", fake_translate({})), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.data == {"translated": 0, "failed": 0, "skipped": 1, "alleles_created": 0} + skipped = _cross_level_statuses(session, sample_score_set.id, status="skipped") + assert len(skipped) == 1 + assert skipped[0].annotation_metadata["skip_category"] == "no_coding_transcript" + async def test_translation_config_param_overrides_job_defaults( self, session, From 181360f1f35a813f6ea08d26668a05b9354cbe06 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 9 Jun 2026 11:53:58 -0700 Subject: [PATCH 020/252] feat(hgvs): add strip_protein_prediction_parens utility Forward translation emits predicted protein consequences in parentheses (e.g. p.(Ala222Val)). The parens denote inference, not a distinct variant form, so normalize to the bare p. form before storing. Strings without prediction parens are returned unchanged. Includes parametrized unit tests. --- src/mavedb/lib/hgvs.py | 14 ++++++++++++++ tests/lib/test_hgvs.py | 22 +++++++++++++++++++++- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/src/mavedb/lib/hgvs.py b/src/mavedb/lib/hgvs.py index f330bdfd4..5826e67c3 100644 --- a/src/mavedb/lib/hgvs.py +++ b/src/mavedb/lib/hgvs.py @@ -22,6 +22,20 @@ def extract_accession(hgvs_string: str) -> str: return token.split(":", 1)[0].strip() +_HGVS_P_PREDICTION = re.compile(r":p\.\((.+)\)\s*$") + + +def strip_protein_prediction_parens(hgvs_p: str) -> str: + """Unwrap the prediction parentheses from a protein HGVS: ``p.(Ala222Val)`` -> ``p.Ala222Val``. + + Forward translation (``c_to_p``) emits a *predicted* consequence in parentheses. ga4gh + translates either form fine, but the parens are noise we don't want in the stored HGVS -- + they denote inference, not a different variant -- so we normalize to the bare form for + consistency. A string with no prediction parens is returned unchanged. + """ + return _HGVS_P_PREDICTION.sub(r":p.\1", hgvs_p) + + def split_cis_phased_hgvs(hgvs_string: str) -> list[str]: """Split a cis-phased multivariant HGVS expression into fully-qualified component strings. diff --git a/tests/lib/test_hgvs.py b/tests/lib/test_hgvs.py index 355174fb1..45793034a 100644 --- a/tests/lib/test_hgvs.py +++ b/tests/lib/test_hgvs.py @@ -1,6 +1,26 @@ import pytest -from mavedb.lib.hgvs import extract_accession, join_cis_phased_hgvs, split_cis_phased_hgvs +from mavedb.lib.hgvs import ( + extract_accession, + join_cis_phased_hgvs, + split_cis_phased_hgvs, + strip_protein_prediction_parens, +) + + +@pytest.mark.parametrize( + ("hgvs", "expected"), + [ + ("NP_000456.2:p.(Ala222Val)", "NP_000456.2:p.Ala222Val"), + ("NP_000456.2:p.(Tyr745Ter)", "NP_000456.2:p.Tyr745Ter"), + ("NP_000456.2:p.(Ala222_Val225del)", "NP_000456.2:p.Ala222_Val225del"), + # no prediction parens -> unchanged + ("NP_000456.2:p.Ala222Val", "NP_000456.2:p.Ala222Val"), + ("NM_000001.1:c.5A>G", "NM_000001.1:c.5A>G"), + ], +) +def test_strip_protein_prediction_parens(hgvs, expected): + assert strip_protein_prediction_parens(hgvs) == expected @pytest.mark.parametrize( From bd6f6a5e0e7a24f0cb4279edbe4a0e3fbe3d639b Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 9 Jun 2026 11:54:59 -0700 Subject: [PATCH 021/252] fix(vrs-utils): normalize and re-identify VRS alleles to prevent digest collisions - Re-identify each component via normalize_and_identify after translate_from so the stored vrs_digest reflects current content, not a stale value cached by the reused AlleleTranslator's Merkle tree. Without this, distinct biological variants at the same position (e.g. A>C and A>T) share one digest and are merged by the digest-keyed get_or_create_allele. - Use ga4gh_identify with in_place="always" in identify_allele and identify_variation so an allele that already carries a translator-stamped id has it recomputed, not retained. - Coerce ReferenceLengthExpression -> LiteralSequenceExpression in normalize_and_identify, mirroring dcd_mapping's _rle_to_lse, so RT-built alleles hash identically to the mapper's authoritative alleles and dedup correctly across sources. - Add regression tests: stale-id overwrite, distinct-alt digest isolation, cis-phased ordering canonicalization, and RLE->LSE coercion. --- src/mavedb/lib/vrs_utils.py | 61 ++++++++++++++++-- tests/lib/test_vrs_utils.py | 120 ++++++++++++++++++++++++++++++++++-- 2 files changed, 170 insertions(+), 11 deletions(-) diff --git a/src/mavedb/lib/vrs_utils.py b/src/mavedb/lib/vrs_utils.py index 7872f2ed6..052c4b702 100644 --- a/src/mavedb/lib/vrs_utils.py +++ b/src/mavedb/lib/vrs_utils.py @@ -9,11 +9,18 @@ current content. """ +from itertools import cycle from typing import Any from ga4gh.core import ga4gh_identify from ga4gh.vrs.extras.translator import AlleleTranslator -from ga4gh.vrs.models import Allele, CisPhasedBlock, LiteralSequenceExpression, SequenceLocation +from ga4gh.vrs.models import ( + Allele, + CisPhasedBlock, + LiteralSequenceExpression, + ReferenceLengthExpression, + SequenceLocation, +) from ga4gh.vrs.normalize import normalize from mavedb.lib.hgvs import split_cis_phased_hgvs @@ -62,11 +69,22 @@ def translate_hgvs_to_variation(hgvs: str, translator: AlleleTranslator) -> Alle identifies to one ``ga4gh:CPB.`` digest and dedups to a single row regardless of component ordering. + Every component is normalized and re-identified through :func:`normalize_and_identify` + before use. ``translate_from`` is called with ``do_normalize=False`` and stamps ``id`` via + plain ``ga4gh_identify`` on the reused translator, so without this step a component carries a + non-canonical digest computed from the unnormalized object and from the Merkle tree's cached + sub-object digests — distinct biological variants can then collide onto one ``vrs_digest`` and + be merged by the digest-keyed ``get_or_create_allele``. Normalizing here also keeps RT digests + consistent with the mapper's, which is what lets the same allele dedup across sources. + :param hgvs: a single- or cis-phased-multivariant HGVS string :param translator: caller-owned AlleleTranslator reused across calls :return: an Allele for a single variant, or a CisPhasedBlock for a cis-phased set """ - members = [translate_hgvs_to_vrs(component, translator) for component in split_cis_phased_hgvs(hgvs)] + members = [ + normalize_and_identify(translate_hgvs_to_vrs(component, translator), translator.data_proxy) + for component in split_cis_phased_hgvs(hgvs) + ] if len(members) == 1: return members[0] @@ -83,12 +101,18 @@ def identify_allele(allele: Allele) -> str: the location digest and the allele digest first ensures the id is always derived from the current object content — not from a value set before a refgetAccession mutation or normalization. + + ``id`` is recomputed as well: ``ga4gh_identify`` returns a non-empty ``id`` as-is + (its default ``in_place`` only fills an *empty* id), so an allele that already + carries one — e.g. stamped by an ``identify=True`` ``AlleleTranslator`` — would + otherwise keep its stale id even with the digests cleared. Use in_place="always" + to force it to be recomputed from the content-derived digest. """ if isinstance(allele.location, SequenceLocation): allele.location.digest = None allele.digest = None - digest = ga4gh_identify(allele) + digest = ga4gh_identify(allele, in_place="always") if digest is None: raise ValueError("Failed to compute GA4GH identifier for allele") # noqa: EM101 @@ -113,7 +137,7 @@ def identify_variation(variation: Allele | CisPhasedBlock) -> str: member.digest = None variation.digest = None - digest = ga4gh_identify(variation) + digest = ga4gh_identify(variation, in_place="always") if digest is None: raise ValueError("Failed to compute GA4GH identifier for variation") # noqa: EM101 @@ -123,13 +147,40 @@ def identify_variation(variation: Allele | CisPhasedBlock) -> str: def normalize_and_identify(allele: Allele, data_proxy: Any) -> Allele: """Normalize *allele* and stamp it with a freshly computed GA4GH digest. - Pairs the two finalize steps every VRS allele construction path needs. + Pairs the finalize steps every VRS allele construction path needs. Routing identification through :func:`identify_allele` (rather than ``ga4gh_identify`` directly) is the invariant that protects against the Merkle-tree's stale-digest behavior after mutation -- so any allele construction site that bypasses this helper risks reintroducing the stale-digest bug. + + Normalization can leave an indel as a ``ReferenceLengthExpression``; this coerces + it to a ``LiteralSequenceExpression`` so the result matches dcd_mapping's + authoritative alleles (``vrs_map._rle_to_lse``), which always store LSE. Without + the coercion the same biological indel hashes to two digests -- RLE here, LSE from + the mapper -- so reverse translation's regenerated genomic form fails to dedup + against the authoritative row and a duplicate allele is linked. """ allele = normalize(allele, data_proxy=data_proxy) + if isinstance(allele.state, ReferenceLengthExpression): + allele.state = _rle_to_lse(allele.state, allele.location, data_proxy) allele.id = identify_allele(allele) return allele + + +def _rle_to_lse( + rle: ReferenceLengthExpression, location: SequenceLocation, data_proxy: Any +) -> LiteralSequenceExpression: + """Coerce a ReferenceLengthExpression to an equivalent LiteralSequenceExpression. + + Mirrors ``dcd_mapping:vrs_map.py::_rle_to_lse`` byte-for-byte so an allele built here + hashes identically to the mapper's authoritative allele for the same variant. Derives + the literal sequence by tiling the repeat subunit out to ``rle.length``. + """ + sequence_id = location.sequenceReference.refgetAccession + start: int = location.start + end = start + rle.repeatSubunitLength + subsequence = data_proxy.get_sequence(f"ga4gh:{sequence_id}", start, end) + c = cycle(subsequence) + derived_sequence = "".join(next(c) for _ in range(rle.length)) + return LiteralSequenceExpression(sequence=derived_sequence) diff --git a/tests/lib/test_vrs_utils.py b/tests/lib/test_vrs_utils.py index 6f2bd382f..f40dce976 100644 --- a/tests/lib/test_vrs_utils.py +++ b/tests/lib/test_vrs_utils.py @@ -1,5 +1,7 @@ # ruff: noqa: E402 +from types import SimpleNamespace + import pytest pytest.importorskip("ga4gh.vrs") @@ -8,15 +10,25 @@ Allele, CisPhasedBlock, LiteralSequenceExpression, + ReferenceLengthExpression, SequenceLocation, SequenceReference, ) from mavedb.lib import vrs_utils -from mavedb.lib.vrs_utils import identify_variation, translate_hgvs_to_variation +from mavedb.lib.vrs_utils import ( + _rle_to_lse, + identify_variation, + normalize_and_identify, + translate_hgvs_to_variation, +) _SQ = "SQ." + "a" * 32 +# A digest a reused ga4gh AlleleTranslator might leave on a component before it is +# re-identified — stale from the Merkle cache, or carried over from a sibling variant. +_STALE_ID = "ga4gh:VA." + "Z" * 32 + def _allele(start: int, alt: str) -> Allele: return Allele( @@ -38,14 +50,72 @@ def _fake(hgvs: str, translator=None) -> Allele: monkeypatch.setattr(vrs_utils, "translate_hgvs_to_vrs", _fake) +def _stub_normalize(monkeypatch) -> None: + """Make normalization a no-op so identification can be exercised without a seqrepo. + + Identification (the regression surface) runs for real; only the proxy-backed + normalize step is stubbed out. + """ + monkeypatch.setattr(vrs_utils, "normalize", lambda allele, data_proxy: allele) + + +def _translator() -> SimpleNamespace: + return SimpleNamespace(data_proxy=object()) + + def test_single_variant_returns_a_bare_allele(monkeypatch): allele = _allele(1000, "G") _patch_component_translator(monkeypatch, {"NC_000001.11:g.1000A>G": allele}) + _stub_normalize(monkeypatch) - result = translate_hgvs_to_variation("NC_000001.11:g.1000A>G", translator=None) + result = translate_hgvs_to_variation("NC_000001.11:g.1000A>G", translator=_translator()) assert isinstance(result, Allele) - assert result is allele + assert not isinstance(result, CisPhasedBlock) + + +def test_single_variant_is_reidentified_not_left_with_translator_digest(monkeypatch): + # ga4gh's translate_from (do_normalize=False) stamps id via plain ga4gh_identify on + # the reused translator; translate_hgvs_to_variation must re-identify so the persisted + # vrs_digest reflects current content rather than that carried-over value. + allele = _allele(1000, "G") + allele.id = _STALE_ID + _patch_component_translator(monkeypatch, {"NC_000001.11:g.1000A>G": allele}) + _stub_normalize(monkeypatch) + + result = translate_hgvs_to_variation("NC_000001.11:g.1000A>G", translator=_translator()) + + assert result.id is not None + assert result.id != _STALE_ID + assert result.id.startswith("ga4gh:VA.") + + +def test_same_position_distinct_alts_get_distinct_digests(monkeypatch): + # Regression: NC_000002.12:g.214809459A>C and g.214809459A>T are different MaveDB + # variants that cannot encode the same codon, yet were merged onto one Allele row + # (and thus one coding equivalent like NM_000465.4:c.109_111delinsCGA). The single + # component returned by the reused translator carried a stale/shared digest, which + # the digest-keyed get_or_create_allele then deduplicated. Re-identification must + # give distinct content distinct digests so they never collide. + a_c = _allele(214809458, "C") + a_t = _allele(214809458, "T") + a_c.id = a_t.id = _STALE_ID # what the unfixed path would persist for both + _patch_component_translator( + monkeypatch, + { + "NC_000002.12:g.214809459A>C": a_c, + "NC_000002.12:g.214809459A>T": a_t, + }, + ) + _stub_normalize(monkeypatch) + translator = _translator() + + res_c = translate_hgvs_to_variation("NC_000002.12:g.214809459A>C", translator=translator) + res_t = translate_hgvs_to_variation("NC_000002.12:g.214809459A>T", translator=translator) + + assert res_c.id != _STALE_ID + assert res_t.id != _STALE_ID + assert res_c.id != res_t.id def test_cis_phased_multivariant_returns_an_identified_block(monkeypatch): @@ -54,8 +124,9 @@ def test_cis_phased_multivariant_returns_an_identified_block(monkeypatch): "NC_000001.11:g.1002T>C": _allele(1002, "C"), } _patch_component_translator(monkeypatch, members) + _stub_normalize(monkeypatch) - result = translate_hgvs_to_variation("NC_000001.11:g.[1000A>G;1002T>C]", translator=None) + result = translate_hgvs_to_variation("NC_000001.11:g.[1000A>G;1002T>C]", translator=_translator()) assert isinstance(result, CisPhasedBlock) assert len(result.members) == 2 @@ -68,9 +139,10 @@ def test_cis_phased_block_digest_is_order_independent(monkeypatch): "NC_000001.11:g.1002T>C": _allele(1002, "C"), } _patch_component_translator(monkeypatch, members) + _stub_normalize(monkeypatch) - forward = translate_hgvs_to_variation("NC_000001.11:g.[1000A>G;1002T>C]", translator=None) - reverse = translate_hgvs_to_variation("NC_000001.11:g.[1002T>C;1000A>G]", translator=None) + forward = translate_hgvs_to_variation("NC_000001.11:g.[1000A>G;1002T>C]", translator=_translator()) + reverse = translate_hgvs_to_variation("NC_000001.11:g.[1002T>C;1000A>G]", translator=_translator()) # The same biological cis-phased set dedups to one row regardless of component ordering. assert forward.id == reverse.id @@ -84,3 +156,39 @@ def test_identify_variation_clears_stale_block_digest(): assert digest != "STALE" assert digest.startswith("ga4gh:CPB.") # recomputed from content, not the stale cache + + +def _rle_allele(start: int, *, length: int, repeat_subunit_length: int) -> Allele: + return Allele( + location=SequenceLocation( + sequenceReference=SequenceReference(refgetAccession=_SQ), + start=start, + end=start + repeat_subunit_length, + ), + state=ReferenceLengthExpression(length=length, repeatSubunitLength=repeat_subunit_length), + ) + + +def test_rle_to_lse_tiles_repeat_subunit(): + # repeatSubunitLength=2 reads 2 bases from the proxy; length=4 tiles them out to "ACAC". + location = SequenceLocation(sequenceReference=SequenceReference(refgetAccession=_SQ), start=10, end=12) + rle = ReferenceLengthExpression(length=4, repeatSubunitLength=2) + data_proxy = SimpleNamespace(get_sequence=lambda identifier, start, end: "AC") + + result = _rle_to_lse(rle, location, data_proxy) + + assert isinstance(result, LiteralSequenceExpression) + assert result.sequence.root == "ACAC" + + +def test_normalize_and_identify_coerces_rle_to_lse(monkeypatch): + # Regression: normalization can yield an RLE for an indel, but dcd_mapping stores LSE; the + # finalize step must coerce so the same variant doesn't hash to two digests (and duplicate). + rle = _rle_allele(10, length=2, repeat_subunit_length=2) + monkeypatch.setattr(vrs_utils, "normalize", lambda allele, data_proxy: rle) + data_proxy = SimpleNamespace(get_sequence=lambda identifier, start, end: "AC") + + result = normalize_and_identify(rle, data_proxy=data_proxy) + + assert isinstance(result.state, LiteralSequenceExpression) + assert result.id is not None and result.id.startswith("ga4gh:VA.") From c17caf71499d0f522afcb314cb0805529cfbd23b Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 9 Jun 2026 13:23:36 -0700 Subject: [PATCH 022/252] refactor(reverse-translation): add p. alleles to equivalence set and tighten comments - Emit the protein consequence (result.hgvs_p) as a protein-level member of the equivalence set alongside the coding/genomic candidates. Prediction parens (p.(Ala222Val)) are stripped via strip_protein_prediction_parens before translation and storage. Protein-assay inputs are excluded (the protein is already the authoritative allele). - Trim all inline and docstring comments to essential why; remove redundant prose throughout reverse_translation.py. - Add tests: protein allele persistence, skip-classification for all three skip categories (no_assay_level_hgvs, transcript_unresolved, no_coding_transcript). --- .../variant_processing/reverse_translation.py | 147 +++++++----------- .../test_reverse_translation.py | 56 ++++++- 2 files changed, 109 insertions(+), 94 deletions(-) diff --git a/src/mavedb/worker/jobs/variant_processing/reverse_translation.py b/src/mavedb/worker/jobs/variant_processing/reverse_translation.py index c5f14bf68..872044e91 100644 --- a/src/mavedb/worker/jobs/variant_processing/reverse_translation.py +++ b/src/mavedb/worker/jobs/variant_processing/reverse_translation.py @@ -2,7 +2,8 @@ For each mapped variant in a score set, calls construct_equivalent_variants from the variant-annotation library to produce all coding and genomic HGVS candidates -encoding the same protein consequence. The candidates are written as non-authoritative +encoding the same protein consequence, plus that protein consequence itself. The +candidates (coding, genomic, and the protein) are written as non-authoritative Allele rows linked to the existing MappingRecord via MappingRecordAllele. """ @@ -24,7 +25,7 @@ from variant_annotation.lib.translation.types import TranslationConfig, VariantInput, WtCodonMode from mavedb.lib.annotation_status_manager import AnnotationStatusManager -from mavedb.lib.hgvs import extract_accession +from mavedb.lib.hgvs import extract_accession, strip_protein_prediction_parens from mavedb.lib.types.workflow import JobExecutionOutcome from mavedb.lib.variant_translations import get_or_create_allele from mavedb.lib.vrs_utils import translate_hgvs_to_variation @@ -46,9 +47,8 @@ logger = logging.getLogger(__name__) -# Job defaults when no `translation_config` override is supplied: enumerate the full coding -# equivalence class — every synonymous codon, indels included. wt_codon_mode "all" requires -# include_indels=True (TranslationConfig enforces this). +# Job defaults: full coding equivalence class — every synonymous codon, indels included. +# wt_codon_mode "all" requires include_indels=True. _DEFAULT_TRANSLATION_CONFIG: dict[str, Any] = { "include_indels": True, "wt_codon_mode": WtCodonMode.ALL, @@ -56,10 +56,9 @@ class _TranscriptResolution(NamedTuple): - """A mapping record paired with what we know about its coding transcript before the - batched UTA lookup: the gene-level cdna transcript if the mapper supplied one, and the - protein accession still awaiting NP_→NM_ resolution otherwise. Exactly one of the two is - set for a resolvable record; both are None when neither path applies (record is skipped).""" + """Pairs a mapping record with its pre-UTA transcript info: gene_transcript if the mapper + supplied a cdna row, protein_accession if NP_→NM_ resolution is still needed. Both None + means the record will be skipped.""" rec: MappingRecord variant: Variant @@ -77,11 +76,10 @@ class _TranscriptResolutionSkipReason(Enum): def classify( cls, resolution: _TranscriptResolution, category: TargetCategory | None ) -> tuple["_TranscriptResolutionSkipReason", str]: - """Classify why a record was skipped, distinguishing recoverable from correct skips. + """Classify why a record was skipped. - A protein-coding target with no resolvable transcript is *recoverable* (it should now be - rare -- the mapper selects a MANE transcript for coding genomic-accession targets); a - regulatory / non-coding target has no protein consequence and is a *correct* skip. + Protein-coding target with no transcript → recoverable (transcript_unresolved). + Non-coding/regulatory target → correct skip (no_coding_transcript). """ if not resolution.rec.hgvs_assay_level: return ( @@ -102,14 +100,11 @@ def classify( def _coding_transcripts_for_proteins(protein_accessions: set[str]) -> dict[str, str]: - """Resolve each RefSeq protein accession (NP_/XP_…) to its preferred coding transcript - via UTA's associated_accessions table. - - Protein-level mappings carry no coding transcript — the mapper emits reference_accession - only for cdna alignments — so reverse-translating them relies on the NP_→NM_ association - UTA records. Resolution and the multi-transcript preference order both live in the - variant-annotation library's UtaClient; we own only the connection lifecycle here so the - long-lived worker process does not leak UTA connections across jobs. + """Resolve RefSeq protein accessions (NP_/XP_) to their preferred coding transcripts via UTA. + + Protein-level mappings carry no cdna transcript in TargetGeneMapping, so reverse + translation falls back to the NP_→NM_ association in UTA. Connection is opened and + closed per-call to avoid leaking connections across long-lived worker jobs. """ if not protein_accessions: return {} @@ -128,17 +123,11 @@ def _coding_transcripts_for_proteins(protein_accessions: set[str]) -> dict[str, def _build_translation_config(overrides: dict[str, Any] | None) -> TranslationConfig: - """Build the variant-annotation TranslationConfig from optional job-param overrides. - - Each key in `overrides` is a TranslationConfig field (that dataclass is the source of truth - for the available knobs) and wins over the job defaults in _DEFAULT_TRANSLATION_CONFIG; - absent or None means use the defaults. wt_codon_mode is coerced to the enum so a JSON string - value ("all"/"unambiguous"/"none") works. + """Build a TranslationConfig from optional job-param overrides merged with job defaults. - Raises ValueError with an actionable message — listing the offending value and the valid - options — for an unknown field, an invalid wt_codon_mode, or an invalid combination (e.g. a - wt_codon_mode other than "none" without include_indels). The allowed-field set is derived - from TranslationConfig itself, so it never drifts from the library. + Overrides win over _DEFAULT_TRANSLATION_CONFIG; wt_codon_mode accepts string values and + is coerced to the enum. Raises ValueError for unknown fields, invalid modes, or invalid + combinations (e.g. wt_codon_mode != "none" without include_indels). """ config_kwargs: dict[str, Any] = {**_DEFAULT_TRANSLATION_CONFIG, **(overrides or {})} @@ -172,20 +161,12 @@ async def reverse_translate_variants_for_score_set( ) -> JobExecutionOutcome: """Build the cross-level HGVS equivalence class for every mapped variant in the score set. - Reads current MappingRecords that carry an hgvs_assay_level string, collapses each - to its ProteinConsequence, and expands to all coding/genomic HGVS candidates via a - single batched subprocess call to the variant-annotation library. Each candidate is - written as a non-authoritative Allele linked to the MappingRecord. - - Required job_params: - - score_set_id (int): ID of the ScoreSet to process - - correlation_id (str): Correlation ID for tracking + For each current MappingRecord with an hgvs_assay_level, collapses to a ProteinConsequence + and expands to all coding/genomic HGVS candidates via the variant-annotation library. + Each candidate is written as a non-authoritative Allele linked to the MappingRecord. - Optional job_params: - - translation_config (dict): Overrides for any variant-annotation TranslationConfig - field (e.g. include_indels, wt_codon_mode, max_indel_size). Omitted keys fall back to - the job defaults in _DEFAULT_TRANSLATION_CONFIG (full codon equivalence class, indels - included). + job_params: score_set_id (int), correlation_id (str), + translation_config (dict, optional) — TranslationConfig overrides. """ job = job_manager.get_job() validate_job_params(["score_set_id", "correlation_id"], job) @@ -206,25 +187,14 @@ async def reverse_translate_variants_for_score_set( job_manager.update_progress(0, 100, "Starting reverse translation job.") logger.info(msg="Started reverse translation job.", extra=job_manager.logging_context()) - # Build {(target_gene_id, run date) -> NM_ transcript} from the cdna TargetGeneMappings - # the mapper emits. Reverse translation must run against the cdna (NM_) transcript. + # Build {(target_gene_id, run date) -> NM_ transcript} from cdna TargetGeneMappings. + # TargetGeneMapping rows accumulate across re-maps (retired records still FK them), so we + # key by (target_gene_id, mapped_date) to bind each record to its own run's cdna row. + # All TargetGeneMappings in one job share a mapped_date; within a key, highest id wins. # - # TargetGeneMapping is not valid-time-versioned and re-mapping cannot delete prior rows - # (retired MappingRecords still FK them), so a re-mapped target accumulates several cdna - # rows. There is no run anchor on the row, so we approximate one with mapped_date and key - # the lookup by (target gene, run date): each record resolves only the cdna transcript - # from *its own* run, matched below against the run date of the genomic TargetGeneMapping - # the record points to. All of a run's TargetGeneMappings share one mapped_date (stamped - # once per job), so a record and its run's cdna row match even if the job spans midnight. - # Within a single key, order by id so the newest row wins. - # - # TODO#763: mapped_date is day-granular, so two re-maps on the *same calendar day* collide - # on the key. If a later same-day re-map emits no cdna row (e.g. transcript selection hit - # a transient Ensembl/gene-normalizer failure, or the target genuinely lost its coding - # transcript) but an earlier same-day one did, RT can still bind the earlier (stale) - # transcript. This keying narrows the window from "any prior run ever" to "a same-day - # re-map" -- the bug now requires re-runs in quick succession -- but only an actual run - # anchor / versioned home for reference identity closes it fully. + # TODO#763: mapped_date is day-granular, so two re-maps on the same calendar day collide. + # A later same-day run that emits no cdna row can still bind the earlier run's stale + # transcript. Only a versioned run anchor closes this fully. cdna_transcript_by_run: dict[tuple[int, date | None], str | None] = { (target_gene_id, mapped_date): reference_accession for target_gene_id, mapped_date, reference_accession in job_manager.db.execute( @@ -243,10 +213,8 @@ async def reverse_translate_variants_for_score_set( .all() } - # Load current, authoritative, and successfully-mapped MappingRecords along with their - # target gene (for the coding-transcript lookup) and parent Variant. - # The joined TargetGeneMapping is the record's own (genomic) mapping row, so its - # mapped_date is this record's run date -- used to anchor the cdna lookup to the run. + # Load current authoritative MappingRecords with their Variant and TargetGeneMapping. + # mapped_date from the joined (genomic) TargetGeneMapping anchors the cdna transcript lookup. rows: Sequence[tuple[MappingRecord, Variant, int, date | None]] = ( job_manager.db.execute( select( @@ -275,10 +243,8 @@ async def reverse_translate_variants_for_score_set( job_manager.db.flush() return JobExecutionOutcome.succeeded(data={"translated": 0, "failed": 0, "skipped": 0, "alleles_created": 0}) - # Resolve each record's coding transcript. Genomic/cdna mappings share their target - # gene's cdna alignment reference; protein mappings have none (the mapper emits - # reference_accession only for cdna alignments), so gather their protein accessions and - # resolve NP_→NM_ from UTA in a single batch query below. + # Genomic/cdna records resolve via the cdna TargetGeneMapping; protein records have no + # cdna reference_accession, so collect their NP_ accessions for a batched NP_→NM_ UTA lookup. transcript_resolutions: list[_TranscriptResolution] = [] protein_accessions: set[str] = set() for rec, variant, target_gene_id, mapped_date in rows: @@ -296,9 +262,8 @@ async def reverse_translate_variants_for_score_set( transcript_by_protein = _coding_transcripts_for_proteins(protein_accessions) - # Target gene category per gene, so a skip can be classified honestly: a coding target - # with no resolvable transcript is recoverable (should now be rare), while a regulatory / - # non-coding target has no protein consequence and is a correct skip. + # Target category per gene — used to classify skips as recoverable (coding target, + # transcript unresolved) vs. correct (non-coding/regulatory, no protein consequence). target_category_by_gene: dict[int, TargetCategory] = dict( job_manager.db.execute( select(TargetGene.id, TargetGene.category).where(TargetGene.score_set_id == score_set_id) @@ -307,14 +272,9 @@ async def reverse_translate_variants_for_score_set( .all() ) - # Build VariantInputs, supplying the resolved coding transcript for every input - # regardless of its own assay level (p./c./g. all collapse to a ProteinConsequence - # anchored on that transcript). Object identity on each VariantInput is preserved - # through the call so we can correlate TranslationResult.input back to its MappingRecord. - # - # A record with no coding transcript (e.g. a regulatory element aligned only at the - # genomic level, or a protein with no UTA association) has no protein consequence to - # reverse-translate; it is skipped and recorded as SKIPPED rather than counted as a failure. + # Build VariantInputs with the resolved coding transcript (p./c./g. all collapse to a + # ProteinConsequence on that transcript). Object identity is preserved so TranslationResult + # can be correlated back to its MappingRecord. Records with no transcript are skipped. variant_inputs: list[Any] = [] variant_input_map: dict[int, tuple[MappingRecord, Variant]] = {} skipped_variants: list[_TranscriptResolution] = [] @@ -378,9 +338,8 @@ async def reverse_translate_variants_for_score_set( .where(MappingRecord.current.is_(True)) ) - # Collect new MappingRecord -> Allele links and defer linkage until all candidates are processed. - # This allows us to supersede the prior live derived links with the new set in one atomic operation at the end, - # so the retire and insert share a timestamp and there is no gap between the old and new sets of live links. + # Defer linkage until all candidates are processed so the prior derived links can be + # superseded atomically — retire and insert share a timestamp with no gap. new_links: list[MappingRecordAllele] = [] # The live authoritative (record, allele) pairs. A derived candidate that equals a record's @@ -410,12 +369,15 @@ async def reverse_translate_variants_for_score_set( (hgvs_g, AnnotationLayer.genomic, "hgvs_g") for hgvs_g in result.hgvs_g_candidates ] + [(hgvs_c, AnnotationLayer.cdna, "hgvs_c") for hgvs_c in result.hgvs_c_candidates] + # Emit the protein consequence as the protein-level member of the equivalence set. + # Prediction parens (p.(Ala222Val)) are stripped before translation and storage. + # None for protein-assay inputs where the protein is already the authoritative allele. + if result.hgvs_p: + candidates.append((strip_protein_prediction_parens(result.hgvs_p), AnnotationLayer.protein, "hgvs_p")) + for hgvs, level, hgvs_field in candidates: - # A candidate the equivalence class produced may be a form ga4gh cannot - # translate (an intronic projection, an unsupported edge case, a malformed - # bracketed expression). A variant with at least one translatable candidate still - # advances the pipeline; only a variant where every candidate fails is - # recorded as failed (with the per-candidate errors retained as metadata). + # A candidate may not be translatable (intronic projection, malformed expression). + # Track per-candidate failures; a variant fails only if *all* candidates fail. try: variation = translate_hgvs_to_variation(hgvs, allele_translator) except Exception as e: @@ -483,10 +445,9 @@ async def reverse_translate_variants_for_score_set( annotation_data={"annotation_metadata": annotation_metadata}, ) - # Supersede the prior live derived links with the new set in one gap-free operation. - # TODO#765: a re-run supersedes the whole derived set wholesale because re-mapping re-mints the - # records, so we cannot tell which derivations are unchanged; idempotent mapping records would let - # unchanged derived links stay live instead of being retired and recreated. + # Supersede prior live derived links atomically. + # TODO#765: re-runs retire and recreate the whole derived set because re-mapping re-mints + # records; idempotent records would allow unchanged links to stay live. MappingRecordAllele.supersede_live_where( job_manager.db, new_links, diff --git a/tests/worker/jobs/variant_processing/test_reverse_translation.py b/tests/worker/jobs/variant_processing/test_reverse_translation.py index 459ec53db..ef8104b94 100644 --- a/tests/worker/jobs/variant_processing/test_reverse_translation.py +++ b/tests/worker/jobs/variant_processing/test_reverse_translation.py @@ -69,12 +69,15 @@ def _construct(inputs, *, transcripts, coordinates, config): if inp.hgvs in errors_by_hgvs: errors.append(TranslationError(input=inp, error=errors_by_hgvs[inp.hgvs])) elif inp.hgvs in results_by_hgvs: - c_candidates, g_candidates = results_by_hgvs[inp.hgvs] + entry = results_by_hgvs[inp.hgvs] + c_candidates, g_candidates = entry[0], entry[1] + hgvs_p = entry[2] if len(entry) > 2 else None results.append( TranslationResult( input=inp, hgvs_c_candidates=list(c_candidates), hgvs_g_candidates=list(g_candidates), + hgvs_p=hgvs_p, ) ) else: @@ -323,6 +326,57 @@ async def test_transcript_is_queryable_via_derived_expression( coding = session.query(Allele).filter(Allele.transcript == "NM_000001.1").all() assert [a.vrs_digest for a in coding] == ["ga4gh:VA.coding"] + async def test_protein_consequence_is_persisted_as_a_protein_allele( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """The deterministic protein consequence (``result.hgvs_p``) is emitted as the + protein-level member of the equivalence set -- a non-authoritative ``level=protein`` + allele linked to the record, not dropped.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + assay_hgvs = "NM_000000.1:c.1A>G" + g_candidate = "NC_000001.11:g.1000A>G" + c_candidate = "NM_000001.1:c.5A>G" + # c_to_p emits a predicted consequence in parens; the job strips them before use. + p_consequence = "NP_000001.1:p.(Met1Val)" + p_stripped = "NP_000001.1:p.Met1Val" + construct = fake_construct({assay_hgvs: ([c_candidate], [g_candidate], p_consequence)}) + translate = fake_translate( + { + g_candidate: "ga4gh:VA.genomic", + c_candidate: "ga4gh:VA.coding", + p_stripped: "ga4gh:VA.protein", # the stripped form is what reaches the translator + } + ) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + protein_allele = session.query(Allele).filter(Allele.vrs_digest == "ga4gh:VA.protein").one() + assert protein_allele.level == AnnotationLayer.protein.value + assert protein_allele.hgvs_p == p_stripped # stored without the prediction parens + # Linked to the record as a non-authoritative member of the equivalence set. + assert protein_allele.id in {link.allele_id for link in _non_authoritative_links(session)} + async def test_cis_phased_multivariant_candidate_is_persisted_as_a_block( self, session, From 08634e45e251afbeefb813c0e1da7f29cf8cd8be Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 9 Jun 2026 15:18:22 -0700 Subject: [PATCH 023/252] feat(mapping): use typed MappingOutcome to distinguish benign skips from failures Previously, variant mapping success was determined solely by the presence of pre/post-mapped alleles. This conflated genuinely failed mappings with benign absences (intronic variants, no-protein-consequence variants) that legitimately produce no allele. - Add `MappingOutcome` enum to `lib/mapping/schema.py` mirroring `dcd_mapping.schemas.MappingOutcome`, with an `is_benign_absence` helper to classify INTRONIC and NO_PROTEIN_CONSEQUENCE outcomes - Rewrite the per-record outcome logic in `map_variants_for_score_set` to branch on the typed outcome rather than allele presence: MAPPED -> SUCCESS, benign absence -> SKIPPED, FAILED -> FAILED - Replace the `successful_mapped_variants` scalar with a typed `Counter[MappingOutcome]` that feeds `mapped_count`, `failed_count`, and `skipped_count` tallies in logs and final state decision - Derive `mapping_state` from genuine failures only; all-benign result sets are treated as complete, not failed - Raise `NonexistentMappingResultsError` when a score annotation has no `outcome` field (malformed/older payload) - Expand test coverage for intronic and no-protein-consequence scenarios, verifying correct status and failure-category assignment --- src/mavedb/lib/mapping/schema.py | 22 +++++ .../worker/jobs/variant_processing/mapping.py | 83 ++++++++++++----- tests/helpers/util/setup/worker.py | 6 ++ .../jobs/variant_processing/test_mapping.py | 90 +++++++++++++++++++ 4 files changed, 180 insertions(+), 21 deletions(-) diff --git a/src/mavedb/lib/mapping/schema.py b/src/mavedb/lib/mapping/schema.py index 939b60401..02d377780 100644 --- a/src/mavedb/lib/mapping/schema.py +++ b/src/mavedb/lib/mapping/schema.py @@ -7,9 +7,31 @@ """ from datetime import datetime +from enum import Enum from typing import Any, Optional, TypedDict +class MappingOutcome(str, Enum): + """Per-record outcome stamped by dcd-mapping on every emitted score annotation. + + Mirrors ``dcd_mapping.schemas.MappingOutcome``. Lets a benign absence of a VRS allele + be told apart from a genuine failure -- a distinction ``error_message`` alone cannot + make (benign outcomes leave it ``None``). ``MAPPED`` is a success; ``INTRONIC`` and + ``NO_PROTEIN_CONSEQUENCE`` are benign skips (no allele, not failures); ``FAILED`` is the + only genuine failure. + """ + + MAPPED = "mapped" + INTRONIC = "intronic" + NO_PROTEIN_CONSEQUENCE = "no_protein_consequence" + FAILED = "failed" + + @property + def is_benign_absence(self) -> bool: + """True for outcomes that legitimately produce no allele yet are not failures.""" + return self in (MappingOutcome.INTRONIC, MappingOutcome.NO_PROTEIN_CONSEQUENCE) + + class GeneInfo(TypedDict, total=False): hgnc_symbol: Optional[str] selection_method: Optional[str] diff --git a/src/mavedb/worker/jobs/variant_processing/mapping.py b/src/mavedb/worker/jobs/variant_processing/mapping.py index d8b02cebb..9e6a8da74 100644 --- a/src/mavedb/worker/jobs/variant_processing/mapping.py +++ b/src/mavedb/worker/jobs/variant_processing/mapping.py @@ -8,6 +8,7 @@ import asyncio import functools import logging +from collections import Counter from datetime import date from typing import Any @@ -23,6 +24,7 @@ ) from mavedb.lib.logging.context import format_raised_exception_info_as_dict from mavedb.lib.mapping import EXCLUDED_PREMAPPED_ANNOTATION_KEYS +from mavedb.lib.mapping.schema import MappingOutcome from mavedb.lib.types.workflow import JobExecutionOutcome from mavedb.lib.variant_translations import get_or_create_allele from mavedb.lib.variants import get_hgvs_from_post_mapped @@ -258,7 +260,10 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan total_variants = len(mapped_scores) job_manager.save_to_context({"total_variants_to_process": total_variants}) - successful_mapped_variants = 0 + # Tally every record by its typed outcome; the mapped/failed/benign buckets are + # derived from this after the loop. Keeping all four keys preserves the + # intronic-vs-no-protein distinction in logs. + outcome_counts: Counter[MappingOutcome] = Counter() logger.info( f"Processing {total_variants} mapped variants for score set {score_set.urn}.", extra=job_manager.logging_context(), @@ -282,10 +287,17 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan if existing_mapped_variant: job_manager.save_to_context({"existing_mapped_variant": existing_mapped_variant.id}) - annotation_was_successful = mapped_score.get("pre_mapped") and mapped_score.get("post_mapped") - if annotation_was_successful: - successful_mapped_variants += 1 - job_manager.save_to_context({"successful_mapped_variants": successful_mapped_variants}) + # The typed outcome -- not allele presence -- decides success/benign/failure. + # Absent outcome means an older or malformed payload; fail fast. + raw_outcome = mapped_score.get("outcome") + if not raw_outcome: + raise NonexistentMappingResultsError( + f"ScoreAnnotation for variant {variant_urn!r} is missing its outcome." + ) + outcome = MappingOutcome(raw_outcome) + + outcome_counts[outcome] += 1 + job_manager.save_to_context({"outcome_counts": {o.value: n for o, n in outcome_counts.items()}}) # dcd-mapping guarantees both fields are set on every ScoreAnnotation, # including failed variants (the annotate step re-attributes failures to @@ -337,25 +349,36 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan else: job_manager.db.add(mapping_record) + # MAPPED -> success; benign absences -> skipped; FAILED -> failed. The raw outcome + # is preserved in annotation_metadata so the benign distinction survives. + if outcome is MappingOutcome.MAPPED: + annotation_status = AnnotationStatus.SUCCESS + annotation_failure_category = None + elif outcome.is_benign_absence: + annotation_status = AnnotationStatus.SKIPPED + annotation_failure_category = None + else: + annotation_status = AnnotationStatus.FAILED + annotation_failure_category = AnnotationFailureCategory.EXTERNAL_SERVICE_REJECTED + annotation_manager.add_annotation( variant_id=variant.id, # type: ignore annotation_type=AnnotationType.VRS_MAPPING, version=tool_version, - status=AnnotationStatus.SUCCESS if annotation_was_successful else AnnotationStatus.FAILED, - failure_category=None - if annotation_was_successful - else AnnotationFailureCategory.EXTERNAL_SERVICE_REJECTED, + status=annotation_status, + failure_category=annotation_failure_category, annotation_data={ "error_message": mapped_score.get("error_message", null()), "annotation_metadata": { + "outcome": outcome.value, "mapped_assay_level_hgvs": assay_level_hgvs, }, }, current=True, ) - # Only variants with a post-mapped representation yield an authoritative Allele. Failed variants - # get a MappingRecord (with null VRS data) but no linked allele. + # Only variants with a post-mapped representation yield an authoritative Allele; + # failed and benign-absent variants get a MappingRecord but no linked allele. if post_mapped_allele: allele_draft = AlleleDbModel( vrs_digest=post_mapped_allele["id"], @@ -383,17 +406,26 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan annotation_manager.flush() - if successful_mapped_variants == 0: + # Collapse the per-outcome tally into the three buckets the rest of the job reasons about. + mapped_count = outcome_counts[MappingOutcome.MAPPED] + failed_count = outcome_counts[MappingOutcome.FAILED] + skipped_count = sum(n for o, n in outcome_counts.items() if o.is_benign_absence) + + # State keys off genuine failures only: no failures -> complete (benign absences + # don't count); failures with nothing mapped -> failed; otherwise incomplete. + if failed_count == 0: + score_set.mapping_state = MappingState.complete + elif mapped_count == 0: score_set.mapping_state = MappingState.failed score_set.mapping_errors = {"error_message": "All variants failed to map."} - elif successful_mapped_variants < total_variants: - score_set.mapping_state = MappingState.incomplete else: - score_set.mapping_state = MappingState.complete + score_set.mapping_state = MappingState.incomplete job_manager.save_to_context( { - "successful_mapped_variants": successful_mapped_variants, + "mapped_count": mapped_count, + "failed_count": failed_count, + "skipped_count": skipped_count, "mapping_state": score_set.mapping_state.name, "mapping_errors": score_set.mapping_errors, "inserted_mapped_variants": len(mapped_scores), @@ -442,7 +474,8 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan logger.info(msg="Inserted mapped variants into db.", extra=job_manager.logging_context()) - if successful_mapped_variants == 0: + # Fail the job only on genuine failure with nothing mapped; all-benign is a success. + if mapped_count == 0 and failed_count > 0: logger.error(msg="No variants were successfully mapped.", extra=job_manager.logging_context()) job_manager.db.flush() return JobExecutionOutcome.failed( @@ -450,19 +483,27 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan data={ "score_set_id": score_set.id, "mapped_count": 0, - "unmapped_count": total_variants, + "failed_count": failed_count, + "skipped_count": skipped_count, "total_count": total_variants, }, failure_category=FailureCategory.VRS_MAPPING_FAILED, ) - logger.info(msg="Variant mapping job completed successfully.", extra=job_manager.logging_context()) + logger.info( + msg=( + f"Variant mapping job completed successfully: {mapped_count} mapped, " + f"{failed_count} failed, {skipped_count} skipped (intronic / no protein consequence)." + ), + extra=job_manager.logging_context(), + ) job_manager.db.flush() return JobExecutionOutcome.succeeded( data={ "score_set_id": score_set.id, - "mapped_count": successful_mapped_variants, - "unmapped_count": total_variants - successful_mapped_variants, + "mapped_count": mapped_count, + "failed_count": failed_count, + "skipped_count": skipped_count, "total_count": total_variants, } ) diff --git a/tests/helpers/util/setup/worker.py b/tests/helpers/util/setup/worker.py index 815dd1a90..252471ca8 100644 --- a/tests/helpers/util/setup/worker.py +++ b/tests/helpers/util/setup/worker.py @@ -183,6 +183,12 @@ async def construct_mock_mapping_output( if not with_all_variants and idx % 2 == 0: mapped_score["post_mapped"] = {} + # Mirror the mapper's per-record outcome: both alleles present -> MAPPED, else + # FAILED. Tests needing benign outcomes set ``outcome`` explicitly afterward. + mapped_score["outcome"] = ( + "mapped" if mapped_score["pre_mapped"] and mapped_score["post_mapped"] else "failed" + ) + mapping_output["mapped_scores"].append(mapped_score) if not mapping_output["mapped_scores"]: diff --git a/tests/worker/jobs/variant_processing/test_mapping.py b/tests/worker/jobs/variant_processing/test_mapping.py index 74c9acb1c..3a0c6db2f 100644 --- a/tests/worker/jobs/variant_processing/test_mapping.py +++ b/tests/worker/jobs/variant_processing/test_mapping.py @@ -684,6 +684,96 @@ async def dummy_mapping_job(): assert len(annotation_status_failed) == 1 assert annotation_status_failed[0].annotation_type == "vrs_mapping" + async def test_map_variants_for_score_set_benign_outcomes_are_not_failures( + self, + session, + with_independent_processing_runs, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_score_set, + ): + """A score set whose only unmapped variants are benign absences (intronic / no + protein consequence) is ``complete``, not ``incomplete``/``failed``: benign + outcomes carry no allele but are skips, not failures.""" + + async def dummy_mapping_job(): + mapping_output = await construct_mock_mapping_output( + session=session, + score_set=sample_score_set, + with_gene_info=True, + with_layers={"g", "c", "p"}, + with_pre_mapped=True, + with_post_mapped=True, + with_reference_metadata=True, + with_mapped_scores=True, + with_all_variants=True, + ) + # Re-stamp the emitted records as benign absences: no allele, no failure. The + # helper only produces MAPPED/FAILED, so we override here to exercise the path. + benign_outcomes = ["intronic", "no_protein_consequence"] + for idx, mapped_score in enumerate(mapping_output["mapped_scores"]): + mapped_score["pre_mapped"] = {} + mapped_score["post_mapped"] = {} + mapped_score["outcome"] = benign_outcomes[idx % len(benign_outcomes)] + return mapping_output + + variant1 = Variant( + score_set_id=sample_score_set.id, + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + urn="variant:1", + ) + variant2 = Variant( + score_set_id=sample_score_set.id, + hgvs_nt="NM_000000.1:c.2G>T", + hgvs_pro="NP_000000.1:p.Val2Leu", + data={}, + urn="variant:2", + ) + session.add_all([variant1, variant2]) + session.commit() + + with ( + patch.object( + _UnixSelectorEventLoop, + "run_in_executor", + return_value=dummy_mapping_job(), + ), + ): + result = await map_variants_for_score_set( + mock_worker_ctx, + sample_independent_variant_mapping_run.id, + JobManager(session, mock_worker_ctx["redis"], sample_independent_variant_mapping_run.id), + ) + + # The run succeeded and the score set is complete -- nothing genuinely failed. + assert isinstance(result, JobExecutionOutcome) + assert result.status == JobStatus.SUCCEEDED + assert result.data["mapped_count"] == 0 + assert result.data["failed_count"] == 0 + assert result.data["skipped_count"] == 2 + assert sample_score_set.mapping_state == MappingState.complete + assert sample_score_set.mapping_errors is None + + # A record exists per variant, but with no authoritative allele. + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 2 + assert all(authoritative_allele_for(session, r) is None for r in mapping_records) + + # Benign outcomes are SKIPPED (not FAILED), and the finer outcome is preserved in metadata. + annotation_statuses = ( + session.query(VariantAnnotationStatus) + .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) + .filter(Variant.score_set_id == sample_score_set.id) + .all() + ) + assert len(annotation_statuses) == 2 + assert all(s.status == "skipped" for s in annotation_statuses) + assert all(s.failure_category is None for s in annotation_statuses) + recorded_outcomes = {s.annotation_metadata["outcome"] for s in annotation_statuses} + assert recorded_outcomes == {"intronic", "no_protein_consequence"} + async def test_map_variants_for_score_set_complete_mapping( self, session, From 0f1ea170f263eb3ec2db89974d6b2b0f74b76594 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 9 Jun 2026 22:55:11 -0700 Subject: [PATCH 024/252] refactor(reverse-translation): replace NullTranscriptSource with live UTA-backed source WtCodonMode.ALL reads the reference codon via TranscriptSource.codon_at, which requires a real UTA connection. The previous NullTranscriptSource always returned None, silently breaking WT-codon resolution. - Extract `uta_transcript_source()` context manager into `translation_ports.py`; removes the ad-hoc UTA connection setup that was duplicated in the job and the now-deleted NullTranscriptSource - Scope the live UTA client around the full `run_in_executor` call so the connection outlives the synchronous executor block - Remove NullTranscriptSource; callers that only need transcript_for_protein (already resolved by the job) also benefit from the real client without extra cost - Add a TODO noting that non-substitution consequences (del/ins/delins/ fs/ext) are miscounted as FAILED rather than SKIPPED (#767) --- .../variant_processing/reverse_translation.py | 43 ++++++++++--------- src/mavedb/worker/lib/translation_ports.py | 37 ++++++++++------ .../test_reverse_translation.py | 17 +++++--- 3 files changed, 56 insertions(+), 41 deletions(-) diff --git a/src/mavedb/worker/jobs/variant_processing/reverse_translation.py b/src/mavedb/worker/jobs/variant_processing/reverse_translation.py index 872044e91..1bf918b0a 100644 --- a/src/mavedb/worker/jobs/variant_processing/reverse_translation.py +++ b/src/mavedb/worker/jobs/variant_processing/reverse_translation.py @@ -8,11 +8,9 @@ """ import asyncio -import contextlib import dataclasses import functools import logging -import os from datetime import date from enum import Enum from typing import Any, NamedTuple, Sequence @@ -20,7 +18,6 @@ from ga4gh.vrs.extras.translator import AlleleTranslator from sqlalchemy import select from variant_annotation.lib.accessions import looks_like_refseq_protein_accession -from variant_annotation.lib.clients.uta import UtaClient, connect_uta from variant_annotation.lib.translation import construct_equivalent_variants from variant_annotation.lib.translation.types import TranslationConfig, VariantInput, WtCodonMode @@ -43,7 +40,7 @@ from mavedb.worker.jobs.utils.setup import validate_job_params from mavedb.worker.lib.decorators.pipeline_management import with_pipeline_management from mavedb.worker.lib.managers.job_manager import JobManager -from mavedb.worker.lib.translation_ports import NullTranscriptSource, WorkerCoordinateTranslator +from mavedb.worker.lib.translation_ports import WorkerCoordinateTranslator, uta_transcript_source logger = logging.getLogger(__name__) @@ -109,12 +106,7 @@ def _coding_transcripts_for_proteins(protein_accessions: set[str]) -> dict[str, if not protein_accessions: return {} - uta_db_url = (os.environ.get("UTA_DB_URL") or "").strip() - if not uta_db_url: - raise RuntimeError("UTA_DB_URL must be set to resolve protein→transcript associations.") - - with contextlib.closing(connect_uta(uta_db_url)) as conn: - client = UtaClient(conn) + with uta_transcript_source() as client: return { pro_ac: transcript for pro_ac in sorted(protein_accessions) @@ -305,19 +297,24 @@ async def reverse_translate_variants_for_score_set( # construct_equivalent_variants is I/O-bound (blocks on a subprocess); run in the # default thread pool rather than the process pool to avoid pickling the port objects. coordinates = WorkerCoordinateTranslator(ctx["hdp"]) - transcripts = NullTranscriptSource() loop = asyncio.get_running_loop() job_manager.update_progress(20, 100, "Running reverse translation subprocess.") - results, errors = await loop.run_in_executor( - None, - functools.partial( - construct_equivalent_variants, - variant_inputs, - transcripts=transcripts, - coordinates=coordinates, - config=translation_config, - ), - ) + # WtCodonMode.ALL reads the reference codon via TranscriptSource.codon_at, so pass a + # live UTA-backed source. codon_at is only touched in the post-subprocess WT-codon step, + # but the connection must outlive the executor call, so scope it around the whole call. + # (transcript_for_protein is redundant here -- the job already resolved every transcript + # and supplies it via VariantInput.transcript.) + with uta_transcript_source() as transcripts: + results, errors = await loop.run_in_executor( + None, + functools.partial( + construct_equivalent_variants, + variant_inputs, + transcripts=transcripts, + coordinates=coordinates, + config=translation_config, + ), + ) job_manager.update_progress(70, 100, "Writing translated alleles to database.") logger.info( @@ -455,6 +452,10 @@ async def reverse_translate_variants_for_score_set( MappingRecordAllele.mapping_record_id.in_(current_record_ids), ) + # TODO#767: non-substitution consequences (del/ins/delins/fs/ext) have no synonymous + # equivalence class, so they arrive here as TranslationErrors and are miscounted as + # FAILED. Classify by the protein consequence's edit type up front and map these to + # SKIPPED instead of pattern-matching engine error strings. for error in errors: _rec, variant = variant_input_map[id(error.input)] failed += 1 diff --git a/src/mavedb/worker/lib/translation_ports.py b/src/mavedb/worker/lib/translation_ports.py index 467d2f93a..b715aace4 100644 --- a/src/mavedb/worker/lib/translation_ports.py +++ b/src/mavedb/worker/lib/translation_ports.py @@ -1,8 +1,9 @@ """Worker-side adapters for the variant_annotation translation ports. construct_equivalent_variants depends on two protocols — CoordinateTranslator -and TranscriptSource. These are the worker's concrete implementations, so jobs -can pass them in without importing the port definitions directly. +and TranscriptSource. This module supplies the worker's CoordinateTranslator +(WorkerCoordinateTranslator) and a factory (uta_transcript_source) for the +TranscriptSource, which is variant_annotation's own UTA-backed UtaClient. WorkerCoordinateTranslator defers AssemblyMapper initialization until the first call — the hgvs library makes network calls on mapper construction that must not @@ -11,8 +12,28 @@ from __future__ import annotations +import contextlib +import os +from collections.abc import Generator from typing import Any +from variant_annotation.lib.clients.uta import UtaClient, connect_uta + + +@contextlib.contextmanager +def uta_transcript_source() -> Generator[UtaClient]: + """Yield a UTA-backed TranscriptSource over a connection scoped to the block. + + Backs both the NP_→NM_ association lookup and the WT-codon read used by + WtCodonMode.ALL (TranscriptSource.codon_at). The connection is closed on exit, + so callers must use the client within the ``with`` block. + """ + uta_db_url = (os.environ.get("UTA_DB_URL") or "").strip() + if not uta_db_url: + raise RuntimeError("UTA_DB_URL must be set to resolve transcript facts (NP_→NM_ associations and WT codons).") + with contextlib.closing(connect_uta(uta_db_url)) as conn: + yield UtaClient(conn) + class WorkerCoordinateTranslator: """CoordinateTranslator backed by the worker's HGVS data provider.""" @@ -45,15 +66,3 @@ def g_to_c(self, g_hgvs: str, transcript: str) -> str: def c_to_g(self, c_hgvs: str) -> str: self._ensure_initialized() return str(self._mapper.c_to_g(self._parser.parse(c_hgvs))) - - -class NullTranscriptSource: - """Null TranscriptSource — the reverse-translation job resolves every coding transcript - itself (including the UTA NP_→NM_ lookup) and always supplies it via VariantInput.transcript, - so the library never needs to resolve one.""" - - def transcript_for_protein(self, protein_accession: str) -> str | None: - return None - - def codon_at(self, transcript: str, aa_position: int) -> str | None: - return None diff --git a/tests/worker/jobs/variant_processing/test_reverse_translation.py b/tests/worker/jobs/variant_processing/test_reverse_translation.py index ef8104b94..ec926f267 100644 --- a/tests/worker/jobs/variant_processing/test_reverse_translation.py +++ b/tests/worker/jobs/variant_processing/test_reverse_translation.py @@ -4,9 +4,10 @@ pytest.importorskip("arq") +import contextlib from asyncio.unix_events import _UnixSelectorEventLoop from datetime import timedelta -from unittest.mock import patch +from unittest.mock import MagicMock, patch from variant_annotation.lib.translation.types import TranslationError, TranslationResult, WtCodonMode @@ -130,11 +131,15 @@ async def dummy_mapping_job(): async def _reverse_translate(session, mock_worker_ctx, rt_run): - return await reverse_translate_variants_for_score_set( - mock_worker_ctx, - rt_run.id, - JobManager(session, mock_worker_ctx["redis"], rt_run.id), - ) + # The job opens a UTA-backed TranscriptSource to back codon_at (WtCodonMode.ALL). + # construct_equivalent_variants is mocked in these tests, so the source is never + # queried -- stub the factory with a no-op context yielding a dummy client. + with patch(f"{RT_MODULE}.uta_transcript_source", lambda: contextlib.nullcontext(MagicMock())): + return await reverse_translate_variants_for_score_set( + mock_worker_ctx, + rt_run.id, + JobManager(session, mock_worker_ctx["redis"], rt_run.id), + ) def _cross_level_statuses(session, score_set_id, status=None): From 6e45f562f61e1aacc3a9f5ad2524be02ab5d121d Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Thu, 9 Jul 2026 12:33:21 -0700 Subject: [PATCH 025/252] fix(migration): correct down_revision reference in allele closure tables migration --- .../versions/a1b2c3d4e5f6_add_vrs_allele_closure_tables.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/alembic/versions/a1b2c3d4e5f6_add_vrs_allele_closure_tables.py b/alembic/versions/a1b2c3d4e5f6_add_vrs_allele_closure_tables.py index 49c502d05..6c1fc1d99 100644 --- a/alembic/versions/a1b2c3d4e5f6_add_vrs_allele_closure_tables.py +++ b/alembic/versions/a1b2c3d4e5f6_add_vrs_allele_closure_tables.py @@ -1,7 +1,7 @@ """Add mapping_records, alleles, and mapping_record_alleles tables Revision ID: a1b2c3d4e5f6 -Revises: 398067c53257 +Revises: a7f3c2e9b104 Create Date: 2026-05-29 New parallel tables for the Better Reverse Translation epic (#746). @@ -15,7 +15,7 @@ # revision identifiers, used by Alembic. revision = "a1b2c3d4e5f6" -down_revision = "398067c53257" +down_revision = "a7f3c2e9b104" branch_labels = None depends_on = None From d3f4da9fc02bdd9ab579164631d5538c43299f55 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Thu, 18 Jun 2026 15:07:06 -0700 Subject: [PATCH 026/252] fix(vrs): stamp HGVS expressions on reverse-translated alleles and handle null gracefully - `translate_hgvs_to_variation` now attaches an `Expression` to each allele produced from a cis-phased or single HGVS string, mirroring the dcd_mapping authoritative-allele convention so `post_mapped` is self-describing without a separate round-trip. - `hgvs_from_vrs_allele` returns `None` instead of crashing when `expressions` is null or empty (valid for cis-phased block members), and `get_hgvs_from_post_mapped` propagates that as a `None` result. - `post_mapped` is now serialized with `exclude_none=True`, matching the mapper's output format. - Minor formatting clean-ups in reverse_translation.py (no logic change). --- src/mavedb/lib/variants.py | 37 +++++++++++-------- src/mavedb/lib/vrs_utils.py | 33 +++++++++++++++-- .../variant_processing/reverse_translation.py | 3 +- tests/lib/test_variants.py | 20 ++++++++++ .../test_reverse_translation.py | 3 +- 5 files changed, 74 insertions(+), 22 deletions(-) diff --git a/src/mavedb/lib/variants.py b/src/mavedb/lib/variants.py index a941006cf..c2d5595f9 100644 --- a/src/mavedb/lib/variants.py +++ b/src/mavedb/lib/variants.py @@ -10,20 +10,23 @@ HGVS_P_REGEX = re.compile(r"(^|:)p\.") -def hgvs_from_vrs_allele(allele: dict) -> str: +def hgvs_from_vrs_allele(allele: dict) -> Optional[str]: """ - Extract the HGVS notation from the VRS allele. + Extract the HGVS notation from the VRS allele, or None if it carries no expression. """ try: - # VRS 2.X - return allele["expressions"][0]["value"] + expressions = allele["expressions"] # VRS 2.X except KeyError: if "variation" in allele: raise ValueError("VRS 1.X format not supported.") # VRS 1.X. We don't want to allow this. - # return allele["variation"]["expressions"][0]["value"] - else: - raise KeyError("Invalid VRS allele structure. Expected 'expressions'.") + raise KeyError("Invalid VRS allele structure. Expected 'expressions'.") + + # A valid VRS allele may simply carry no HGVS expression (None or empty) — e.g. a member of a + # cis-phased block. That is "no HGVS", not a crash. + if not expressions: + return None + return expressions[0]["value"] def get_hgvs_from_post_mapped(post_mapped_vrs: Optional[Any], *, combine_cis: bool = False) -> Optional[str]: @@ -38,22 +41,24 @@ def get_hgvs_from_post_mapped(post_mapped_vrs: Optional[Any], *, combine_cis: bo if not post_mapped_vrs: return None - if post_mapped_vrs["type"] == "Haplotype": # type: ignore - variations_hgvs = [hgvs_from_vrs_allele(allele) for allele in post_mapped_vrs["members"]] - elif post_mapped_vrs["type"] == "CisPhasedBlock": # type: ignore - variations_hgvs = [hgvs_from_vrs_allele(allele) for allele in post_mapped_vrs["members"]] + if post_mapped_vrs["type"] in ("Haplotype", "CisPhasedBlock"): # type: ignore + members = post_mapped_vrs["members"] elif post_mapped_vrs["type"] == "Allele": # type: ignore - variations_hgvs = [hgvs_from_vrs_allele(post_mapped_vrs)] + members = [post_mapped_vrs] else: return None - if len(variations_hgvs) == 0: + member_hgvs = [hgvs_from_vrs_allele(allele) for allele in members] + + # No members, or a member carrying no HGVS expression — no single/combinable HGVS to return. + if not member_hgvs or any(h is None for h in member_hgvs): return None - if len(variations_hgvs) > 1: - return join_cis_phased_hgvs(variations_hgvs) if combine_cis else None + hgvs_values: list[str] = [h for h in member_hgvs if h is not None] + if len(hgvs_values) > 1: + return join_cis_phased_hgvs(hgvs_values) if combine_cis else None - return variations_hgvs[0] + return hgvs_values[0] def get_digest_from_post_mapped(post_mapped_vrs: Optional[Any]) -> Optional[str]: diff --git a/src/mavedb/lib/vrs_utils.py b/src/mavedb/lib/vrs_utils.py index 052c4b702..d5a6432ee 100644 --- a/src/mavedb/lib/vrs_utils.py +++ b/src/mavedb/lib/vrs_utils.py @@ -17,14 +17,35 @@ from ga4gh.vrs.models import ( Allele, CisPhasedBlock, + Expression, LiteralSequenceExpression, ReferenceLengthExpression, SequenceLocation, + Syntax, ) from ga4gh.vrs.normalize import normalize from mavedb.lib.hgvs import split_cis_phased_hgvs +# HGVS type letter (``accession:g.``) → VRS Expression syntax. +_HGVS_SYNTAX_BY_TYPE = { + "g": Syntax.HGVS_G, + "c": Syntax.HGVS_C, + "p": Syntax.HGVS_P, + "n": Syntax.HGVS_N, + "m": Syntax.HGVS_M, + "r": Syntax.HGVS_R, +} + + +def _hgvs_syntax(hgvs: str) -> Syntax: + """Map an HGVS string to its VRS Expression syntax via the type letter after the accession.""" + _, _, rest = hgvs.partition(":") + try: + return _HGVS_SYNTAX_BY_TYPE[rest[:1]] + except KeyError: + raise ValueError(f"Cannot determine HGVS syntax for {hgvs!r}") + def translate_hgvs_to_vrs(hgvs: str, translator: AlleleTranslator) -> Allele: """Convert HGVS variation description to VRS object. @@ -81,10 +102,14 @@ def translate_hgvs_to_variation(hgvs: str, translator: AlleleTranslator) -> Alle :param translator: caller-owned AlleleTranslator reused across calls :return: an Allele for a single variant, or a CisPhasedBlock for a cis-phased set """ - members = [ - normalize_and_identify(translate_hgvs_to_vrs(component, translator), translator.data_proxy) - for component in split_cis_phased_hgvs(hgvs) - ] + members = [] + for component in split_cis_phased_hgvs(hgvs): + allele = normalize_and_identify(translate_hgvs_to_vrs(component, translator), translator.data_proxy) + # Stamp the source HGVS as the allele's expression so post_mapped is self-describing. + # Mirrors the mapper's authoritative alleles. + allele.expressions = [Expression(syntax=_hgvs_syntax(component), value=component)] + members.append(allele) + if len(members) == 1: return members[0] diff --git a/src/mavedb/worker/jobs/variant_processing/reverse_translation.py b/src/mavedb/worker/jobs/variant_processing/reverse_translation.py index 1bf918b0a..07df529fd 100644 --- a/src/mavedb/worker/jobs/variant_processing/reverse_translation.py +++ b/src/mavedb/worker/jobs/variant_processing/reverse_translation.py @@ -391,7 +391,8 @@ async def reverse_translate_variants_for_score_set( seen_digests.add(variation.id) draft_allele = AlleleDbModel( vrs_digest=variation.id, - post_mapped=variation.model_dump(), + # exclude_none mirrors the mapper's serialization. + post_mapped=variation.model_dump(exclude_none=True), level=level, **{hgvs_field: hgvs}, # type: ignore[arg-type] ) diff --git a/tests/lib/test_variants.py b/tests/lib/test_variants.py index 9cffaa793..8b25197ca 100644 --- a/tests/lib/test_variants.py +++ b/tests/lib/test_variants.py @@ -83,6 +83,26 @@ def test_get_hgvs_from_post_mapped_invalid_structure(): get_hgvs_from_post_mapped({"invalid_key": "InvalidType"}) +def test_hgvs_from_vrs_allele_null_or_empty_expressions(): + # A VRS allele may carry `expressions: null` or `[]` — that is "no HGVS", not a crash. + assert hgvs_from_vrs_allele({"type": "Allele", "expressions": None}) is None + assert hgvs_from_vrs_allele({"type": "Allele", "expressions": []}) is None + + +def test_get_hgvs_from_post_mapped_member_without_expression(): + # Regression: a cis-phased block member whose `expressions` is null must yield None, not raise + # `TypeError: 'NoneType' object is not subscriptable` (which previously killed the CAR job). + block = { + "type": "CisPhasedBlock", + "members": [ + {"type": "Allele", "expressions": [{"value": "NM_003345:p.Asp5Phe"}]}, + {"type": "Allele", "expressions": None}, + ], + } + assert get_hgvs_from_post_mapped(block) is None + assert get_hgvs_from_post_mapped(block, combine_cis=True) is None + + ### Tests for get_digest_from_post_mapped function ### diff --git a/tests/worker/jobs/variant_processing/test_reverse_translation.py b/tests/worker/jobs/variant_processing/test_reverse_translation.py index ec926f267..970fa8544 100644 --- a/tests/worker/jobs/variant_processing/test_reverse_translation.py +++ b/tests/worker/jobs/variant_processing/test_reverse_translation.py @@ -50,7 +50,8 @@ def __init__(self, vrs_id: str, vrs_type: str = "Allele"): self.id = vrs_id self.vrs_type = vrs_type - def model_dump(self) -> dict: + def model_dump(self, **kwargs) -> dict: + # Accept (and ignore) model_dump kwargs like exclude_none, mirroring the real VRS model. return {"type": self.vrs_type, "id": self.id} From 225f824f01aa665d2fe15fe77c325f847d8b410f Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Thu, 18 Jun 2026 15:10:28 -0700 Subject: [PATCH 027/252] feat(clingen): migrate CAR/LDH jobs from MappedVariant to Allele data model - `submit_score_set_mappings_to_car` now operates on Allele rows (authoritative + RT-derived) rather than MappedVariant, deduplicating by allele_id so each VRS allele is registered exactly once regardless of how many variants share it. Adds `force_reregister` param and per-allele outcome counters. - `submit_score_set_mappings_to_ldh` queries MappingRecord + Allele for pre/post-mapped data instead of the deprecated MappedVariant join. - `construct_ldh_submission_entity` signature updated to accept MappingRecord and Allele separately, since those fields now live on different models. - `warm_clingen_cache` switched to the shared `get_alleles_for_score_set` helper to keep allele scope consistent across all three jobs. - Extracts `get_alleles_for_score_set` and `ScoreSetAlleleRow` into `lib/clingen/alleles.py` as the single canonical query for both CAR and cache jobs. --- src/mavedb/lib/clingen/alleles.py | 61 ++ .../lib/clingen/content_constructors.py | 26 +- src/mavedb/models/score_set.py | 2 +- .../worker/jobs/external_services/clingen.py | 481 +++++++---- .../jobs/external_services/clingen_cache.py | 26 +- .../lib/clingen/test_content_constructors.py | 27 +- tests/lib/conftest.py | 17 + .../jobs/external_services/test_clingen.py | 771 ++++++++++++------ .../external_services/test_clingen_cache.py | 185 +++-- 9 files changed, 1100 insertions(+), 496 deletions(-) create mode 100644 src/mavedb/lib/clingen/alleles.py diff --git a/src/mavedb/lib/clingen/alleles.py b/src/mavedb/lib/clingen/alleles.py new file mode 100644 index 000000000..527572233 --- /dev/null +++ b/src/mavedb/lib/clingen/alleles.py @@ -0,0 +1,61 @@ +"""Query helpers for fetching score-set alleles for ClinGen registration. + +Both submit_score_set_mappings_to_car and warm_clingen_cache use the same allele +scope: all current MappingRecordAllele links (authoritative and RT-derived) for a +score set. A single definition here prevents the two jobs from drifting apart. +""" + +from typing import NamedTuple + +from sqlalchemy import select +from sqlalchemy.orm import Session + +from mavedb.models.allele import Allele +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.variant import Variant + + +class ScoreSetAlleleRow(NamedTuple): + """One (allele, variant) link for a score set. An allele shared by multiple variants + appears once per variant so callers can fan annotation statuses out correctly. + + ``is_authoritative`` is a property of the link, not the allele: the same VRS allele can be + the authoritative measurement for one variant and an RT-derived equivalence for another. + """ + + allele_id: int + post_mapped: dict | None + clingen_allele_id: str | None + variant_id: int + is_authoritative: bool + + +def get_alleles_for_score_set(db: Session, score_set_id: int) -> list[ScoreSetAlleleRow]: + """Return all current alleles for a score set with their linked variant IDs. + + Covers both authoritative mapper alleles and RT-derived equivalence alleles — + the full set that requires ClinGen registration before the annotation fan-out + can run. + + Only alleles with a non-null ``post_mapped`` are returned — variants that failed + or were benignly absent have no allele link and cannot receive a CAID. + """ + rows = db.execute( + select( + Allele.id, + Allele.post_mapped, + Allele.clingen_allele_id, + Variant.id.label("variant_id"), + MappingRecordAllele.is_authoritative, + ) + .join(MappingRecordAllele, MappingRecordAllele.allele_id == Allele.id) + .join(MappingRecord, MappingRecord.id == MappingRecordAllele.mapping_record_id) + .join(Variant, Variant.id == MappingRecord.variant_id) + .where(Variant.score_set_id == score_set_id) + .where(MappingRecord.current) + .where(MappingRecordAllele.current) + .where(Allele.post_mapped.is_not(None)) + ).all() + + return [ScoreSetAlleleRow(r.id, r.post_mapped, r.clingen_allele_id, r.variant_id, r.is_authoritative) for r in rows] diff --git a/src/mavedb/lib/clingen/content_constructors.py b/src/mavedb/lib/clingen/content_constructors.py index 1f55437f4..ce62ebbbd 100644 --- a/src/mavedb/lib/clingen/content_constructors.py +++ b/src/mavedb/lib/clingen/content_constructors.py @@ -1,5 +1,4 @@ from datetime import datetime -from typing import Optional from uuid import uuid4 from urllib.parse import quote_plus @@ -7,7 +6,8 @@ from mavedb.constants import MAVEDB_BASE_GIT, MAVEDB_FRONTEND_URL from mavedb.lib.types.clingen import LdhContentLinkedData, LdhContentSubject, LdhEvent, LdhSubmission from mavedb.lib.clingen.constants import LDH_ENTITY_NAME, LDH_SUBMISSION_TYPE -from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.allele import Allele +from mavedb.models.mapping_record import MappingRecord from mavedb.models.variant import Variant @@ -32,8 +32,12 @@ def construct_ldh_submission_subject(hgvs: str) -> LdhContentSubject: return {"Variant": {"hgvs": hgvs}} -def construct_ldh_submission_entity(variant: Variant, mapped_variant: Optional[MappedVariant]) -> LdhContentLinkedData: - entity: LdhContentLinkedData = { +def construct_ldh_submission_entity( + variant: Variant, mapping_record: MappingRecord, allele: Allele +) -> LdhContentLinkedData: + # Pre-mapped data and the mapping API version live on the per-variant MappingRecord; + # post-mapped data lives on the (cross-variant deduped) Allele. + return { # TODO#372: We try to make all possible fields that are non-nullable represented that way. "MaveDBMapping": [ { @@ -41,27 +45,25 @@ def construct_ldh_submission_entity(variant: Variant, mapped_variant: Optional[M "mavedb_id": variant.urn, # type: ignore "score": variant.data["score_data"]["score"], # type: ignore "score_set_description": variant.score_set.short_description, # type: ignore + "pre_mapped": mapping_record.pre_mapped, + "post_mapped": allele.post_mapped, + "mapping_api_version": mapping_record.mapping_api_version, }, "entId": variant.urn, # type: ignore "entIri": f"{MAVEDB_FRONTEND_URL}/score-sets/{quote_plus(variant.score_set.urn)}?variant={quote_plus(variant.urn)}", # type: ignore } ] } - if mapped_variant is not None: - entity["MaveDBMapping"][0]["entContent"]["pre_mapped"] = mapped_variant.pre_mapped - entity["MaveDBMapping"][0]["entContent"]["post_mapped"] = mapped_variant.post_mapped - entity["MaveDBMapping"][0]["entContent"]["mapping_api_version"] = mapped_variant.mapping_api_version - return entity def construct_ldh_submission( - variant_content: list[tuple[str, Variant, Optional[MappedVariant]]], + variant_content: list[tuple[str, Variant, MappingRecord, Allele]], ) -> list[LdhSubmission]: content_submission: list[LdhSubmission] = [] - for hgvs, variant, mapped_variant in variant_content: + for hgvs, variant, mapping_record, allele in variant_content: subject = construct_ldh_submission_subject(hgvs) event = construct_ldh_submission_event(subject) - entity = construct_ldh_submission_entity(variant, mapped_variant) + entity = construct_ldh_submission_entity(variant, mapping_record, allele) content_submission.append( { diff --git a/src/mavedb/models/score_set.py b/src/mavedb/models/score_set.py index ccdcb73d5..b61704ffe 100644 --- a/src/mavedb/models/score_set.py +++ b/src/mavedb/models/score_set.py @@ -74,7 +74,7 @@ class ScoreSet(Base): __tablename__ = "scoresets" - id = Column(Integer, primary_key=True) + id: Mapped[int] = Column(Integer, primary_key=True) # TODO(#372) urn = Column(String(64), default=generate_temp_urn, index=True, nullable=True, unique=True) diff --git a/src/mavedb/worker/jobs/external_services/clingen.py b/src/mavedb/worker/jobs/external_services/clingen.py index 5a65cc728..27aa1b962 100644 --- a/src/mavedb/worker/jobs/external_services/clingen.py +++ b/src/mavedb/worker/jobs/external_services/clingen.py @@ -3,7 +3,6 @@ This module contains jobs for submitting mapped variants to ClinGen services: - ClinGen Allele Registry (CAR) for allele registration - ClinGen Linked Data Hub (LDH) for data submission -- Variant linking and association management These jobs enable integration with the ClinGen ecosystem for clinical variant interpretation and data sharing. @@ -12,10 +11,12 @@ import asyncio import functools import logging +from dataclasses import dataclass, field from sqlalchemy import select from mavedb.lib.annotation_status_manager import AnnotationStatusManager +from mavedb.lib.clingen.alleles import get_alleles_for_score_set from mavedb.lib.clingen.constants import ( CAR_SUBMISSION_ENDPOINT, CLIN_GEN_SUBMISSION_ENABLED, @@ -29,9 +30,11 @@ ) from mavedb.lib.types.workflow import JobExecutionOutcome from mavedb.lib.variants import get_hgvs_from_post_mapped +from mavedb.models.allele import Allele as AlleleModel from mavedb.models.enums.annotation_type import AnnotationType from mavedb.models.enums.job_pipeline import AnnotationFailureCategory, AnnotationStatus, FailureCategory -from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.score_set import ScoreSet from mavedb.models.variant import Variant from mavedb.worker.jobs.utils.setup import validate_job_params @@ -41,6 +44,50 @@ logger = logging.getLogger(__name__) +@dataclass +class _AlleleEntry: + post_mapped: dict | None + existing_caid: str | None + # Variants for which THIS allele is the authoritative measurement — the only ones that receive a + # per-variant VAS row. INTERIM BANDAID (do not deploy as final): keying clingen's per-variant + # status to the single authoritative link sidesteps the multiple "current" rows a full allele + # fan-out would write for one variant. Durable fix is an allele-level event log; rationale and + # migration seam in docs/design/allele-annotation-status.md. + authoritative_variant_ids: list[int] = field(default_factory=list) + + +def _annotate_caid( + annotation_manager: AnnotationStatusManager, + variant_ids: list[int], + status: AnnotationStatus, + *, + failure_category: AnnotationFailureCategory | None = None, + error_message: str | None = None, + metadata: dict | None = None, +) -> None: + """Fan a CLINGEN_ALLELE_ID annotation out to every variant served by an allele. + + AAS migration seam: the single choke point for clingen's per-variant VAS writes. At migration it + becomes an allele-keyed event writer; the per-variant fan-out goes away, and the variant + association narrows to provenance (who caused the registration). See + docs/design/allele-annotation-status.md. + """ + annotation_data: dict = {"annotation_metadata": metadata or {}} + if error_message is not None: + annotation_data["error_message"] = error_message + + for variant_id in variant_ids: + annotation_manager.add_annotation( + variant_id=variant_id, + annotation_type=AnnotationType.CLINGEN_ALLELE_ID, + version=None, + status=status, + failure_category=failure_category, + annotation_data=annotation_data, + current=True, + ) + + @with_pipeline_management async def submit_score_set_mappings_to_car(ctx: dict, job_id: int, job_manager: JobManager) -> JobExecutionOutcome: """ @@ -58,11 +105,11 @@ async def submit_score_set_mappings_to_car(ctx: dict, job_id: int, job_manager: job_manager (JobManager): Manager for job lifecycle and DB operations Side Effects: - - Updates MappedVariant records with ClinGen Allele IDs + - Updates Allele records with ClinGen Allele IDs - Submits data to ClinGen Allele Registry Returns: - dict: Result indicating success and any exception details + JobExecutionOutcome: outcome with per-allele counts (submitted/registered/already-registered/failed). """ # Get the job definition we are working on job = job_manager.get_job() @@ -70,11 +117,10 @@ async def submit_score_set_mappings_to_car(ctx: dict, job_id: int, job_manager: _job_required_params = ["score_set_id", "correlation_id"] validate_job_params(_job_required_params, job) - # Fetch required resources based on param inputs. Safely ignore mypy warnings here, as they were checked above. score_set = job_manager.db.scalars(select(ScoreSet).where(ScoreSet.id == job.job_params["score_set_id"])).one() # type: ignore correlation_id = job.job_params["correlation_id"] # type: ignore + force_reregister = bool(job.job_params.get("force_reregister", False)) # type: ignore[union-attr] - # Setup initial context and progress job_manager.save_to_context( { "application": "mavedb-worker", @@ -86,7 +132,6 @@ async def submit_score_set_mappings_to_car(ctx: dict, job_id: int, job_manager: job_manager.update_progress(0, 100, "Starting CAR mapped resource submission.") logger.info(msg="Started CAR mapped resource submission", extra=job_manager.logging_context()) - # Ensure we've enabled ClinGen submission if not CLIN_GEN_SUBMISSION_ENABLED: logger.warning( msg="ClinGen submission is disabled via configuration, skipping submission of mapped variants to CAR.", @@ -95,7 +140,6 @@ async def submit_score_set_mappings_to_car(ctx: dict, job_id: int, job_manager: job_manager.db.flush() return JobExecutionOutcome.skipped(data={"reason": "ClinGen submission disabled"}) - # Check for CAR submission endpoint if not CAR_SUBMISSION_ENDPOINT: logger.warning( msg="ClinGen Allele Registry submission is disabled (no submission endpoint), unable to complete submission of mapped variants to CAR.", @@ -107,183 +151,300 @@ async def submit_score_set_mappings_to_car(ctx: dict, job_id: int, job_manager: failure_category=FailureCategory.CONFIGURATION_ERROR, ) - # Fetch mapped variants with post-mapped data for the score set - variant_post_mapped_objects = job_manager.db.execute( - select(MappedVariant.id, MappedVariant.post_mapped) - .join(Variant) - .join(ScoreSet) - .where(ScoreSet.urn == score_set.urn) - .where(MappedVariant.post_mapped.is_not(None)) - .where(MappedVariant.current.is_(True)) - ).all() - - # Track total variants to submit - job_manager.save_to_context({"total_variants_to_submit_car": len(variant_post_mapped_objects)}) - if not variant_post_mapped_objects: + allele_rows = get_alleles_for_score_set(job_manager.db, score_set.id) + job_manager.save_to_context({"total_allele_variant_pairs": len(allele_rows)}) + if not allele_rows: logger.warning( - msg="No current mapped variants with post mapped metadata were found for this score set. Skipping CAR submission.", + msg="No current alleles found for this score set. Skipping CAR submission.", extra=job_manager.logging_context(), ) job_manager.db.flush() - return JobExecutionOutcome.succeeded(data={"submitted_count": 0, "matched_count": 0}) + return JobExecutionOutcome.succeeded( + data={ + "submitted_allele_count": 0, + "registered_allele_count": 0, + "already_registered_allele_count": 0, + "failed_allele_count": 0, + } + ) - job_manager.update_progress( - 10, 100, f"Preparing {len(variant_post_mapped_objects)} mapped variants for CAR submission." - ) + # Group by allele_id: one allele may serve multiple variants (cross-score-set dedup). + allele_data: dict[int, _AlleleEntry] = {} + for row in allele_rows: + if row.allele_id not in allele_data: + allele_data[row.allele_id] = _AlleleEntry(post_mapped=row.post_mapped, existing_caid=row.clingen_allele_id) + + if row.is_authoritative: + allele_data[row.allele_id].authoritative_variant_ids.append(row.variant_id) - # Build HGVS strings for submission. Don't do duplicate submissions-- store mapped variant IDs by HGVS. - # Variants that can't produce an HGVS string are annotated as failures immediately. annotation_manager = AnnotationStatusManager(job_manager.db, job_run_id=job_manager.job_id) - variant_post_mapped_hgvs: dict[str, list[int]] = {} - no_hgvs_count = 0 - for mapped_variant_id, post_mapped in variant_post_mapped_objects: - # Intentionally not combine_cis=True: multi-variant cis-phased blocks have no single - # CAID, so they are skipped here pending ClinGen guidance on how to register them - # (https://github.com/VariantEffect/mavedb-api/issues/764). - hgvs_for_post_mapped = get_hgvs_from_post_mapped(post_mapped) - if not hgvs_for_post_mapped: - no_hgvs_count += 1 + # Track outcomes by distinct allele_id. clingen_allele_id is an allele-level fact (the CAID + # lives on the Allele) and CAR's operation is per-allele, so the reported counts are in allele + # units — and they cover every allele submitted, including the RT-derived ones that produce no + # per-variant status row. Each allele has exactly one outcome (submitted once → one response), so + # these sets are disjoint by construction. (Per-variant VAS rows are still written via the + # authoritative link below — that is the interim bandaid, separate from these operation counts.) + linked_allele_ids: set[int] = set() + preexisting_allele_ids: set[int] = set() + failed_allele_ids: set[int] = set() + + # Pre-existing CAIDs: record success without re-submitting unless force_reregister is set. + preexisting = [aid for aid, entry in allele_data.items() if entry.existing_caid] if not force_reregister else [] + for allele_id in preexisting: + entry = allele_data[allele_id] + + preexisting_allele_ids.add(allele_id) + _annotate_caid( + annotation_manager, + entry.authoritative_variant_ids, + AnnotationStatus.SUCCESS, + metadata={"clingen_allele_id": entry.existing_caid, "registration_source": "preexisting"}, + ) + + # Alleles that need CAR submission: new ones, or all when force_reregister=True. + pending_allele_ids = [aid for aid, entry in allele_data.items() if force_reregister or not entry.existing_caid] + job_manager.update_progress(10, 100, f"Preparing {len(pending_allele_ids)} alleles for CAR submission.") + + # Build HGVS → [allele_ids] map. Multi-variant cis-phased blocks produce no HGVS + # (combine_cis defaults to False); those alleles are annotated as failures immediately. + hgvs_to_allele_ids: dict[str, list[int]] = {} + for allele_id in pending_allele_ids: + entry = allele_data[allele_id] + hgvs = get_hgvs_from_post_mapped(entry.post_mapped) + + if hgvs: + hgvs_to_allele_ids.setdefault(hgvs, []).append(allele_id) + + # Allele is registered but post_mapped can no longer produce HGVS — data + # regression worth surfacing, but the CAID is still valid so treat it as + # preexisting rather than failing the variant. + elif entry.existing_caid: + preexisting_allele_ids.add(allele_id) logger.warning( - msg=f"Could not construct a valid HGVS string for mapped variant {mapped_variant_id}. Skipping submission of this variant.", + msg=( + f"Could not construct HGVS for allele {allele_id} during force re-registration " + f"(existing CAID: {entry.existing_caid!r}). Reconfirmation skipped; existing CAID retained." + ), extra=job_manager.logging_context(), ) + _annotate_caid( + annotation_manager, + entry.authoritative_variant_ids, + AnnotationStatus.SUCCESS, + metadata={"clingen_allele_id": entry.existing_caid, "registration_source": "reconfirmation_skipped"}, + ) - mapped_variant = job_manager.db.scalars( - select(MappedVariant).where(MappedVariant.id == mapped_variant_id) - ).one() - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.CLINGEN_ALLELE_ID, - version=None, - status=AnnotationStatus.FAILED, + # No HGVS-- un-submittable. + else: + failed_allele_ids.add(allele_id) + logger.warning( + msg=f"Could not construct HGVS for allele {allele_id}. Skipping CAR submission.", + extra=job_manager.logging_context(), + ) + _annotate_caid( + annotation_manager, + entry.authoritative_variant_ids, + AnnotationStatus.FAILED, failure_category=AnnotationFailureCategory.MISSING_IDENTIFIER, - annotation_data={ - "error_message": "Could not extract a valid HGVS string from post-mapped variant data.", - "annotation_metadata": {}, - }, - current=True, + error_message="Could not extract a valid HGVS string from post-mapped allele data.", ) - continue - if hgvs_for_post_mapped in variant_post_mapped_hgvs: - variant_post_mapped_hgvs[hgvs_for_post_mapped].append(mapped_variant_id) - else: - variant_post_mapped_hgvs[hgvs_for_post_mapped] = [mapped_variant_id] + job_manager.save_to_context({"unique_hgvs_to_submit_car": len(hgvs_to_allele_ids)}) - job_manager.save_to_context({"unique_variants_to_submit_car": len(variant_post_mapped_hgvs)}) - job_manager.update_progress(15, 100, "Submitting mapped variants to CAR.") + # Distinct alleles actually sent to CAR this run (pending alleles that yielded HGVS). + submitted_allele_ids: set[int] = { + allele_id for allele_ids in hgvs_to_allele_ids.values() for allele_id in allele_ids + } + + def _outcome_data() -> dict[str, int]: + return { + "submitted_allele_count": len(submitted_allele_ids), + "registered_allele_count": len(linked_allele_ids), + "already_registered_allele_count": len(preexisting_allele_ids), + "failed_allele_count": len(failed_allele_ids), + } - # Do submission + # All pending alleles failed HGVS extraction; annotations already written above. + if not hgvs_to_allele_ids: + annotation_manager.flush() + job_manager.db.flush() + if preexisting_allele_ids: + return JobExecutionOutcome.succeeded(data=_outcome_data()) + + return JobExecutionOutcome.failed( + reason=f"No submittable alleles for score set {score_set.urn}.", + data=_outcome_data(), + failure_category=FailureCategory.DEPENDENCY_FAILURE, + ) + + job_manager.update_progress(15, 100, "Submitting alleles to CAR.") car_service = ClinGenAlleleRegistryService(url=CAR_SUBMISSION_ENDPOINT) - hgvs_list = list(variant_post_mapped_hgvs.keys()) + hgvs_list = list(hgvs_to_allele_ids.keys()) registered_alleles = car_service.dispatch_submissions(hgvs_list) job_manager.update_progress(60, 100, "Processing registered alleles from CAR.") - # CAR returns one response per submitted HGVS in the same order (see CAR API docs). - # Zip the submissions with the responses and annotate each based on success or error. - linked_count = 0 - error_count = 0 + # Bulk-load every allele that could be linked in one query, keyed by id, rather than + # issuing a SELECT per CAID inside the response loop. + alleles_by_id = { + allele.id: allele + for allele in job_manager.db.scalars(select(AlleleModel).where(AlleleModel.id.in_(submitted_allele_ids))).all() + } + + # CAR's contract is one result per submitted HGVS, in order — that is what makes the + # positional zip below valid. A different count (including the empty list dispatch returns + # on request failure) means alignment cannot be trusted for ANY position, so we register + # nothing and fail the whole batch rather than risk writing a CAID to the wrong allele. + aligned = len(registered_alleles) == len(hgvs_list) + if not aligned: + logger.error( + msg=( + f"CAR returned {len(registered_alleles)} results for {len(hgvs_list)} submitted HGVS; " + "positional alignment cannot be trusted. Failing the entire batch." + ), + extra=job_manager.logging_context(), + ) - for hgvs_string, response in zip(hgvs_list, registered_alleles): - mapped_variant_ids = variant_post_mapped_hgvs[hgvs_string] - mapped_variants = job_manager.db.scalars( - select(MappedVariant).where(MappedVariant.id.in_(mapped_variant_ids)) - ).all() + for hgvs_string, response in zip(hgvs_list, registered_alleles if aligned else []): + allele_ids_for_hgvs = hgvs_to_allele_ids[hgvs_string] if "errorType" in response: - error_count += 1 logger.warning( msg=f"CAR rejected HGVS '{hgvs_string}' ({response.get('errorType', 'unknown')}): {response.get('message', 'unknown')}", extra=job_manager.logging_context(), ) + for allele_id in allele_ids_for_hgvs: + failed_allele_ids.add(allele_id) + _annotate_caid( + annotation_manager, + allele_data[allele_id].authoritative_variant_ids, + AnnotationStatus.FAILED, + failure_category=AnnotationFailureCategory.EXTERNAL_SERVICE_REJECTED, + error_message="Failed to register allele with ClinGen Allele Registry.", + metadata={ + "submitted_hgvs": hgvs_string, + "car_error_type": response.get("errorType"), + "car_error_message": response.get("message"), + }, + ) + + continue + + # A response that is neither an error nor a registration (no "@id") is malformed. + caid_iri = response.get("@id") + if not caid_iri: + logger.error( + msg=f"CAR returned a response for HGVS '{hgvs_string}' with neither an error nor an allele identifier.", + extra=job_manager.logging_context(), + ) + for allele_id in allele_ids_for_hgvs: + failed_allele_ids.add(allele_id) + _annotate_caid( + annotation_manager, + allele_data[allele_id].authoritative_variant_ids, + AnnotationStatus.FAILED, + failure_category=AnnotationFailureCategory.EXTERNAL_SERVICE_REJECTED, + error_message="ClinGen Allele Registry returned a malformed response with no allele identifier.", + metadata={"submitted_hgvs": hgvs_string}, + ) + + continue - for mapped_variant in mapped_variants: - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.CLINGEN_ALLELE_ID, - version=None, - status=AnnotationStatus.FAILED, + caid = caid_iri.split("/")[-1] + for allele_id in allele_ids_for_hgvs: + entry = allele_data[allele_id] + prior_caid = entry.existing_caid + + # CAID is immutable — a different value returned by CAR is a hard invariant + # violation. Do not overwrite; record a failure with full audit context. + if prior_caid and prior_caid != caid: + logger.error( + msg=( + f"CAR returned a different CAID for allele {allele_id}: " + f"stored={prior_caid!r}, returned={caid!r}. " + "Not overwriting. Investigate immediately." + ), + extra=job_manager.logging_context(), + ) + + failed_allele_ids.add(allele_id) + _annotate_caid( + annotation_manager, + entry.authoritative_variant_ids, + AnnotationStatus.FAILED, failure_category=AnnotationFailureCategory.EXTERNAL_SERVICE_REJECTED, - annotation_data={ - "error_message": "Failed to register variant with ClinGen Allele Registry.", - "annotation_metadata": { - "submitted_hgvs": hgvs_string, - "car_error_type": response.get("errorType"), - "car_error_message": response.get("message"), - }, + error_message="CAR returned a CAID that conflicts with the stored value.", + metadata={ + "clingen_allele_id": prior_caid, + "conflicting_caid": caid, + "submitted_hgvs": hgvs_string, }, - current=True, ) - else: - linked_count += 1 - caid = response["@id"].split("/")[-1] - for mapped_variant in mapped_variants: - mapped_variant.clingen_allele_id = caid - job_manager.db.add(mapped_variant) - - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.CLINGEN_ALLELE_ID, - version=None, - status=AnnotationStatus.SUCCESS, - annotation_data={"annotation_metadata": {"clingen_allele_id": caid}}, - current=True, + # CAID is new or matches the stored value — link it to the allele and record success. + else: + linked_allele_ids.add(allele_id) + allele = alleles_by_id[allele_id] + allele.clingen_allele_id = caid + + registration_source = "reconfirmed" if prior_caid else "this_run" + if prior_caid: + logger.info( + msg=f"Force re-registration confirmed same CAID {caid!r} for allele {allele_id}.", + extra=job_manager.logging_context(), + ) + + _annotate_caid( + annotation_manager, + entry.authoritative_variant_ids, + AnnotationStatus.SUCCESS, + metadata={"clingen_allele_id": caid, "registration_source": registration_source}, ) - # Any HGVS strings CAR did not respond to (network drop, service-side omission). - # Use EXTERNAL_SERVICE_REJECTED for explicit CAR errors, EXTERNAL_API_ERROR for silent failures. - no_response_hgvs = hgvs_list[len(registered_alleles) :] - for hgvs_string in no_response_hgvs: - mapped_variants = job_manager.db.scalars( - select(MappedVariant).where(MappedVariant.id.in_(variant_post_mapped_hgvs[hgvs_string])) - ).all() - for mapped_variant in mapped_variants: - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.CLINGEN_ALLELE_ID, - version=None, - status=AnnotationStatus.FAILED, + # Submitted HGVS with no trustworthy response: the truncated tail when the counts line up + # (network drop, service-side omission), or the entire batch when the response count + # violated CAR's one-result-per-input contract and we rejected it above. + unattributed_hgvs = hgvs_list if not aligned else hgvs_list[len(registered_alleles) :] + for hgvs_string in unattributed_hgvs: + for allele_id in hgvs_to_allele_ids[hgvs_string]: + failed_allele_ids.add(allele_id) + _annotate_caid( + annotation_manager, + allele_data[allele_id].authoritative_variant_ids, + AnnotationStatus.FAILED, failure_category=AnnotationFailureCategory.EXTERNAL_API_ERROR, - annotation_data={ - "error_message": "Failed to register variant with ClinGen Allele Registry.", - "annotation_metadata": {"submitted_hgvs": hgvs_string}, - }, - current=True, + error_message="Failed to register allele with ClinGen Allele Registry.", + metadata={"submitted_hgvs": hgvs_string}, ) annotation_manager.flush() - failed_count = no_hgvs_count + error_count + len(no_response_hgvs) + outcome_data = _outcome_data() - # When all registrations fail we will not be able to render any annotations. Fail the job - # to explicitly halt the pipeline. - if linked_count == 0: - error_message = f"CAR submission failed for all {len(hgvs_list)} variants in score set {score_set.urn}." + # When no allele ended up with a CAID (none linked this run, none already registered), the + # pipeline cannot continue — downstream jobs need CAIDs to function. + if not linked_allele_ids and not preexisting_allele_ids: + error_message = ( + f"CAR submission failed for all {outcome_data['submitted_allele_count']} " + f"submitted alleles in score set {score_set.urn}." + ) logger.error(msg=error_message, extra=job_manager.logging_context()) job_manager.db.flush() return JobExecutionOutcome.failed( reason=error_message, - data={"submitted_count": len(hgvs_list), "matched_count": 0, "failed_count": failed_count}, + data=outcome_data, failure_category=FailureCategory.DEPENDENCY_FAILURE, ) - if failed_count > 0: - # CAR rejections are typically per-variant data quality issues (e.g. invalid HGVS) rather than - # systemic failures. Per-variant AnnotationStatus.FAILED records are already written above for - # traceability. We continue the pipeline so that successfully registered variants still receive - # downstream annotations (warm_clingen_cache, gnomAD, ClinVar, HGVS, translations). + if outcome_data["failed_allele_count"] > 0: logger.warning( - msg=f"CAR submission failed for {failed_count} of {len(hgvs_list)} variants in score set {score_set.urn}.", + msg=f"CAR submission failed for {outcome_data['failed_allele_count']} alleles in score set {score_set.urn}.", extra=job_manager.logging_context(), ) logger.info(msg="Completed CAR mapped resource submission", extra=job_manager.logging_context()) job_manager.db.flush() - return JobExecutionOutcome.succeeded( - data={"submitted_count": len(hgvs_list), "matched_count": linked_count, "failed_count": failed_count} - ) + return JobExecutionOutcome.succeeded(data=outcome_data) @with_pipeline_management @@ -306,7 +467,7 @@ async def submit_score_set_mappings_to_ldh(ctx: dict, job_id: int, job_manager: - Submits data to ClinGen Linked Data Hub Returns: - dict: Result indicating success and any exception details + JobExecutionOutcome: outcome with per-variant submitted/failed counts. """ # Get the job definition we are working on job = job_manager.get_job() @@ -334,14 +495,21 @@ async def submit_score_set_mappings_to_ldh(ctx: dict, job_id: int, job_manager: ldh_service = ClinGenLdhService(url=LDH_SUBMISSION_ENDPOINT) ldh_service.authenticate() - # Fetch mapped variants with post-mapped data for the score set + # Fetch each variant's authoritative allele for the score set. Post-mapped data and HGVS + # come from the Allele; pre-mapped data and the mapping API version come from the + # MappingRecord. RT-derived equivalence alleles are intentionally excluded — LDH links each + # MaveDB score to its canonical mapped variant, not to every equivalent allele (unlike CAR, + # which registers a CAID per allele). variant_objects = job_manager.db.execute( - select(Variant, MappedVariant) - .join(MappedVariant) - .join(ScoreSet) - .where(ScoreSet.urn == score_set.urn) - .where(MappedVariant.post_mapped.is_not(None)) - .where(MappedVariant.current.is_(True)) + select(Variant, MappingRecord, AlleleModel) + .join(MappingRecord, MappingRecord.variant_id == Variant.id) + .join(MappingRecordAllele, MappingRecordAllele.mapping_record_id == MappingRecord.id) + .join(AlleleModel, AlleleModel.id == MappingRecordAllele.allele_id) + .where(Variant.score_set_id == score_set.id) + .where(MappingRecord.current) + .where(MappingRecordAllele.current) + .where(MappingRecordAllele.is_authoritative.is_(True)) + .where(AlleleModel.post_mapped.is_not(None)) ).all() # Track total variants to submit @@ -359,19 +527,19 @@ async def submit_score_set_mappings_to_ldh(ctx: dict, job_id: int, job_manager: # Build submission content variant_content = [] variant_for_urn = {} - for variant, mapped_variant in variant_objects: + for variant, mapping_record, allele in variant_objects: # See the note above: cis-phased blocks are skipped here pending ClinGen guidance # (https://github.com/VariantEffect/mavedb-api/issues/764). - variation = get_hgvs_from_post_mapped(mapped_variant.post_mapped) + variation = get_hgvs_from_post_mapped(allele.post_mapped) if not variation: logger.warning( - msg=f"Could not construct a valid HGVS string for mapped variant {mapped_variant.id}. Skipping submission of this variant.", + msg=f"Could not construct a valid HGVS string for allele {allele.id} (variant {variant.urn}). Skipping submission of this variant.", extra=job_manager.logging_context(), ) continue - variant_content.append((variation, variant, mapped_variant)) + variant_content.append((variation, variant, mapping_record, allele)) variant_for_urn[variant.urn] = variant if not variant_content: @@ -409,7 +577,15 @@ async def submit_score_set_mappings_to_ldh(ctx: dict, job_id: int, job_manager: ) submitted_urn = success["data"]["entId"] - submitted_variant = variant_for_urn[submitted_urn] + submitted_variant = variant_for_urn.get(submitted_urn) + if submitted_variant is None: + # LDH echoed back an entId we never submitted — record it for investigation rather + # than crashing the whole job mid-batch. + logger.warning( + msg=f"LDH returned an unrecognized entId not in this submission: {submitted_urn!r}.", + extra=job_manager.logging_context(), + ) + continue annotation_manager.add_annotation( variant_id=submitted_variant.id, @@ -427,7 +603,8 @@ async def submit_score_set_mappings_to_ldh(ctx: dict, job_id: int, job_manager: # especially when submission occurred in batch. Save all failures generically here. # Note that failures may not be present in the submission failures list, but they are # guaranteed to be absent from the successes list. - for failure_urn in set(variant_for_urn.keys()) - submitted_variant_urns: + failed_variant_urns = set(variant_for_urn.keys()) - submitted_variant_urns + for failure_urn in failed_variant_urns: logger.error( msg=f"Failed to submit mapped variant to LDH: {failure_urn}", extra=job_manager.logging_context(), @@ -449,9 +626,15 @@ async def submit_score_set_mappings_to_ldh(ctx: dict, job_id: int, job_manager: annotation_manager.flush() + # Report per-variant counts (matching the annotations written above), not the per-batch + # counts returned by the service — the two use different denominators. + submitted_count = len(submitted_variant_urns) + failed_count = len(failed_variant_urns) + if submission_failures: logger.warning( - msg=f"LDH mapped resource submission encountered {len(submission_failures)} failures.", + msg=f"LDH mapped resource submission encountered {len(submission_failures)} batch failures " + f"({failed_count} variants unconfirmed).", extra=job_manager.logging_context(), ) @@ -465,7 +648,7 @@ async def submit_score_set_mappings_to_ldh(ctx: dict, job_id: int, job_manager: job_manager.db.flush() return JobExecutionOutcome.failed( reason=error_message, - data={"submitted_count": 0, "failed_count": len(submission_failures)}, + data={"submitted_count": submitted_count, "failed_count": failed_count}, failure_category=FailureCategory.DEPENDENCY_FAILURE, ) @@ -475,6 +658,4 @@ async def submit_score_set_mappings_to_ldh(ctx: dict, job_id: int, job_manager: ) job_manager.db.flush() - return JobExecutionOutcome.succeeded( - data={"submitted_count": len(submission_successes), "failed_count": len(submission_failures)} - ) + return JobExecutionOutcome.succeeded(data={"submitted_count": submitted_count, "failed_count": failed_count}) diff --git a/src/mavedb/worker/jobs/external_services/clingen_cache.py b/src/mavedb/worker/jobs/external_services/clingen_cache.py index 10890f23f..d530d881f 100644 --- a/src/mavedb/worker/jobs/external_services/clingen_cache.py +++ b/src/mavedb/worker/jobs/external_services/clingen_cache.py @@ -15,11 +15,10 @@ from sqlalchemy import select from mavedb.lib.clingen.allele_registry import get_clingen_allele_data +from mavedb.lib.clingen.alleles import get_alleles_for_score_set from mavedb.lib.clingen.constants import CLINGEN_CACHE_WARMING_CONCURRENCY from mavedb.lib.types.workflow import JobExecutionOutcome -from mavedb.models.mapped_variant import MappedVariant from mavedb.models.score_set import ScoreSet -from mavedb.models.variant import Variant from mavedb.worker.jobs.utils.setup import validate_job_params from mavedb.worker.lib.decorators.pipeline_management import with_pipeline_management from mavedb.worker.lib.managers.job_manager import JobManager @@ -55,19 +54,16 @@ async def warm_clingen_cache(ctx: dict, job_id: int, job_manager: JobManager) -> job_manager.update_progress(0, 100, "Starting ClinGen cache pre-warming.") logger.info("Starting ClinGen cache pre-warming", extra=job_manager.logging_context()) - # Get distinct clingen_allele_ids for this score set's current mapped variants - allele_ids = job_manager.db.scalars( - select(MappedVariant.clingen_allele_id) - .join(Variant) - .where( - Variant.score_set_id == score_set.id, - MappedVariant.current.is_(True), - MappedVariant.clingen_allele_id.isnot(None), - # Exclude multi-variant IDs (comma-separated) — they can't be fetched individually - MappedVariant.clingen_allele_id.not_like("%,%"), - ) - .distinct() - ).all() + # Get distinct clingen_allele_ids registered for alleles in this score set. + # Exclude None and comma-separated multi-variant IDs that can't be fetched individually. + allele_rows = get_alleles_for_score_set(job_manager.db, score_set.id) + allele_ids = list( + { + row.clingen_allele_id + for row in allele_rows + if row.clingen_allele_id is not None and "," not in row.clingen_allele_id + } + ) total = len(allele_ids) job_manager.save_to_context({"total_allele_ids_to_warm": total}) diff --git a/tests/lib/clingen/test_content_constructors.py b/tests/lib/clingen/test_content_constructors.py index 7aab47f76..f691a4497 100644 --- a/tests/lib/clingen/test_content_constructors.py +++ b/tests/lib/clingen/test_content_constructors.py @@ -11,7 +11,6 @@ ) from mavedb.lib.clingen.constants import LDH_ENTITY_NAME, LDH_SUBMISSION_TYPE from mavedb import __version__ -import pytest from tests.helpers.constants import ( TEST_HGVS_IDENTIFIER, @@ -54,10 +53,8 @@ def test_construct_ldh_submission_event(): } -@pytest.mark.parametrize("has_mapped_variant", [(True), (False)]) -def test_construct_ldh_submission_entity(mock_variant, mock_mapped_variant, has_mapped_variant: bool): - mapped_variant = mock_mapped_variant if has_mapped_variant else None - result = construct_ldh_submission_entity(mock_variant, mapped_variant) +def test_construct_ldh_submission_entity(mock_variant, mock_mapping_record, mock_allele): + result = construct_ldh_submission_entity(mock_variant, mock_mapping_record, mock_allele) assert "MaveDBMapping" in result assert len(result["MaveDBMapping"]) == 1 @@ -65,15 +62,9 @@ def test_construct_ldh_submission_entity(mock_variant, mock_mapped_variant, has_ assert mapping["entContent"]["mavedb_id"] == VALID_VARIANT_URN assert mapping["entContent"]["score"] == 1.0 - - if has_mapped_variant: - assert mapping["entContent"]["pre_mapped"] == TEST_VALID_PRE_MAPPED_VRS_ALLELE_VRS2_X - assert mapping["entContent"]["post_mapped"] == TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X - assert mapping["entContent"]["mapping_api_version"] == "pytest.mapping.1.0" - else: - assert "pre_mapped" not in mapping["entContent"] - assert "post_mapped" not in mapping["entContent"] - assert "mapping_api_version" not in mapping["entContent"] + assert mapping["entContent"]["pre_mapped"] == TEST_VALID_PRE_MAPPED_VRS_ALLELE_VRS2_X + assert mapping["entContent"]["post_mapped"] == TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X + assert mapping["entContent"]["mapping_api_version"] == "pytest.mapping.1.0" assert mapping["entId"] == VALID_VARIANT_URN assert ( @@ -82,12 +73,10 @@ def test_construct_ldh_submission_entity(mock_variant, mock_mapped_variant, has_ ) -@pytest.mark.parametrize("has_mapped_variant", [(True), (False)]) -def test_construct_ldh_submission(mock_variant, mock_mapped_variant, has_mapped_variant: bool): - mapped_variant = mock_mapped_variant if has_mapped_variant else None +def test_construct_ldh_submission(mock_variant, mock_mapping_record, mock_allele): variant_content = [ - (TEST_HGVS_IDENTIFIER, mock_variant, mapped_variant), - (TEST_HGVS_IDENTIFIER, mock_variant, mapped_variant), + (TEST_HGVS_IDENTIFIER, mock_variant, mock_mapping_record, mock_allele), + (TEST_HGVS_IDENTIFIER, mock_variant, mock_mapping_record, mock_allele), ] uuid_1 = UUID("12345678-1234-5678-1234-567812345678") diff --git a/tests/lib/conftest.py b/tests/lib/conftest.py index 2ac2183fb..adfe2a1b8 100644 --- a/tests/lib/conftest.py +++ b/tests/lib/conftest.py @@ -307,6 +307,23 @@ def mock_mapped_variant(mock_variant): return mv +@pytest.fixture +def mock_mapping_record(mock_mapped_variant): + # Pre-mapped data and the mapping API version now live on the per-variant MappingRecord. + rec = mock.Mock() + rec.pre_mapped = mock_mapped_variant.pre_mapped + rec.mapping_api_version = mock_mapped_variant.mapping_api_version + return rec + + +@pytest.fixture +def mock_allele(mock_mapped_variant): + # Post-mapped data now lives on the (cross-variant deduped) Allele. + allele = mock.Mock() + allele.post_mapped = mock_mapped_variant.post_mapped + return allele + + @pytest.fixture def mock_mapped_variant_with_functional_calibration_score_set( mock_mapped_variant, mock_variant_with_functional_calibration_score_set diff --git a/tests/worker/jobs/external_services/test_clingen.py b/tests/worker/jobs/external_services/test_clingen.py index 616263dfa..76b1b0b2d 100644 --- a/tests/worker/jobs/external_services/test_clingen.py +++ b/tests/worker/jobs/external_services/test_clingen.py @@ -5,14 +5,16 @@ pytest.importorskip("arq") from asyncio.unix_events import _UnixSelectorEventLoop +from copy import deepcopy from unittest.mock import patch from sqlalchemy import select from mavedb.lib.types.workflow import JobExecutionOutcome from mavedb.lib.variants import get_hgvs_from_post_mapped +from mavedb.models.allele import Allele from mavedb.models.enums.job_pipeline import JobStatus, PipelineStatus -from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.variant import Variant from mavedb.models.variant_annotation_status import VariantAnnotationStatus from mavedb.worker.jobs.external_services.clingen import ( @@ -51,9 +53,9 @@ async def test_submit_score_set_mappings_to_car_submission_disabled( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SKIPPED - # Verify no variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == 0 + # Verify no alleles have CAIDs assigned + alleles = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles) == 0 async def test_submit_score_set_mappings_to_car_no_mappings( self, @@ -76,9 +78,9 @@ async def test_submit_score_set_mappings_to_car_no_mappings( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - # Verify no variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == 0 + # Verify no alleles have CAIDs assigned + alleles = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles) == 0 async def test_submit_score_set_mappings_to_car_submission_endpoint_not_set( self, @@ -101,9 +103,9 @@ async def test_submit_score_set_mappings_to_car_submission_endpoint_not_set( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.FAILED - # Verify no variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == 0 + # Verify no alleles have CAIDs assigned + alleles = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles) == 0 async def test_submit_score_set_mappings_to_car_no_registered_alleles( self, @@ -147,11 +149,11 @@ async def test_submit_score_set_mappings_to_car_no_registered_alleles( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.FAILED - # Verify no variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == 0 + # Verify no alleles have CAIDs assigned + alleles = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles) == 0 - # Verify annotation statuses were rendered as failed + # Verify annotation statuses were rendered as failed — 4 variants, all failed annotation_statuses = session.scalars( select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") ).all() @@ -184,18 +186,18 @@ async def test_submit_score_set_mappings_to_car_all_car_errors( dummy_variant_mapping_job_run, ) - # Build error responses for every submitted HGVS — CAR explicitly rejected all of them - mapped_variants = session.scalars(select(MappedVariant)).all() + # All 4 variants share 1 allele (same VRS digest). Build an error response for that 1 HGVS. + alleles = session.scalars(select(Allele)).all() + assert len(alleles) == 1 registered_alleles_mock = [ { "errorType": "InvalidHGVS", - "hgvs": get_hgvs_from_post_mapped(mv.post_mapped) or "", + "hgvs": get_hgvs_from_post_mapped(alleles[0].post_mapped) or "", "message": "Invalid HGVS expression.", "description": "", - "inputLine": get_hgvs_from_post_mapped(mv.post_mapped) or "", - "position": str(i), + "inputLine": get_hgvs_from_post_mapped(alleles[0].post_mapped) or "", + "position": "0", } - for i, mv in enumerate(mapped_variants) ] with ( @@ -215,11 +217,11 @@ async def test_submit_score_set_mappings_to_car_all_car_errors( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.FAILED - # Verify no variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == 0 + # Verify no alleles have CAIDs assigned + alleles = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles) == 0 - # Verify annotation statuses were rendered as failed + # 1 allele failed → all 4 variant annotations are failed annotation_statuses = session.scalars( select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") ).all() @@ -228,6 +230,211 @@ async def test_submit_score_set_mappings_to_car_all_car_errors( assert ann.status == "failed" assert ann.annotation_type == "clingen_allele_id" + async def test_submit_score_set_mappings_to_car_derived_allele_no_duplicate_annotation( + self, + mock_worker_ctx, + session, + with_submit_score_set_mappings_to_car_job, + submit_score_set_mappings_to_car_sample_job_run, + mock_s3_client, + sample_score_dataframe, + sample_count_dataframe, + with_dummy_setup_jobs, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ): + """A variant linked to an authoritative AND a derived allele gets exactly one VAS row (from the + authoritative link), while the derived allele is still registered with a CAID.""" + await create_mappings_in_score_set( + session, + mock_s3_client, + mock_worker_ctx, + sample_score_dataframe, + sample_count_dataframe, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ) + + # The sample's 4 variants dedup to one authoritative allele. + authoritative_allele = session.scalars(select(Allele)).one() + + # Attach a second, derived (is_authoritative=False) allele to one variant's current mapping + # record. It shares the authoritative allele's HGVS (different vrs_digest) so it is submitted + # under the same line — the realistic shape is a different level, but identical HGVS is enough + # to exercise the multi-allele-per-variant fan-out. + a_link = session.scalars( + select(MappingRecordAllele).where(MappingRecordAllele.allele_id == authoritative_allele.id) + ).first() + derived_allele = Allele( + vrs_digest=f"{authoritative_allele.vrs_digest}-derived", + level=authoritative_allele.level, + post_mapped={**deepcopy(authoritative_allele.post_mapped), "id": "derived-allele-id"}, + ) + session.add(derived_allele) + session.flush() + session.add( + MappingRecordAllele( + mapping_record_id=a_link.mapping_record_id, + allele_id=derived_allele.id, + is_authoritative=False, + ) + ) + session.flush() + + def fake_dispatch(hgvs_list): + return [{"@id": f"CA{idx}", "type": "nucleotide", "genomicAlleles": []} for idx, _ in enumerate(hgvs_list)] + + with ( + patch( + "mavedb.worker.jobs.external_services.clingen.ClinGenAlleleRegistryService.dispatch_submissions", + side_effect=fake_dispatch, + ), + patch("mavedb.worker.jobs.external_services.clingen.CAR_SUBMISSION_ENDPOINT", "http://fake-endpoint"), + patch("mavedb.worker.jobs.external_services.clingen.CLIN_GEN_SUBMISSION_ENABLED", True), + ): + result = await submit_score_set_mappings_to_car( + mock_worker_ctx, + submit_score_set_mappings_to_car_sample_job_run.id, + JobManager(session, mock_worker_ctx["redis"], submit_score_set_mappings_to_car_sample_job_run.id), + ) + + assert result.status == JobStatus.SUCCEEDED + + # Exactly one current VAS row per variant (4 total) — no duplicate from the derived allele. + current_statuses = session.scalars( + select(VariantAnnotationStatus).where( + VariantAnnotationStatus.annotation_type == "clingen_allele_id", + VariantAnnotationStatus.current.is_(True), + ) + ).all() + assert len(current_statuses) == 4 + assert len({s.variant_id for s in current_statuses}) == 4 + + # Both alleles registered: registration breadth is preserved even though the derived allele + # produced no VAS row. + session.refresh(authoritative_allele) + session.refresh(derived_allele) + assert authoritative_allele.clingen_allele_id is not None + assert derived_allele.clingen_allele_id is not None + + async def test_submit_score_set_mappings_to_car_response_count_mismatch( + self, + mock_worker_ctx, + session, + with_submit_score_set_mappings_to_car_job, + submit_score_set_mappings_to_car_sample_job_run, + mock_s3_client, + sample_score_dataframe, + sample_count_dataframe, + with_dummy_setup_jobs, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ): + # Create mappings in the score set + await create_mappings_in_score_set( + session, + mock_s3_client, + mock_worker_ctx, + sample_score_dataframe, + sample_count_dataframe, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ) + + # All 4 variants share 1 allele → 1 submitted HGVS, but CAR returns 2 results. The count + # violates the one-result-per-input contract, so positional alignment can't be trusted + # and the whole batch must be rejected without writing any CAID. + alleles = session.scalars(select(Allele)).all() + assert len(alleles) == 1 + registered_alleles_mock = [ + {"@id": "CA111111", "type": "nucleotide", "genomicAlleles": []}, + {"@id": "CA222222", "type": "nucleotide", "genomicAlleles": []}, + ] + + with ( + patch( + "mavedb.worker.jobs.external_services.clingen.ClinGenAlleleRegistryService.dispatch_submissions", + return_value=registered_alleles_mock, + ), + patch("mavedb.worker.jobs.external_services.clingen.CAR_SUBMISSION_ENDPOINT", "http://fake-endpoint"), + patch("mavedb.worker.jobs.external_services.clingen.CLIN_GEN_SUBMISSION_ENABLED", True), + ): + result = await submit_score_set_mappings_to_car( + mock_worker_ctx, + submit_score_set_mappings_to_car_sample_job_run.id, + JobManager(session, mock_worker_ctx["redis"], submit_score_set_mappings_to_car_sample_job_run.id), + ) + + assert isinstance(result, JobExecutionOutcome) + assert result.status == JobStatus.FAILED + + # No CAID written — neither of the returned values is trusted. + assert len(session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all()) == 0 + annotation_statuses = session.scalars( + select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + ).all() + assert len(annotation_statuses) == 4 + for ann in annotation_statuses: + assert ann.status == "failed" + assert ann.failure_category == "external_api_error" + + async def test_submit_score_set_mappings_to_car_malformed_response( + self, + mock_worker_ctx, + session, + with_submit_score_set_mappings_to_car_job, + submit_score_set_mappings_to_car_sample_job_run, + mock_s3_client, + sample_score_dataframe, + sample_count_dataframe, + with_dummy_setup_jobs, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ): + # Create mappings in the score set + await create_mappings_in_score_set( + session, + mock_s3_client, + mock_worker_ctx, + sample_score_dataframe, + sample_count_dataframe, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ) + + # CAR returns a response that is neither an error (no "errorType") nor a registration + # (no "@id"). This must be surfaced as a rejection, not crash the loop with a KeyError. + alleles = session.scalars(select(Allele)).all() + assert len(alleles) == 1 + registered_alleles_mock = [{"type": "nucleotide", "genomicAlleles": []}] + + with ( + patch( + "mavedb.worker.jobs.external_services.clingen.ClinGenAlleleRegistryService.dispatch_submissions", + return_value=registered_alleles_mock, + ), + patch("mavedb.worker.jobs.external_services.clingen.CAR_SUBMISSION_ENDPOINT", "http://fake-endpoint"), + patch("mavedb.worker.jobs.external_services.clingen.CLIN_GEN_SUBMISSION_ENABLED", True), + ): + result = await submit_score_set_mappings_to_car( + mock_worker_ctx, + submit_score_set_mappings_to_car_sample_job_run.id, + JobManager(session, mock_worker_ctx["redis"], submit_score_set_mappings_to_car_sample_job_run.id), + ) + + assert isinstance(result, JobExecutionOutcome) + assert result.status == JobStatus.FAILED + + # No CAID assigned, and all 4 variant annotations are failed as rejected. + assert len(session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all()) == 0 + annotation_statuses = session.scalars( + select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + ).all() + assert len(annotation_statuses) == 4 + for ann in annotation_statuses: + assert ann.status == "failed" + assert ann.failure_category == "external_service_rejected" + async def test_submit_score_set_mappings_to_car_repeated_hgvs( self, mock_worker_ctx, @@ -252,13 +459,14 @@ async def test_submit_score_set_mappings_to_car_repeated_hgvs( dummy_variant_mapping_job_run, ) - # Patch ClinGenAlleleRegistryService to return registered alleles with repeated HGVS - mapped_variants = session.scalars(select(MappedVariant)).all() + # 4 variants share 1 allele; CAR returns 1 response for the 1 submitted HGVS + alleles = session.scalars(select(Allele)).all() + assert len(alleles) == 1 registered_alleles_mock = [ { "@id": "CA_DUPLICATE", "type": "nucleotide", - "genomicAlleles": [{"hgvs": get_hgvs_from_post_mapped(mapped_variants[0].post_mapped)}], + "genomicAlleles": [{"hgvs": get_hgvs_from_post_mapped(alleles[0].post_mapped)}], } ] @@ -267,11 +475,6 @@ async def test_submit_score_set_mappings_to_car_repeated_hgvs( "mavedb.worker.jobs.external_services.clingen.ClinGenAlleleRegistryService.dispatch_submissions", return_value=registered_alleles_mock, ), - # Patch get_hgvs_from_post_mapped to return the same HGVS for all variants - patch( - "mavedb.worker.jobs.external_services.clingen.get_hgvs_from_post_mapped", - return_value=get_hgvs_from_post_mapped(mapped_variants[0].post_mapped), - ), patch("mavedb.worker.jobs.external_services.clingen.CAR_SUBMISSION_ENDPOINT", "http://fake-endpoint"), patch("mavedb.worker.jobs.external_services.clingen.CLIN_GEN_SUBMISSION_ENABLED", True), ): @@ -284,13 +487,12 @@ async def test_submit_score_set_mappings_to_car_repeated_hgvs( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - # Verify variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == 4 - for variant in variants: - assert variant.clingen_allele_id == "CA_DUPLICATE" + # 1 allele received the CAID + alleles = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles) == 1 + assert alleles[0].clingen_allele_id == "CA_DUPLICATE" - # Verify annotation statuses were rendered as success + # 4 per-variant annotations — all success annotation_statuses = session.scalars( select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") ).all() @@ -312,7 +514,7 @@ async def test_submit_score_set_mappings_to_car_partial_failure( dummy_variant_creation_job_run, dummy_variant_mapping_job_run, ): - """Test that partial CAR failures (some matched, some not) result in a succeeded outcome with failure annotations.""" + """All 4 variants share 1 allele; CAR returns 1 success → 1 registered allele, 0 failed.""" # Create mappings in the score set await create_mappings_in_score_set( session, @@ -324,16 +526,14 @@ async def test_submit_score_set_mappings_to_car_partial_failure( dummy_variant_mapping_job_run, ) - # Get mapped variants; return a CAR response that only matches the first variant - mapped_variants = session.scalars(select(MappedVariant)).all() - assert len(mapped_variants) == 4 + alleles = session.scalars(select(Allele)).all() + assert len(alleles) == 1 - first_hgvs = get_hgvs_from_post_mapped(mapped_variants[0].post_mapped) registered_alleles_mock = [ { - "@id": f"CA{mapped_variants[0].id}", + "@id": f"CA{alleles[0].id}", "type": "nucleotide", - "genomicAlleles": [{"hgvs": first_hgvs}], + "genomicAlleles": [{"hgvs": get_hgvs_from_post_mapped(alleles[0].post_mapped)}], } ] @@ -353,31 +553,22 @@ async def test_submit_score_set_mappings_to_car_partial_failure( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - assert result.data["matched_count"] == 1 - assert result.data["failed_count"] == 3 + assert result.data["registered_allele_count"] == 1 + assert result.data["failed_allele_count"] == 0 - # Verify only the first variant got a CAID - variants_with_caid = session.scalars( - select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None)) - ).all() - assert len(variants_with_caid) == 1 - assert variants_with_caid[0].clingen_allele_id == f"CA{mapped_variants[0].id}" + # 1 allele got a CAID + alleles_with_caid = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles_with_caid) == 1 + assert alleles_with_caid[0].clingen_allele_id == f"CA{alleles[0].id}" - # Verify annotation statuses: 1 success, 3 failed + # All 4 variant annotations succeeded success_annotations = session.scalars( select(VariantAnnotationStatus).where( VariantAnnotationStatus.annotation_type == "clingen_allele_id", VariantAnnotationStatus.status == "success", ) ).all() - failed_annotations = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.annotation_type == "clingen_allele_id", - VariantAnnotationStatus.status == "failed", - ) - ).all() - assert len(success_annotations) == 1 - assert len(failed_annotations) == 3 + assert len(success_annotations) == 4 async def test_submit_score_set_mappings_to_car_hgvs_not_found( self, @@ -403,32 +594,11 @@ async def test_submit_score_set_mappings_to_car_hgvs_not_found( dummy_variant_mapping_job_run, ) - # Get the mapped variants from score set before submission - mapped_variants = session.scalars( - select(MappedVariant) - .join(Variant) - .where(Variant.score_set_id == submit_score_set_mappings_to_car_sample_job_run.job_params["score_set_id"]) - ).all() - - # Patch ClinGenAlleleRegistryService to return registered alleles - registered_alleles_mock = [ - { - "@id": f"CA{mv.id}", - "type": "nucleotide", - "genomicAlleles": [{"hgvs": get_hgvs_from_post_mapped(mv.post_mapped)}], - } - for mv in mapped_variants - ] - + # Patch get_hgvs_from_post_mapped to return None for all alleles with ( - patch( - "mavedb.worker.jobs.external_services.clingen.ClinGenAlleleRegistryService.dispatch_submissions", - return_value=registered_alleles_mock, - ), - # Patch get_hgvs_from_post_mapped to not find any HGVS in registered alleles - patch("mavedb.worker.jobs.external_services.clingen.get_hgvs_from_post_mapped", return_value=None), patch("mavedb.worker.jobs.external_services.clingen.CAR_SUBMISSION_ENDPOINT", "http://fake-endpoint"), patch("mavedb.worker.jobs.external_services.clingen.CLIN_GEN_SUBMISSION_ENABLED", True), + patch("mavedb.worker.jobs.external_services.clingen.get_hgvs_from_post_mapped", return_value=None), ): result = await submit_score_set_mappings_to_car( mock_worker_ctx, @@ -439,11 +609,11 @@ async def test_submit_score_set_mappings_to_car_hgvs_not_found( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.FAILED - # Verify no variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == 0 + # Verify no alleles have CAIDs assigned + alleles = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles) == 0 - # Verify annotation statuses were rendered as failed + # Verify annotation statuses were rendered as failed — 4 variants, all failed annotation_statuses = session.scalars( select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") ).all() @@ -519,20 +689,16 @@ async def test_submit_score_set_mappings_to_car_success( dummy_variant_mapping_job_run, ) - # Get the mapped variants from score set before submission - mapped_variants = session.scalars( - select(MappedVariant).join(Variant).where(Variant.score_set_id == sample_score_set.id) - ).all() - assert len(mapped_variants) == 4 + # All 4 variants share 1 allele (same VRS digest from construct_mock_mapping_output) + alleles = session.scalars(select(Allele)).all() + assert len(alleles) == 1 - # Patch ClinGenAlleleRegistryService to return registered alleles registered_alleles_mock = [ { - "@id": f"CA{mv.id}", + "@id": f"CA{alleles[0].id}", "type": "nucleotide", - "genomicAlleles": [{"hgvs": get_hgvs_from_post_mapped(mv.post_mapped)}], + "genomicAlleles": [{"hgvs": get_hgvs_from_post_mapped(alleles[0].post_mapped)}], } - for mv in mapped_variants ] with ( @@ -552,13 +718,12 @@ async def test_submit_score_set_mappings_to_car_success( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - # Verify variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == 4 - for variant in variants: - assert variant.clingen_allele_id == f"CA{variant.id}" + # 1 allele received the CAID + alleles_with_caid = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles_with_caid) == 1 + assert alleles_with_caid[0].clingen_allele_id == f"CA{alleles[0].id}" - # Verify annotation statuses were rendered as success + # 4 per-variant annotations — all success annotation_statuses = session.scalars( select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") ).all() @@ -567,6 +732,192 @@ async def test_submit_score_set_mappings_to_car_success( assert ann.status == "success" assert ann.annotation_type == "clingen_allele_id" + async def test_submit_score_set_mappings_to_car_preexisting( + self, + mock_worker_ctx, + session, + with_submit_score_set_mappings_to_car_job, + submit_score_set_mappings_to_car_sample_job_run, + mock_s3_client, + sample_score_dataframe, + sample_count_dataframe, + with_dummy_setup_jobs, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ): + """Already-registered allele is re-annotated as preexisting, not re-submitted.""" + await create_mappings_in_score_set( + session, + mock_s3_client, + mock_worker_ctx, + sample_score_dataframe, + sample_count_dataframe, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ) + + # Pre-set the CAID on the allele to simulate prior registration + allele = session.scalars(select(Allele)).first() + allele.clingen_allele_id = "CA_PRIOR" + session.flush() + + with ( + patch( + "mavedb.worker.jobs.external_services.clingen.ClinGenAlleleRegistryService.dispatch_submissions", + ) as mock_dispatch, + patch("mavedb.worker.jobs.external_services.clingen.CAR_SUBMISSION_ENDPOINT", "http://fake-endpoint"), + patch("mavedb.worker.jobs.external_services.clingen.CLIN_GEN_SUBMISSION_ENABLED", True), + ): + result = await submit_score_set_mappings_to_car( + mock_worker_ctx, + submit_score_set_mappings_to_car_sample_job_run.id, + JobManager(session, mock_worker_ctx["redis"], submit_score_set_mappings_to_car_sample_job_run.id), + ) + + # CAR should NOT be called since the allele is already registered + mock_dispatch.assert_not_called() + assert result.status == JobStatus.SUCCEEDED + assert result.data["already_registered_allele_count"] == 1 + assert result.data["submitted_allele_count"] == 0 + + annotation_statuses = session.scalars( + select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + ).all() + assert len(annotation_statuses) == 4 + for ann in annotation_statuses: + assert ann.status == "success" + assert ann.annotation_metadata["registration_source"] == "preexisting" + assert ann.annotation_metadata["clingen_allele_id"] == "CA_PRIOR" + + async def test_submit_score_set_mappings_to_car_force_reregister_same_caid( + self, + mock_worker_ctx, + session, + with_submit_score_set_mappings_to_car_job, + submit_score_set_mappings_to_car_sample_job_run, + mock_s3_client, + sample_score_dataframe, + sample_count_dataframe, + with_dummy_setup_jobs, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ): + """Force re-registration that returns the same CAID is a success with registration_source=reconfirmed.""" + await create_mappings_in_score_set( + session, + mock_s3_client, + mock_worker_ctx, + sample_score_dataframe, + sample_count_dataframe, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ) + + allele = session.scalars(select(Allele)).first() + allele.clingen_allele_id = "CA_CONFIRMED" + session.flush() + + submit_score_set_mappings_to_car_sample_job_run.job_params = { + **submit_score_set_mappings_to_car_sample_job_run.job_params, + "force_reregister": True, + } + session.flush() + + registered_alleles_mock = [{"@id": "CA_CONFIRMED", "type": "nucleotide", "genomicAlleles": []}] + + with ( + patch( + "mavedb.worker.jobs.external_services.clingen.ClinGenAlleleRegistryService.dispatch_submissions", + return_value=registered_alleles_mock, + ), + patch("mavedb.worker.jobs.external_services.clingen.CAR_SUBMISSION_ENDPOINT", "http://fake-endpoint"), + patch("mavedb.worker.jobs.external_services.clingen.CLIN_GEN_SUBMISSION_ENABLED", True), + ): + result = await submit_score_set_mappings_to_car( + mock_worker_ctx, + submit_score_set_mappings_to_car_sample_job_run.id, + JobManager(session, mock_worker_ctx["redis"], submit_score_set_mappings_to_car_sample_job_run.id), + ) + + assert result.status == JobStatus.SUCCEEDED + assert result.data["registered_allele_count"] == 1 + assert result.data["submitted_allele_count"] == 1 + + annotation_statuses = session.scalars( + select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + ).all() + assert len(annotation_statuses) == 4 + for ann in annotation_statuses: + assert ann.status == "success" + assert ann.annotation_metadata["registration_source"] == "reconfirmed" + + async def test_submit_score_set_mappings_to_car_force_reregister_caid_conflict( + self, + mock_worker_ctx, + session, + with_submit_score_set_mappings_to_car_job, + submit_score_set_mappings_to_car_sample_job_run, + mock_s3_client, + sample_score_dataframe, + sample_count_dataframe, + with_dummy_setup_jobs, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ): + """Force re-registration returning a different CAID fails without overwriting the stored CAID.""" + await create_mappings_in_score_set( + session, + mock_s3_client, + mock_worker_ctx, + sample_score_dataframe, + sample_count_dataframe, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ) + + allele = session.scalars(select(Allele)).first() + allele.clingen_allele_id = "CA_STORED" + session.flush() + + submit_score_set_mappings_to_car_sample_job_run.job_params = { + **submit_score_set_mappings_to_car_sample_job_run.job_params, + "force_reregister": True, + } + session.flush() + + # CAR returns a DIFFERENT CAID — this is an invariant violation + registered_alleles_mock = [{"@id": "CA_DIFFERENT", "type": "nucleotide", "genomicAlleles": []}] + + with ( + patch( + "mavedb.worker.jobs.external_services.clingen.ClinGenAlleleRegistryService.dispatch_submissions", + return_value=registered_alleles_mock, + ), + patch("mavedb.worker.jobs.external_services.clingen.CAR_SUBMISSION_ENDPOINT", "http://fake-endpoint"), + patch("mavedb.worker.jobs.external_services.clingen.CLIN_GEN_SUBMISSION_ENABLED", True), + ): + result = await submit_score_set_mappings_to_car( + mock_worker_ctx, + submit_score_set_mappings_to_car_sample_job_run.id, + JobManager(session, mock_worker_ctx["redis"], submit_score_set_mappings_to_car_sample_job_run.id), + ) + + # Job fails because all CAID-returning submissions had a conflict + assert result.status == JobStatus.FAILED + + # CAID must NOT have been overwritten + session.refresh(allele) + assert allele.clingen_allele_id == "CA_STORED" + + annotation_statuses = session.scalars( + select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + ).all() + assert len(annotation_statuses) == 4 + for ann in annotation_statuses: + assert ann.status == "failed" + assert ann.annotation_metadata["clingen_allele_id"] == "CA_STORED" + assert ann.annotation_metadata["conflicting_caid"] == "CA_DIFFERENT" + @pytest.mark.integration @pytest.mark.asyncio @@ -597,15 +948,14 @@ async def test_submit_score_set_mappings_to_car_independent_ctx( dummy_variant_mapping_job_run, ) - # Patch ClinGenAlleleRegistryService to return registered alleles - mapped_variants = session.scalars(select(MappedVariant)).all() + # All 4 variants share 1 allele; build 1 CAR response + alleles = session.scalars(select(Allele)).all() registered_alleles_mock = [ { - "@id": f"CA{mv.id}", + "@id": f"CA{alleles[0].id}", "type": "nucleotide", - "genomicAlleles": [{"hgvs": get_hgvs_from_post_mapped(mv.post_mapped)}], + "genomicAlleles": [{"hgvs": get_hgvs_from_post_mapped(alleles[0].post_mapped)}], } - for mv in mapped_variants ] with ( @@ -623,17 +973,16 @@ async def test_submit_score_set_mappings_to_car_independent_ctx( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - # Verify variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == len(mapped_variants) - for variant in variants: - assert variant.clingen_allele_id == f"CA{variant.id}" + # 1 allele received the CAID + alleles_with_caid = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles_with_caid) == 1 + assert alleles_with_caid[0].clingen_allele_id == f"CA{alleles[0].id}" - # Verify annotation statuses were rendered as success + # 4 per-variant annotations — all success annotation_statuses = session.scalars( select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == len(mapped_variants) + assert len(annotation_statuses) == 4 for ann in annotation_statuses: assert ann.status == "success" @@ -666,15 +1015,14 @@ async def test_submit_score_set_mappings_to_car_pipeline_ctx( dummy_variant_mapping_job_run, ) - # Patch ClinGenAlleleRegistryService to return registered alleles - mapped_variants = session.scalars(select(MappedVariant)).all() + # All 4 variants share 1 allele; build 1 CAR response + alleles = session.scalars(select(Allele)).all() registered_alleles_mock = [ { - "@id": f"CA{mv.id}", + "@id": f"CA{alleles[0].id}", "type": "nucleotide", - "genomicAlleles": [{"hgvs": get_hgvs_from_post_mapped(mv.post_mapped)}], + "genomicAlleles": [{"hgvs": get_hgvs_from_post_mapped(alleles[0].post_mapped)}], } - for mv in mapped_variants ] with ( @@ -692,17 +1040,16 @@ async def test_submit_score_set_mappings_to_car_pipeline_ctx( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - # Verify variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == len(mapped_variants) - for variant in variants: - assert variant.clingen_allele_id == f"CA{variant.id}" + # 1 allele received the CAID + alleles_with_caid = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles_with_caid) == 1 + assert alleles_with_caid[0].clingen_allele_id == f"CA{alleles[0].id}" - # Verify annotation statuses were rendered as success + # 4 per-variant annotations — all success annotation_statuses = session.scalars( select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == len(mapped_variants) + assert len(annotation_statuses) == 4 for ann in annotation_statuses: assert ann.status == "success" @@ -738,9 +1085,9 @@ async def test_submit_score_set_mappings_to_car_submission_disabled( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SKIPPED - # Verify no variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == 0 + # Verify no alleles have CAIDs assigned + alleles = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles) == 0 # Verify no annotation statuses were created annotation_statuses = session.scalars(select(VariantAnnotationStatus)).all() @@ -777,9 +1124,9 @@ async def test_submit_score_set_mappings_to_car_no_submission_endpoint( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.FAILED - # Verify no variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == 0 + # Verify no alleles have CAIDs assigned + alleles = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles) == 0 # Verify no annotation statuses were created annotation_statuses = session.scalars(select(VariantAnnotationStatus)).all() @@ -808,9 +1155,9 @@ async def test_submit_score_set_mappings_to_car_no_mappings( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - # Verify no variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == 0 + # Verify no alleles have CAIDs assigned + alleles = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles) == 0 # Verify no annotation statuses were created annotation_statuses = session.scalars(select(VariantAnnotationStatus)).all() @@ -862,11 +1209,11 @@ async def test_submit_score_set_mappings_to_car_no_registered_alleles( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.FAILED - # Verify no variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == 0 + # Verify no alleles have CAIDs assigned + alleles = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles) == 0 - # Verify annotation statuses were rendered as failed + # Verify annotation statuses were rendered as failed — 4 variants, all failed annotation_statuses = session.scalars( select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") ).all() @@ -903,9 +1250,6 @@ async def test_submit_score_set_mappings_to_car_no_linked_alleles( # Patch ClinGenAlleleRegistryService to return only errors with no linked alleles registered_alleles_mock = [ {"errorType": "InvalidHGVS", "hgvs": "test"}, - {"errorType": "InvalidHGVS", "hgvs": "test2"}, - {"errorType": "InvalidHGVS", "hgvs": "test3"}, - {"errorType": "InvalidHGVS", "hgvs": "test4"}, ] with ( @@ -925,11 +1269,11 @@ async def test_submit_score_set_mappings_to_car_no_linked_alleles( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.FAILED - # Verify no variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == 0 + # Verify no alleles have CAIDs assigned + alleles = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles) == 0 - # Verify annotation statuses were rendered as failed + # Verify annotation statuses were rendered as failed — 4 variants, all failed annotation_statuses = session.scalars( select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") ).all() @@ -952,7 +1296,7 @@ async def test_submit_score_set_mappings_to_car_partial_failure( dummy_variant_creation_job_run, dummy_variant_mapping_job_run, ): - """Test that partial CAR failures result in SUCCEEDED status with per-variant failure annotations committed.""" + """All 4 variants share 1 allele; CAR returns 1 success → 1 registered allele, 0 failed, job SUCCEEDED.""" # Create mappings in the score set await create_mappings_in_score_set( session, @@ -964,14 +1308,14 @@ async def test_submit_score_set_mappings_to_car_partial_failure( dummy_variant_mapping_job_run, ) - # Return a CAR response that only matches the first variant's HGVS - mapped_variants = session.scalars(select(MappedVariant)).all() - first_hgvs = get_hgvs_from_post_mapped(mapped_variants[0].post_mapped) + alleles = session.scalars(select(Allele)).all() + assert len(alleles) == 1 + registered_alleles_mock = [ { - "@id": f"CA{mapped_variants[0].id}", + "@id": f"CA{alleles[0].id}", "type": "nucleotide", - "genomicAlleles": [{"hgvs": first_hgvs}], + "genomicAlleles": [{"hgvs": get_hgvs_from_post_mapped(alleles[0].post_mapped)}], } ] @@ -991,31 +1335,22 @@ async def test_submit_score_set_mappings_to_car_partial_failure( mock_send_slack_error.assert_not_called() assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - assert result.data["matched_count"] == 1 - assert result.data["failed_count"] == 3 + assert result.data["registered_allele_count"] == 1 + assert result.data["failed_allele_count"] == 0 - # Verify the successfully matched variant got a CAID - variants_with_caid = session.scalars( - select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None)) - ).all() - assert len(variants_with_caid) == 1 - assert variants_with_caid[0].clingen_allele_id == f"CA{mapped_variants[0].id}" + # 1 allele got a CAID + alleles_with_caid = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles_with_caid) == 1 + assert alleles_with_caid[0].clingen_allele_id == f"CA{alleles[0].id}" - # Verify annotation statuses: 1 success, 3 failed + # All 4 variant annotations succeeded success_annotations = session.scalars( select(VariantAnnotationStatus).where( VariantAnnotationStatus.annotation_type == "clingen_allele_id", VariantAnnotationStatus.status == "success", ) ).all() - failed_annotations = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.annotation_type == "clingen_allele_id", - VariantAnnotationStatus.status == "failed", - ) - ).all() - assert len(success_annotations) == 1 - assert len(failed_annotations) == 3 + assert len(success_annotations) == 4 # Verify the job status is updated in the database session.refresh(submit_score_set_mappings_to_car_sample_job_run) @@ -1046,22 +1381,17 @@ async def test_submit_score_set_mappings_to_car_car_error_details_stored_in_anno dummy_variant_mapping_job_run, ) - # Return a CAR response where: first variant succeeds, second has explicit CAR error, rest are silent failures - mapped_variants = session.scalars(select(MappedVariant)).all() - first_hgvs = get_hgvs_from_post_mapped(mapped_variants[0].post_mapped) - second_hgvs = get_hgvs_from_post_mapped(mapped_variants[1].post_mapped) + # All 4 variants share 1 allele. CAR returns an explicit error for that 1 HGVS. + alleles = session.scalars(select(Allele)).all() + assert len(alleles) == 1 + allele_hgvs = get_hgvs_from_post_mapped(alleles[0].post_mapped) registered_alleles_mock = [ - { - "@id": f"CA{mapped_variants[0].id}", - "type": "nucleotide", - "genomicAlleles": [{"hgvs": first_hgvs}], - }, { "errorType": "InvalidHGVS", - "hgvs": second_hgvs, + "hgvs": allele_hgvs, "message": "The HGVS string is invalid.", "description": "error", - "inputLine": second_hgvs, + "inputLine": allele_hgvs, "position": "0", }, ] @@ -1079,31 +1409,18 @@ async def test_submit_score_set_mappings_to_car_car_error_details_stored_in_anno standalone_worker_context, submit_score_set_mappings_to_car_sample_job_run.id ) - # Verify the variant whose HGVS returned an explicit CAR error has error details in annotation_metadata. - # Only 1 annotation should have EXTERNAL_SERVICE_REJECTED since only one CAR error was in the response. + # All 4 variant annotations should have EXTERNAL_SERVICE_REJECTED since the 1 shared allele was rejected car_rejected_annotations = session.scalars( select(VariantAnnotationStatus).where( VariantAnnotationStatus.annotation_type == "clingen_allele_id", VariantAnnotationStatus.failure_category == "external_service_rejected", ) ).all() - assert len(car_rejected_annotations) == 1 - rejected = car_rejected_annotations[0] - assert rejected.annotation_metadata["submitted_hgvs"] == second_hgvs - assert rejected.annotation_metadata["car_error_type"] == "InvalidHGVS" - assert rejected.annotation_metadata["car_error_message"] == "The HGVS string is invalid." - - # The remaining 2 failures (variants 3 and 4) got no CAR response — silent failures get EXTERNAL_API_ERROR. - silent_failure_annotations = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.annotation_type == "clingen_allele_id", - VariantAnnotationStatus.failure_category == "external_api_error", - ) - ).all() - assert len(silent_failure_annotations) == 2 - for ann in silent_failure_annotations: - assert ann.annotation_metadata["submitted_hgvs"] is not None - assert "car_error_type" not in ann.annotation_metadata + assert len(car_rejected_annotations) == 4 + for rejected in car_rejected_annotations: + assert rejected.annotation_metadata["submitted_hgvs"] == allele_hgvs + assert rejected.annotation_metadata["car_error_type"] == "InvalidHGVS" + assert rejected.annotation_metadata["car_error_message"] == "The HGVS string is invalid." async def test_submit_score_set_mappings_to_car_propagates_exception_to_decorator( self, @@ -1185,15 +1502,14 @@ async def test_submit_score_set_mappings_to_car_with_arq_context_independent( dummy_variant_mapping_job_run, ) - # Patch ClinGenAlleleRegistryService to return registered alleles - mapped_variants = session.scalars(select(MappedVariant)).all() + # All 4 variants share 1 allele; build 1 CAR response + alleles = session.scalars(select(Allele)).all() registered_alleles_mock = [ { - "@id": f"CA{mv.id}", + "@id": f"CA{alleles[0].id}", "type": "nucleotide", - "genomicAlleles": [{"hgvs": get_hgvs_from_post_mapped(mv.post_mapped)}], + "genomicAlleles": [{"hgvs": get_hgvs_from_post_mapped(alleles[0].post_mapped)}], } - for mv in mapped_variants ] with ( @@ -1214,13 +1530,12 @@ async def test_submit_score_set_mappings_to_car_with_arq_context_independent( session.refresh(submit_score_set_mappings_to_car_sample_job_run) assert submit_score_set_mappings_to_car_sample_job_run.status == JobStatus.SUCCEEDED - # Verify variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == len(mapped_variants) - for variant in variants: - assert variant.clingen_allele_id == f"CA{variant.id}" + # 1 allele received the CAID + alleles_with_caid = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles_with_caid) == 1 + assert alleles_with_caid[0].clingen_allele_id == f"CA{alleles[0].id}" - # Verify annotation statuses were rendered as success + # 4 per-variant annotations — all success annotation_statuses = session.scalars( select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") ).all() @@ -1255,15 +1570,14 @@ async def test_submit_score_set_mappings_to_car_with_arq_context_pipeline( dummy_variant_mapping_job_run, ) - # Patch ClinGenAlleleRegistryService to return registered alleles - mapped_variants = session.scalars(select(MappedVariant)).all() + # All 4 variants share 1 allele; build 1 CAR response + alleles = session.scalars(select(Allele)).all() registered_alleles_mock = [ { - "@id": f"CA{mv.id}", + "@id": f"CA{alleles[0].id}", "type": "nucleotide", - "genomicAlleles": [{"hgvs": get_hgvs_from_post_mapped(mv.post_mapped)}], + "genomicAlleles": [{"hgvs": get_hgvs_from_post_mapped(alleles[0].post_mapped)}], } - for mv in mapped_variants ] with ( @@ -1288,13 +1602,12 @@ async def test_submit_score_set_mappings_to_car_with_arq_context_pipeline( session.refresh(submit_score_set_mappings_to_car_sample_pipeline) assert submit_score_set_mappings_to_car_sample_pipeline.status == PipelineStatus.SUCCEEDED - # Verify variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == len(mapped_variants) - for variant in variants: - assert variant.clingen_allele_id == f"CA{variant.id}" + # 1 allele received the CAID + alleles_with_caid = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles_with_caid) == 1 + assert alleles_with_caid[0].clingen_allele_id == f"CA{alleles[0].id}" - # Verify annotation statuses were rendered as success + # 4 per-variant annotations — all success annotation_statuses = session.scalars( select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") ).all() @@ -1350,9 +1663,9 @@ async def test_submit_score_set_mappings_to_car_with_arq_context_exception_handl assert submit_score_set_mappings_to_car_sample_job_run.status == JobStatus.ERRORED assert submit_score_set_mappings_to_car_sample_job_run.error_message == "ClinGen service error" - # Verify no variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == 0 + # Verify no alleles have CAIDs assigned + alleles = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles) == 0 # Verify no annotation statuses were created annotation_statuses = session.scalars( @@ -1413,9 +1726,9 @@ async def test_submit_score_set_mappings_to_car_with_arq_context_exception_handl session.refresh(submit_score_set_mappings_to_car_sample_pipeline) assert submit_score_set_mappings_to_car_sample_pipeline.status == PipelineStatus.FAILED - # Verify no variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == 0 + # Verify no alleles have CAIDs assigned + alleles = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles) == 0 # Verify no annotation statuses were created annotation_statuses = session.scalars( diff --git a/tests/worker/jobs/external_services/test_clingen_cache.py b/tests/worker/jobs/external_services/test_clingen_cache.py index a55eb6b88..6d95b917a 100644 --- a/tests/worker/jobs/external_services/test_clingen_cache.py +++ b/tests/worker/jobs/external_services/test_clingen_cache.py @@ -4,11 +4,14 @@ pytest.importorskip("arq") +from datetime import date from unittest.mock import AsyncMock, patch from mavedb.lib.types.workflow import JobExecutionOutcome +from mavedb.models.allele import Allele from mavedb.models.enums.job_pipeline import JobStatus -from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.score_set import ScoreSet from mavedb.models.variant import Variant from mavedb.worker.jobs.external_services.clingen_cache import warm_clingen_cache @@ -17,6 +20,51 @@ pytestmark = pytest.mark.usefixtures("patch_db_session_ctxmgr") +def _make_allele_with_caid(session, score_set_id: int, urn_suffix: str, caid: str | None, vrs_digest: str) -> Allele: + """Create a Variant → MappingRecord → Allele chain and return the Allele. + + get_alleles_for_score_set joins: Allele ← MappingRecordAllele (current) ← MappingRecord (current) ← Variant. + Leaving valid_to=NULL on MappingRecord and MappingRecordAllele makes them current. + """ + variant = Variant( + urn=f"urn:variant:{urn_suffix}", + score_set_id=score_set_id, + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.flush() + + allele = Allele( + vrs_digest=vrs_digest, + level="genomic", + post_mapped={"type": "Allele", "location": {"sequenceReference": {"refgetAccession": vrs_digest}}}, + clingen_allele_id=caid, + ) + session.add(allele) + session.flush() + + mapping_record = MappingRecord( + variant_id=variant.id, + assay_level="genomic", + mapping_api_version="1.0.0", + mapped_date=date.today(), + ) + session.add(mapping_record) + session.flush() + + link = MappingRecordAllele( + mapping_record_id=mapping_record.id, + allele_id=allele.id, + is_authoritative=True, + ) + session.add(link) + session.flush() + + return allele + + @pytest.mark.unit @pytest.mark.asyncio class TestWarmClingenCacheUnit: @@ -49,26 +97,15 @@ async def test_warms_cache_for_variants_with_caids( """Job calls get_clingen_allele_data for each distinct allele ID.""" score_set = session.get(ScoreSet, sample_warm_clingen_cache_job_run.job_params["score_set_id"]) - # Create two variants with the same CAID — should only warm once (distinct) - for i, caid in enumerate(["CA111111", "CA222222", "CA111111"]): - variant = Variant( - urn=f"urn:variant:warm-test-{i}", - score_set_id=score_set.id, - hgvs_nt=f"NM_000000.1:c.{i + 1}A>G", - hgvs_pro=f"NP_000000.1:p.Met{i + 1}Val", - data={}, - ) - session.add(variant) - session.commit() - mapped_variant = MappedVariant( - variant_id=variant.id, - clingen_allele_id=caid, - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant) - session.commit() + # Three variants, two sharing the same CAID — should only warm 2 distinct IDs. + # Two separate allele rows are needed (different VRS digests) to get 2 distinct CAIDs. + _make_allele_with_caid(session, score_set.id, "warm-test-0", "CA111111", "digest-warm-0") + _make_allele_with_caid(session, score_set.id, "warm-test-1", "CA222222", "digest-warm-1") + # Third variant points to same allele as first (same CAID CA111111 via a fresh allele row + # with the same digest — but since get_alleles_for_score_set returns per-variant rows and + # caid dedup happens in the warmer, we can share the digest). + _make_allele_with_caid(session, score_set.id, "warm-test-2", "CA111111", "digest-warm-2") + session.commit() mock_get_allele_data = AsyncMock(return_value={"some": "data"}) @@ -100,24 +137,8 @@ async def test_skips_null_and_multi_variant_caids( caids = ["CA333333", None, "CA-MULTI-001,CA-MULTI-002"] for i, caid in enumerate(caids): - variant = Variant( - urn=f"urn:variant:warm-filter-{i}", - score_set_id=score_set.id, - hgvs_nt=f"NM_000000.1:c.{i + 10}A>G", - hgvs_pro=f"NP_000000.1:p.Met{i + 10}Val", - data={}, - ) - session.add(variant) - session.commit() - mapped_variant = MappedVariant( - variant_id=variant.id, - clingen_allele_id=caid, - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant) - session.commit() + _make_allele_with_caid(session, score_set.id, f"warm-filter-{i}", caid, f"digest-filter-{i}") + session.commit() mock_get_allele_data = AsyncMock(return_value={"some": "data"}) @@ -147,24 +168,8 @@ async def test_continues_on_individual_fetch_failure( score_set = session.get(ScoreSet, sample_warm_clingen_cache_job_run.job_params["score_set_id"]) for i, caid in enumerate(["CA444444", "CA555555"]): - variant = Variant( - urn=f"urn:variant:warm-fail-{i}", - score_set_id=score_set.id, - hgvs_nt=f"NM_000000.1:c.{i + 20}A>G", - hgvs_pro=f"NP_000000.1:p.Met{i + 20}Val", - data={}, - ) - session.add(variant) - session.commit() - mapped_variant = MappedVariant( - variant_id=variant.id, - clingen_allele_id=caid, - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant) - session.commit() + _make_allele_with_caid(session, score_set.id, f"warm-fail-{i}", caid, f"digest-fail-{i}") + session.commit() # First call raises, second succeeds mock_get_allele_data = AsyncMock( @@ -192,7 +197,7 @@ async def test_only_warms_current_mapped_variants( with_warm_clingen_cache_job, sample_warm_clingen_cache_job_run, ): - """Job only fetches allele IDs from current (not superseded) mapped variants.""" + """Job only fetches allele IDs from current (not superseded) mapping records.""" score_set = session.get(ScoreSet, sample_warm_clingen_cache_job_run.job_params["score_set_id"]) variant = Variant( @@ -203,25 +208,65 @@ async def test_only_warms_current_mapped_variants( data={}, ) session.add(variant) - session.commit() + session.flush() - # Non-current mapped variant should be ignored - old_mv = MappedVariant( - variant_id=variant.id, + # Superseded allele (MappingRecord with valid_to set — not current) + old_allele = Allele( + vrs_digest="digest-old-mv", + level="genomic", + post_mapped={"type": "Allele"}, clingen_allele_id="CA666666", - current=False, - mapped_date="2023-01-01T00:00:00Z", - mapping_api_version="0.9.0", ) - # Current mapped variant should be included - current_mv = MappedVariant( + session.add(old_allele) + session.flush() + + from datetime import datetime, timezone + + old_mapping_record = MappingRecord( variant_id=variant.id, + assay_level="genomic", + mapping_api_version="0.9.0", + mapped_date=date(2023, 1, 1), + ) + # Close this record so it is non-current (valid_to set) + old_mapping_record.valid_to = datetime(2024, 1, 1, tzinfo=timezone.utc) + session.add(old_mapping_record) + session.flush() + + old_link = MappingRecordAllele( + mapping_record_id=old_mapping_record.id, + allele_id=old_allele.id, + is_authoritative=True, + ) + old_link.valid_to = datetime(2024, 1, 1, tzinfo=timezone.utc) + session.add(old_link) + session.flush() + + # Current allele (MappingRecord with valid_to=NULL — current) + current_allele = Allele( + vrs_digest="digest-current-mv", + level="genomic", + post_mapped={"type": "Allele"}, clingen_allele_id="CA777777", - current=True, - mapped_date="2024-01-01T00:00:00Z", + ) + session.add(current_allele) + session.flush() + + current_mapping_record = MappingRecord( + variant_id=variant.id, + assay_level="genomic", mapping_api_version="1.0.0", + mapped_date=date.today(), + ) + session.add(current_mapping_record) + session.flush() + + current_link = MappingRecordAllele( + mapping_record_id=current_mapping_record.id, + allele_id=current_allele.id, + is_authoritative=True, ) - session.add_all([old_mv, current_mv]) + session.add(current_link) session.commit() mock_get_allele_data = AsyncMock(return_value={"some": "data"}) From ac4cfa920bde621a0468a4134f82b40d640e0ae8 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Sat, 6 Jun 2026 18:53:03 -0700 Subject: [PATCH 028/252] feat(variants): combine cis-phased members into one HGVS expression MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit get_hgvs_from_post_mapped can now join the members of a multi-variant block (Haplotype/CisPhasedBlock) into a single bracketed cis-phased expression via the new combine_cis flag, replacing the previous behavior of returning None for any multi-variant block. - combine_cis defaults off because some consumers cannot yet handle a bracketed expression — notably ClinGen submission, which has no single CAID for a multi-variant cis block (#764) - the CSV export fallback opts in, so g./p. HGVS columns are populated from cis-phased post-mapped output instead of left empty - drop the stale commented-out error branches and the resolved TODO --- .../worker/jobs/external_services/clingen.py | 38 ++++++++----------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/src/mavedb/worker/jobs/external_services/clingen.py b/src/mavedb/worker/jobs/external_services/clingen.py index 27aa1b962..42cdcb36a 100644 --- a/src/mavedb/worker/jobs/external_services/clingen.py +++ b/src/mavedb/worker/jobs/external_services/clingen.py @@ -121,14 +121,12 @@ async def submit_score_set_mappings_to_car(ctx: dict, job_id: int, job_manager: correlation_id = job.job_params["correlation_id"] # type: ignore force_reregister = bool(job.job_params.get("force_reregister", False)) # type: ignore[union-attr] - job_manager.save_to_context( - { - "application": "mavedb-worker", - "function": "submit_score_set_mappings_to_car", - "resource": score_set.urn, - "correlation_id": correlation_id, - } - ) + job_manager.save_to_context({ + "application": "mavedb-worker", + "function": "submit_score_set_mappings_to_car", + "resource": score_set.urn, + "correlation_id": correlation_id, + }) job_manager.update_progress(0, 100, "Starting CAR mapped resource submission.") logger.info(msg="Started CAR mapped resource submission", extra=job_manager.logging_context()) @@ -480,14 +478,12 @@ async def submit_score_set_mappings_to_ldh(ctx: dict, job_id: int, job_manager: correlation_id = job.job_params["correlation_id"] # type: ignore # Setup initial context and progress - job_manager.save_to_context( - { - "application": "mavedb-worker", - "function": "submit_score_set_mappings_to_ldh", - "resource": score_set.urn, - "correlation_id": correlation_id, - } - ) + job_manager.save_to_context({ + "application": "mavedb-worker", + "function": "submit_score_set_mappings_to_ldh", + "resource": score_set.urn, + "correlation_id": correlation_id, + }) job_manager.update_progress(0, 100, "Starting LDH mapped resource submission.") logger.info(msg="Started LDH mapped resource submission", extra=job_manager.logging_context()) @@ -560,12 +556,10 @@ async def submit_score_set_mappings_to_ldh(ctx: dict, job_id: int, job_manager: loop = asyncio.get_running_loop() submission_successes, submission_failures = await loop.run_in_executor(ctx["pool"], blocking) job_manager.update_progress(90, 100, "Finalizing LDH mapped resource submission.") - job_manager.save_to_context( - { - "ldh_submission_successes": len(submission_successes), - "ldh_submission_failures": len(submission_failures), - } - ) + job_manager.save_to_context({ + "ldh_submission_successes": len(submission_successes), + "ldh_submission_failures": len(submission_failures), + }) # TODO prior to finalizing: Verify typing of ClinGen submission responses. See https://reg.clinicalgenome.org/doc/AlleleRegistry_1.01.xx_api_v1.pdf annotation_manager = AnnotationStatusManager(job_manager.db, job_run_id=job_manager.job_id) From dba7bfebcaa877ac7954e5468cc2c3d77ef27e4d Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Mon, 22 Jun 2026 11:45:14 -0700 Subject: [PATCH 029/252] feat(gnomad): migrate gnomAD linkage to the Allele model with version-keyed refresh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace MappedVariant-based gnomAD linkage with valid-time GnomadAlleleLink rows keyed on the deduplicated Allele. Linking covers every current allele of a score set (authoritative and RT-derived), so protein/coding score sets — whose genomic allele is RT-derived — are no longer dropped; per-variant annotation status flows through the _annotate_gnomad choke point against authoritative links only (interim bandaid, the AnnotationEvent migration seam). Refresh / idempotency model: - One live link per allele (unique index on allele_id); a gnomAD version bump supersedes rather than accumulating one live link per version. - Supersede only on change — an unchanged re-run leaves the live link untouched, so the valid-time history records no spurious boundary. - Version-keyed skip avoids re-fetching alleles already current at the version; a force param bypasses the skip (re-ingestion / heal) without churning unchanged links. - Per-variant status is a per-run audit event: created / preexisting / skipped. Normalize CAIDs across the Athena join: the gnomAD Hail dump drops leading zeros (CA025094 -> CA25094), so exact-string matching silently missed zero-padded CAIDs. Extract group_alleles_for_annotation as the shared allele-grouping primitive for allele-subject annotation jobs (adopted by gnomAD; VEP/ClinVar to follow). Refs #742. Partially addresses #722 (CAID-completeness tracking there stays open). --- .../e5f7a9c1b3d4_add_gnomad_allele_links.py | 71 ++++ src/mavedb/lib/clingen/alleles.py | 56 +++- src/mavedb/lib/gnomad.py | 197 ++++++----- src/mavedb/models/__init__.py | 1 + src/mavedb/models/gnomad_allele_link.py | 60 ++++ src/mavedb/models/gnomad_variant.py | 13 +- .../worker/jobs/external_services/gnomad.py | 215 +++++++----- tests/lib/clingen/test_alleles.py | 69 ++++ tests/lib/test_gnomad.py | 309 +++++++++--------- tests/worker/jobs/conftest.py | 85 +++++ .../jobs/external_services/test_gnomad.py | 245 +++++++++++--- 11 files changed, 949 insertions(+), 372 deletions(-) create mode 100644 alembic/versions/e5f7a9c1b3d4_add_gnomad_allele_links.py create mode 100644 src/mavedb/models/gnomad_allele_link.py create mode 100644 tests/lib/clingen/test_alleles.py diff --git a/alembic/versions/e5f7a9c1b3d4_add_gnomad_allele_links.py b/alembic/versions/e5f7a9c1b3d4_add_gnomad_allele_links.py new file mode 100644 index 000000000..54f3e3d58 --- /dev/null +++ b/alembic/versions/e5f7a9c1b3d4_add_gnomad_allele_links.py @@ -0,0 +1,71 @@ +"""add gnomad_allele_links table + +Revision ID: e5f7a9c1b3d4 +Revises: d4e6f8a0b2c3 +Create Date: 2026-06-18 + +New valid-time link table connecting deduplicated alleles to gnomAD variants, replacing the frozen +gnomad_variants_mapped_variants association for new-model writes (Step 1 of the annotation +infrastructure migration, docs/design/annotation-infrastructure-migration.md). A link is live while +valid_to is NULL; the partial unique index enforces a single live link per (allele, gnomad variant) +pair. The existing gnomad_variants_mapped_variants table is left untouched (frozen serving). +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "e5f7a9c1b3d4" +down_revision = "d4e6f8a0b2c3" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "gnomad_allele_links", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("allele_id", sa.Integer(), nullable=False), + sa.Column("gnomad_variant_id", sa.Integer(), nullable=False), + sa.Column("valid_from", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + sa.Column("valid_to", sa.DateTime(timezone=True), nullable=True), + sa.ForeignKeyConstraint( + ["allele_id"], + ["alleles.id"], + name="fk_gnomad_allele_links_allele_id", + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["gnomad_variant_id"], + ["gnomad_variants.id"], + name="fk_gnomad_allele_links_gnomad_variant_id", + ondelete="RESTRICT", + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + "ix_gnomad_allele_links_allele_id", + "gnomad_allele_links", + ["allele_id"], + ) + op.create_index( + "ix_gnomad_allele_links_gnomad_variant_id", + "gnomad_allele_links", + ["gnomad_variant_id"], + ) + # One live link per allele: gnomAD frequency is a single current value, so a version bump + # supersedes the prior link rather than accumulating one live link per version. + op.create_index( + "uq_gnomad_allele_links_live", + "gnomad_allele_links", + ["allele_id"], + unique=True, + postgresql_where=sa.text("valid_to IS NULL"), + ) + + +def downgrade() -> None: + op.drop_index("uq_gnomad_allele_links_live", table_name="gnomad_allele_links") + op.drop_index("ix_gnomad_allele_links_gnomad_variant_id", table_name="gnomad_allele_links") + op.drop_index("ix_gnomad_allele_links_allele_id", table_name="gnomad_allele_links") + op.drop_table("gnomad_allele_links") diff --git a/src/mavedb/lib/clingen/alleles.py b/src/mavedb/lib/clingen/alleles.py index 527572233..79baf47ff 100644 --- a/src/mavedb/lib/clingen/alleles.py +++ b/src/mavedb/lib/clingen/alleles.py @@ -5,7 +5,8 @@ score set. A single definition here prevents the two jobs from drifting apart. """ -from typing import NamedTuple +from dataclasses import dataclass, field +from typing import Callable, Generic, Iterable, NamedTuple, Optional, TypeVar from sqlalchemy import select from sqlalchemy.orm import Session @@ -15,6 +16,8 @@ from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.variant import Variant +P = TypeVar("P") + class ScoreSetAlleleRow(NamedTuple): """One (allele, variant) link for a score set. An allele shared by multiple variants @@ -59,3 +62,54 @@ def get_alleles_for_score_set(db: Session, score_set_id: int) -> list[ScoreSetAl ).all() return [ScoreSetAlleleRow(r.id, r.post_mapped, r.clingen_allele_id, r.variant_id, r.is_authoritative) for r in rows] + + +@dataclass +class AlleleAnnotationGroup(Generic[P]): + """One allele's annotation work-unit: a job-specific ``payload`` plus the variants that receive a + per-variant annotation-status (VAS) row for it. + + ``authoritative_variant_ids`` is the interim **bandaid seam** (see + docs/design/allele-annotation-status.md). An annotation is an allele-level fact, but VAS is + per-variant; fanning it to every variant the allele backs would write multiple ``current`` rows + for one variant. So status is fanned only to the variants for which this allele is the + *authoritative* measurement. At the AnnotationEvent migration the per-variant fan-out goes away + and this list is no longer needed. The allele is still grouped (and linked/annotated at the + allele level) regardless — a purely RT-derived allele simply has an empty list here. + """ + + payload: P + authoritative_variant_ids: list[int] = field(default_factory=list) + + +def group_alleles_for_annotation( + rows: Iterable[ScoreSetAlleleRow], + payload: Callable[[ScoreSetAlleleRow], Optional[P]], +) -> dict[int, AlleleAnnotationGroup[P]]: + """Collapse the per-(allele, variant) rows from :func:`get_alleles_for_score_set` into one + work-unit per allele, keyed by ``allele_id``. + + The same allele recurs once per variant that links it (and can be authoritative for one variant + while RT-derived for another). This groups those rows so every annotation job shares one shape: + one entry per allele, carrying the authoritative-variant fan-out set for the bandaid. + + ``payload`` builds the job-specific payload from the first row seen for an allele — the CAID for + gnomAD/ClinVar, the HGVS for VEP, etc. Returning ``None`` skips the allele entirely (e.g. it + carries no CAID), replacing each job's ad-hoc ``if row.x is None: continue``. ``payload`` must be + a pure function of allele-level fields so its result is stable across an allele's rows. + + Grouping on ``allele_id`` rather than ``vrs_digest`` is intentional: the two are 1:1 (content + addressing makes ``vrs_digest`` unique), so the groups are identical either way, and ``allele_id`` + is the permanent, never-reused surrogate. If annotation storage later keys on the digest, carry + it on the row and store against it — the grouping contract here does not change. + """ + groups: dict[int, AlleleAnnotationGroup[P]] = {} + for row in rows: + if row.allele_id not in groups: + built = payload(row) + if built is None: + continue + groups[row.allele_id] = AlleleAnnotationGroup(payload=built) + if row.is_authoritative: + groups[row.allele_id].authoritative_variant_ids.append(row.variant_id) + return groups diff --git a/src/mavedb/lib/gnomad.py b/src/mavedb/lib/gnomad.py index 9bfa0fec9..8456ec35d 100644 --- a/src/mavedb/lib/gnomad.py +++ b/src/mavedb/lib/gnomad.py @@ -3,19 +3,24 @@ import re from typing import Any, Sequence, Union -from sqlalchemy import Connection, Row, select, text +from sqlalchemy import Connection, Row, func, select, text from sqlalchemy.orm import Session -from mavedb.lib.annotation_status_manager import AnnotationStatusManager from mavedb.lib.logging.context import logging_context, save_to_logging_context from mavedb.lib.utils import batched -from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationStatus +from mavedb.models.allele import Allele +from mavedb.models.gnomad_allele_link import GnomadAlleleLink from mavedb.models.gnomad_variant import GnomADVariant -from mavedb.models.mapped_variant import MappedVariant GNOMAD_DB_NAME = "gnomAD" -GNOMAD_DATA_VERSION = os.getenv("GNOMAD_DATA_VERSION", "v4.1") # e.g., "v4.1" +GNOMAD_DATA_VERSION = os.getenv("GNOMAD_DATA_VERSION", "v4.1") +_CAID_LEADING_ZERO_RE = r"^(CA)0+([0-9])" +""" +Strip leading zeros from a CAID's numeric portion, keeping at least one digit. +Kept byte-for-byte in sync with the SQL form used to normalize Allele.clingen_allele_id +in link_gnomad_variants_to_alleles. +""" + logger = logging.getLogger(__name__) @@ -46,6 +51,18 @@ def gnomad_table_name() -> str: return table_name +def normalize_caid(caid: str) -> str: + """Normalize a ClinGen CAID by stripping leading zeros from its numeric portion. + + The gnomAD Hail/Athena dump drops leading zeros from CAIDs — MaveDB's ``CA025094`` is recorded as + ``CA25094`` — so an exact-string join silently misses every zero-padded CAID (issue #722). Both + sides of the join are normalized to the unpadded form to repair the match. ``CA025094`` and + ``CA25094`` denote the same ClinGen allele, so this can never collide distinct alleles. A value + that is not a recognizable CAID (no ``CA`` prefix + digits) is returned unchanged. + """ + return re.sub(_CAID_LEADING_ZERO_RE, r"\1\2", caid) + + def allele_list_from_list_like_string(alleles_string: str) -> list[str]: """ Convert a list-like string representation of alleles into a Python list. @@ -94,8 +111,9 @@ def gnomad_variant_data_for_caids( Raises: sqlalchemy.exc.SQLAlchemyError: If there is an error executing the query. """ + # Normalize to the unpadded form the dump stores so the IN-list matches zero-padded CAIDs (see issue #722). chunked_caids = batched(caids, 16250) - caid_strs = [",".join(f"'{caid}'" for caid in chunk) for chunk in chunked_caids] + caid_strs = [",".join(f"'{normalize_caid(caid)}'" for caid in chunk) for chunk in chunked_caids] save_to_logging_context({"num_caids": len(caids), "num_chunks": len(caid_strs)}) result_rows: list[Row[Any]] = [] @@ -132,31 +150,40 @@ def gnomad_variant_data_for_caids( return result_rows -def link_gnomad_variants_to_mapped_variants( - db: Session, gnomad_variant_data: Sequence[Row[Any]], only_current: bool = True -) -> int: - """ - Links gnomAD variants to mapped variants in the database based on CAIDs. Note that this function does - not commit this data to the database; it only prepares the relationships. +def link_gnomad_variants_to_alleles(db: Session, gnomad_variant_data: Sequence[Row[Any]]) -> set[int]: + """Link gnomAD variants to deduplicated alleles by CAID, superseding only on change. - Args: - caids (list[str]): A list of CAIDs to link with gnomAD variants. + Every ``Allele`` carrying the row's ``clingen_allele_id`` (populated by CAR) is linked through a + valid-time :class:`GnomadAlleleLink`, so one gnomAD variant fans out to every allele sharing the + CAID (cross-score-set dedup included). Each allele holds at most one live link, superseded **only + on change**: a live link already pointing to the resolved variant is left untouched (an unchanged + re-run writes no spurious valid-time boundary); a new/different/older-version target retires it and + inserts a successor, keyed on ``allele_id`` so a version bump replaces rather than accumulates. The + guard is load-bearing despite the job's upstream skip — shared CAIDs and ``force`` runs still reach + it. A current-version link to a *different* identifier (a CAID re-resolved within one release) is + logged and superseded newest-wins, not raised. + + Does not commit. Returns the allele ids whose link was created or superseded this run. """ save_to_logging_context({"num_gnomad_variant_rows": len(gnomad_variant_data)}) - save_to_logging_context({"only_current": only_current}) - logger.debug(msg="Linking gnomAD variants to mapped variants", extra=logging_context()) + logger.debug(msg="Linking gnomAD variants to alleles", extra=logging_context()) - linked_gnomad_variants = 0 - annotation_manager = AnnotationStatusManager(db) + changed_allele_ids: set[int] = set() for index, row in enumerate(gnomad_variant_data, start=1): logger.info( msg=f"Processing gnomAD variant row {index}/{len(gnomad_variant_data)}: {row.caid}", extra=logging_context() ) - mapped_variants_with_caids_query = select(MappedVariant).where(MappedVariant.clingen_allele_id == row.caid) - if only_current: - mapped_variants_with_caids_query = mapped_variants_with_caids_query.where(MappedVariant.current.is_(True)) - mapped_variants_with_caids = db.scalars(mapped_variants_with_caids_query).all() + # Match on the unpadded CAID: the dump's caid is already stripped, while the stored CAID may + # be zero-padded, so normalize both sides (issue #722). regexp_replace mirrors normalize_caid. + alleles_with_caid = db.scalars( + select(Allele).where( + func.regexp_replace(Allele.clingen_allele_id, _CAID_LEADING_ZERO_RE, r"\1\2") + == normalize_caid(row.caid) + ) + ).all() + if not alleles_with_caid: + continue gnomad_identifier_for_variant = gnomad_identifier( row.__getattribute__("locus.contig"), @@ -172,80 +199,86 @@ def link_gnomad_variants_to_mapped_variants( if faf95_max is not None: faf95_max = float(faf95_max) - for mapped_variant in mapped_variants_with_caids: - # Remove any existing gnomAD variants for this mapped variant that match the current gnomAD data version to avoid data duplication. - # There should only be one gnomAD variant per mapped variant per gnomAD data version, since each gnomAD variant can only match to one - # CAID. - for linked_gnomad_variant in mapped_variant.gnomad_variants: - if linked_gnomad_variant.db_version == GNOMAD_DATA_VERSION: - logger.debug( - msg=f"Removing existing gnomAD variant {linked_gnomad_variant.db_identifier} from mapped variant {mapped_variant.id} ({mapped_variant.clingen_allele_id})", - extra=logging_context(), - ) - mapped_variant.gnomad_variants.remove(linked_gnomad_variant) - - existing_gnomad_variant = db.scalar( - select(GnomADVariant).where( - GnomADVariant.db_name == "gnomAD", - GnomADVariant.db_identifier == gnomad_identifier_for_variant, - GnomADVariant.db_version == GNOMAD_DATA_VERSION, - ) + # One gnomAD variant per (identifier, version): get-or-create so repeated CAIDs and re-runs + # reuse the same row. Flush so a freshly created variant has an id for the link below. + gnomad_variant = db.scalar( + select(GnomADVariant).where( + GnomADVariant.db_name == GNOMAD_DB_NAME, + GnomADVariant.db_identifier == gnomad_identifier_for_variant, + GnomADVariant.db_version == GNOMAD_DATA_VERSION, + ) + ) + if gnomad_variant is None: + logger.debug( + msg=f"Creating new gnomAD variant for identifier {gnomad_identifier_for_variant}", + extra=logging_context(), + ) + gnomad_variant = GnomADVariant( + db_name=GNOMAD_DB_NAME, + db_identifier=gnomad_identifier_for_variant, + db_version=GNOMAD_DATA_VERSION, + allele_count=allele_count, + allele_number=allele_number, + allele_frequency=allele_frequency, # type: ignore + faf95_max_ancestry=faf95_max_ancestry, + faf95_max=faf95_max, # type: ignore + ) + db.add(gnomad_variant) + db.flush() + else: + logger.debug( + msg=f"Found existing gnomAD variant for identifier {gnomad_identifier_for_variant}", + extra=logging_context(), ) - if existing_gnomad_variant is None: - logger.debug( - msg=f"Creating new gnomAD variant for identifier {gnomad_identifier_for_variant}", - extra=logging_context(), + for allele in alleles_with_caid: + live_link = db.scalar( + select(GnomadAlleleLink).where( + GnomadAlleleLink.allele_id == allele.id, + GnomadAlleleLink.current, ) - gnomad_variant = GnomADVariant( - db_name=GNOMAD_DB_NAME, - db_identifier=gnomad_identifier_for_variant, - db_version=GNOMAD_DATA_VERSION, - allele_count=allele_count, - allele_number=allele_number, - allele_frequency=allele_frequency, # type: ignore - faf95_max_ancestry=faf95_max_ancestry, - faf95_max=faf95_max, # type: ignore - ) - else: - logger.debug( - msg=f"Found existing gnomAD variant for identifier {gnomad_identifier_for_variant}", + ) + + # No change: live link already points here — leave it untouched (no spurious boundary). + if live_link is not None and live_link.gnomad_variant_id == gnomad_variant.id: + continue + + if ( + live_link is not None + and live_link.gnomad_variant.db_version == GNOMAD_DATA_VERSION + and live_link.gnomad_variant.db_identifier != gnomad_identifier_for_variant + ): + logger.warning( + msg=( + f"CAID {allele.clingen_allele_id} for allele {allele.id} resolved to " + f"{gnomad_identifier_for_variant} at version {GNOMAD_DATA_VERSION}, but a live link " + f"already points to {live_link.gnomad_variant.db_identifier} at the same version. " + "Superseding (newest wins); investigate the gnomAD source for a re-resolved CAID." + ), extra=logging_context(), ) - gnomad_variant = existing_gnomad_variant - if gnomad_variant not in mapped_variant.gnomad_variants: - mapped_variant.gnomad_variants.append(gnomad_variant) - linked_gnomad_variants += 1 - - db.add(gnomad_variant) - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.GNOMAD_ALLELE_FREQUENCY, - version=GNOMAD_DATA_VERSION, - status=AnnotationStatus.SUCCESS, - annotation_data={ - "annotation_metadata": { - "gnomad_db_identifier": gnomad_variant.db_identifier, - } - }, - current=True, + # Change: retire any live link for the allele, insert the successor (allele-keyed, so a + # version bump replaces rather than accumulates). + GnomadAlleleLink.supersede_live_where( + db, + [GnomadAlleleLink(allele_id=allele.id, gnomad_variant_id=gnomad_variant.id)], + GnomadAlleleLink.allele_id == allele.id, ) + changed_allele_ids.add(allele.id) logger.debug( - msg=f"Linked gnomAD variant {gnomad_variant.db_identifier} to mapped variant {mapped_variant.id} ({mapped_variant.clingen_allele_id})", + msg=f"Linked gnomAD variant {gnomad_variant.db_identifier} to allele {allele.id} ({allele.clingen_allele_id})", extra=logging_context(), ) logger.info( - f"Linked {len(mapped_variants_with_caids)} mapped variants with CAID {row.caid} to gnomAD variant {gnomad_identifier_for_variant}. ({index}/{len(gnomad_variant_data)})" + f"Processed {len(alleles_with_caid)} alleles with CAID {row.caid} for gnomAD variant {gnomad_identifier_for_variant}. ({index}/{len(gnomad_variant_data)})" ) - annotation_manager.flush() - - save_to_logging_context({"linked_gnomad_variants": linked_gnomad_variants}) + save_to_logging_context({"changed_allele_count": len(changed_allele_ids)}) logger.info( - msg=f"Linked a total of {linked_gnomad_variants} gnomAD variants to mapped variants.", + msg=f"Created or superseded {len(changed_allele_ids)} allele links this run.", extra=logging_context(), ) - return linked_gnomad_variants + return changed_allele_ids diff --git a/src/mavedb/models/__init__.py b/src/mavedb/models/__init__.py index 35c1ba0a5..5284da11b 100644 --- a/src/mavedb/models/__init__.py +++ b/src/mavedb/models/__init__.py @@ -11,6 +11,7 @@ "experiment", "experiment_set", "genome_identifier", + "gnomad_allele_link", "gnomad_variant", "job_dependency", "job_run", diff --git a/src/mavedb/models/gnomad_allele_link.py b/src/mavedb/models/gnomad_allele_link.py new file mode 100644 index 000000000..75294ba66 --- /dev/null +++ b/src/mavedb/models/gnomad_allele_link.py @@ -0,0 +1,60 @@ +from typing import TYPE_CHECKING + +from sqlalchemy import Column, ForeignKey, Index, Integer, text +from sqlalchemy.orm import Mapped, relationship + +from mavedb.db.base import Base +from mavedb.db.mixins import ValidTime + +if TYPE_CHECKING: + from .allele import Allele + from .gnomad_variant import GnomADVariant + + +class GnomadAlleleLink(ValidTime, Base): + """Valid-time link between an :class:`Allele` and a gnomAD variant. + + Replaces the frozen ``gnomad_variants_mapped_variants`` association table for new-model writes. + A link is live while ``valid_to`` is NULL; a gnomAD version bump retires the live row and inserts + a successor rather than deleting, so prior-version frequency links remain queryable point-in-time. + The partial unique index enforces **a single live link per allele** — gnomAD frequency is one + current value, so a new version supersedes the old (unlike ClinVar, which keeps one live link per + release). This matches the VEP consequence shape, not the ClinVar control shape. + """ + + __tablename__ = "gnomad_allele_links" + + id: Mapped[int] = Column(Integer, primary_key=True) + allele_id: Mapped[int] = Column( + Integer, + ForeignKey("alleles.id", ondelete="RESTRICT"), + nullable=False, + ) + gnomad_variant_id: Mapped[int] = Column( + Integer, + ForeignKey("gnomad_variants.id", ondelete="RESTRICT"), + nullable=False, + ) + + allele: Mapped["Allele"] = relationship("Allele") + gnomad_variant: Mapped["GnomADVariant"] = relationship("GnomADVariant", back_populates="allele_links") + + __table_args__ = ( + Index( + "ix_gnomad_allele_links_allele_id", + "allele_id", + ), + Index( + "ix_gnomad_allele_links_gnomad_variant_id", + "gnomad_variant_id", + ), + # At most one live link per allele. A version bump supersedes (retires the old, inserts the + # new) rather than accumulating per-version live links; superseded rows stay for point-in-time + # queries. Only the live row participates in this constraint. + Index( + "uq_gnomad_allele_links_live", + "allele_id", + unique=True, + postgresql_where=text("valid_to IS NULL"), + ), + ) diff --git a/src/mavedb/models/gnomad_variant.py b/src/mavedb/models/gnomad_variant.py index 0f4a00a50..bfc7c8026 100644 --- a/src/mavedb/models/gnomad_variant.py +++ b/src/mavedb/models/gnomad_variant.py @@ -1,20 +1,21 @@ from datetime import date from typing import TYPE_CHECKING -from sqlalchemy import Column, Date, Integer, Float, String +from sqlalchemy import Column, Date, Float, Integer, String from sqlalchemy.orm import Mapped, relationship from mavedb.db.base import Base from mavedb.models.gnomad_variant_mapped_variant import gnomad_variants_mapped_variants_association_table if TYPE_CHECKING: + from mavedb.models.gnomad_allele_link import GnomadAlleleLink from mavedb.models.mapped_variant import MappedVariant class GnomADVariant(Base): __tablename__ = "gnomad_variants" - id = Column(Integer, primary_key=True) + id: Mapped[int] = Column(Integer, primary_key=True) db_name = Column(String, nullable=False) db_identifier = Column(String, nullable=False, index=True) @@ -30,8 +31,16 @@ class GnomADVariant(Base): creation_date = Column(Date, nullable=False, default=date.today) modification_date = Column(Date, nullable=False, default=date.today, onupdate=date.today) + # Frozen association to the old MappedVariant model — read by serving for existing data, never + # written for new score sets (which link through ``allele_links`` instead). mapped_variants: Mapped[list["MappedVariant"]] = relationship( "MappedVariant", secondary=gnomad_variants_mapped_variants_association_table, back_populates="gnomad_variants", ) + + # Valid-time links to deduplicated alleles (new-model writes). + allele_links: Mapped[list["GnomadAlleleLink"]] = relationship( + "GnomadAlleleLink", + back_populates="gnomad_variant", + ) diff --git a/src/mavedb/worker/jobs/external_services/gnomad.py b/src/mavedb/worker/jobs/external_services/gnomad.py index 969c3a20e..8daa5c9af 100644 --- a/src/mavedb/worker/jobs/external_services/gnomad.py +++ b/src/mavedb/worker/jobs/external_services/gnomad.py @@ -1,29 +1,29 @@ """gnomAD variant linking jobs for population frequency annotation. -This module handles linking of mapped variants to gnomAD (Genome Aggregation Database) +This module handles linking of deduplicated alleles to gnomAD (Genome Aggregation Database) variants to provide population frequency and other genomic context information. This enrichment helps researchers understand the clinical significance and rarity of variants in their datasets. """ import logging -from typing import Sequence from sqlalchemy import select from mavedb.db import athena from mavedb.lib.annotation_status_manager import AnnotationStatusManager +from mavedb.lib.clingen.alleles import get_alleles_for_score_set, group_alleles_for_annotation from mavedb.lib.gnomad import ( GNOMAD_DATA_VERSION, gnomad_variant_data_for_caids, - link_gnomad_variants_to_mapped_variants, + link_gnomad_variants_to_alleles, ) from mavedb.lib.types.workflow import JobExecutionOutcome from mavedb.models.enums.annotation_type import AnnotationType from mavedb.models.enums.job_pipeline import AnnotationFailureCategory, AnnotationStatus -from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.gnomad_allele_link import GnomadAlleleLink +from mavedb.models.gnomad_variant import GnomADVariant from mavedb.models.score_set import ScoreSet -from mavedb.models.variant import Variant from mavedb.worker.jobs.utils.setup import validate_job_params from mavedb.worker.lib.decorators.pipeline_management import with_pipeline_management from mavedb.worker.lib.managers.job_manager import JobManager @@ -31,17 +31,51 @@ logger = logging.getLogger(__name__) +def _annotate_gnomad( + annotation_manager: AnnotationStatusManager, + variant_ids: list[int], + status: AnnotationStatus, + *, + failure_category: AnnotationFailureCategory | None = None, + error_message: str | None = None, + metadata: dict | None = None, +) -> None: + """Fan a GNOMAD_ALLELE_FREQUENCY annotation out to every variant served by an allele. + + AAS migration seam: the single choke point for gnomAD's per-variant VAS writes. At migration it + becomes an allele-keyed event writer; the per-variant fan-out goes away, and the variant + association narrows to provenance (which variant's mapping drove the linkage). See + docs/design/allele-annotation-status.md. + """ + annotation_data: dict = {"annotation_metadata": metadata or {}} + if error_message is not None: + annotation_data["error_message"] = error_message + + for variant_id in variant_ids: + annotation_manager.add_annotation( + variant_id=variant_id, + annotation_type=AnnotationType.GNOMAD_ALLELE_FREQUENCY, + version=GNOMAD_DATA_VERSION, + status=status, + failure_category=failure_category, + annotation_data=annotation_data, + current=True, + ) + + @with_pipeline_management async def link_gnomad_variants(ctx: dict, job_id: int, job_manager: JobManager) -> JobExecutionOutcome: """ - Link mapped variants to gnomAD variants based on ClinGen Allele IDs (CAIDs). - This job fetches mapped variants associated with a given score set that have CAIDs, - retrieves corresponding gnomAD variant data, and establishes links between them - in the database. + Link deduplicated alleles to gnomAD variants based on ClinGen Allele IDs (CAIDs). + This job fetches the current authoritative alleles of a score set that carry CAIDs, + retrieves corresponding gnomAD variant data, and establishes valid-time links between them. Job Parameters: - - score_set_id (int): The ID of the ScoreSet containing mapped variants to process. + - score_set_id (int): The ID of the ScoreSet whose alleles to process. - correlation_id (str): Correlation ID for tracing requests across services. + - force (bool, optional): Bypass the version-keyed skip and re-fetch every CAID-bearing + allele. The linker still supersedes only on change, so a forced re-run of unchanged data + writes no new links. Use for re-ingestion or to heal suspected link corruption. Args: ctx (dict): The job context dictionary. @@ -49,10 +83,10 @@ async def link_gnomad_variants(ctx: dict, job_id: int, job_manager: JobManager) job_manager (JobManager): The job manager instance for database and logging operations. Side Effects: - - Updates MappedVariant records to link to gnomAD variants. + - Creates GnomadAlleleLink rows linking alleles to gnomAD variants. Returns: - dict: Result indicating success and any exception details + JobExecutionOutcome: outcome with per-allele created/preexisting/skipped counts. """ # Get the job definition we are working on job = job_manager.get_job() @@ -63,6 +97,7 @@ async def link_gnomad_variants(ctx: dict, job_id: int, job_manager: JobManager) # Fetch required resources based on param inputs. Safely ignore mypy warnings here, as they were checked above. score_set = job_manager.db.scalars(select(ScoreSet).where(ScoreSet.id == job.job_params["score_set_id"])).one() # type: ignore correlation_id = job.job_params["correlation_id"] # type: ignore + force = bool(job.job_params.get("force", False)) # type: ignore[union-attr] # Setup initial context and progress job_manager.save_to_context( @@ -76,87 +111,117 @@ async def link_gnomad_variants(ctx: dict, job_id: int, job_manager: JobManager) job_manager.update_progress(0, 100, "Starting gnomAD mapped resource linkage.") logger.info(msg="Started gnomAD mapped resource linkage", extra=job_manager.logging_context()) - # We filter out mapped variants that do not have a CAID, so this query is typed # as a Sequence[str]. Ignore MyPy's type checking here. - variant_caids: Sequence[str] = job_manager.db.scalars( - select(MappedVariant.clingen_allele_id) - .join(Variant) - .join(ScoreSet) - .where( - ScoreSet.urn == score_set.urn, - MappedVariant.current.is_(True), - MappedVariant.clingen_allele_id.is_not(None), - ) - ).all() # type: ignore + # One work-unit per allele (payload = CAID; alleles without one are skipped). Covers ALL the score + # set's alleles — authoritative and RT-derived — since the genomic allele gnomAD knows is often the + # RT-derived one; VAS still fans only to authoritative_variant_ids (the bandaid seam). + allele_data = group_alleles_for_annotation( + get_alleles_for_score_set(job_manager.db, score_set.id), + payload=lambda row: row.clingen_allele_id, + ) - num_variant_caids = len(variant_caids) - job_manager.save_to_context({"num_variants_to_link_gnomad": num_variant_caids}) + num_alleles_with_caids = len(allele_data) + job_manager.save_to_context({"num_alleles_to_link_gnomad": num_alleles_with_caids}) - if not variant_caids: + if not allele_data: logger.warning( - msg="No current mapped variants with CAIDs were found for this score set. Skipping gnomAD linkage (nothing to do).", + msg="No current alleles with CAIDs were found for this score set. Skipping gnomAD linkage (nothing to do).", extra=job_manager.logging_context(), ) job_manager.db.flush() - return JobExecutionOutcome.succeeded(data={"linked_count": 0, "skipped_count": 0}) + return JobExecutionOutcome.succeeded( + data={"created_allele_count": 0, "preexisting_allele_count": 0, "skipped_allele_count": 0} + ) - job_manager.update_progress(10, 100, f"Found {num_variant_caids} variants with CAIDs to link to gnomAD variants.") - logger.info( - msg="Found current mapped variants with CAIDs for this score set. Attempting to link them to gnomAD variants.", - extra=job_manager.logging_context(), + job_manager.update_progress( + 10, 100, f"Found {num_alleles_with_caids} alleles with CAIDs to link to gnomAD variants." ) - # Fetch gnomAD variant data for the CAIDs - with athena.engine.connect() as athena_session: - logger.debug("Fetching gnomAD variants from Athena.") - gnomad_variant_data = gnomad_variant_data_for_caids(athena_session, variant_caids) - - num_gnomad_variants_with_caid_match = len(gnomad_variant_data) + def alleles_linked_at_current_version(allele_ids: set[int]) -> set[int]: + """Allele ids (within the given set) holding a live gnomAD link at the current gnomAD version.""" + if not allele_ids: + return set() + return set( + job_manager.db.scalars( + select(GnomadAlleleLink.allele_id) + .join(GnomADVariant, GnomADVariant.id == GnomadAlleleLink.gnomad_variant_id) + .where(GnomadAlleleLink.allele_id.in_(allele_ids)) + .where(GnomadAlleleLink.current) + .where(GnomADVariant.db_version == GNOMAD_DATA_VERSION) + ).all() + ) - # NOTE: Proceed intentionally with linking even if no matches were found, to record skipped annotations. + # Cost: skip alleles already linked at the current version (they can't change). force re-fetches + # all; the linker still supersedes only on change, so a forced no-op writes nothing. + already_current = set() if force else alleles_linked_at_current_version(set(allele_data.keys())) + variant_caids = sorted({allele_data[aid].payload for aid in allele_data if aid not in already_current}) + job_manager.save_to_context( + { + "num_alleles_already_current": len(already_current), + "num_caids_to_query": len(variant_caids), + "force": force, + } + ) - job_manager.save_to_context({"num_gnomad_variants_with_caid_match": num_gnomad_variants_with_caid_match}) - job_manager.update_progress(75, 100, f"Found {num_gnomad_variants_with_caid_match} gnomAD variants matching CAIDs.") + changed_allele_ids: set[int] = set() + if variant_caids: + with athena.engine.connect() as athena_session: + logger.debug("Fetching gnomAD variants from Athena.") + gnomad_variant_data = gnomad_variant_data_for_caids(athena_session, variant_caids) - # Link mapped variants to gnomAD variants - logger.info(msg="Attempting to link mapped variants to gnomAD variants.", extra=job_manager.logging_context()) - num_linked_gnomad_variants = link_gnomad_variants_to_mapped_variants(job_manager.db, gnomad_variant_data) - job_manager.db.flush() + num_gnomad_variants_with_caid_match = len(gnomad_variant_data) + job_manager.save_to_context({"num_gnomad_variants_with_caid_match": num_gnomad_variants_with_caid_match}) + job_manager.update_progress( + 75, 100, f"Found {num_gnomad_variants_with_caid_match} gnomAD variants matching CAIDs." + ) - # For variants which are not linked, create annotation status records indicating skipped linkage - mapped_variants_with_caids = job_manager.db.scalars( - select(MappedVariant) - .join(Variant) - .join(ScoreSet) - .where( - ScoreSet.urn == score_set.urn, - MappedVariant.current.is_(True), - MappedVariant.clingen_allele_id.is_not(None), + logger.info(msg="Attempting to link alleles to gnomAD variants.", extra=job_manager.logging_context()) + changed_allele_ids = link_gnomad_variants_to_alleles(job_manager.db, gnomad_variant_data) + job_manager.db.flush() + else: + logger.info( + msg="All CAID-bearing alleles are already linked at the current gnomAD version; skipping Athena query.", + extra=job_manager.logging_context(), ) - ).all() + job_manager.update_progress(75, 100, "All alleles already current at this gnomAD version.") + + # Status is an audit event, written every run: SUCCESS/created if linked this run, SUCCESS/preexisting + # if already current, SKIPPED if no current-version link (gnomAD had no data for the CAID). + current_after = alleles_linked_at_current_version(set(allele_data.keys())) + annotation_manager = AnnotationStatusManager(job_manager.db, job_run_id=job_manager.job_id) - for mapped_variant in mapped_variants_with_caids: - if not mapped_variant.gnomad_variants: - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.GNOMAD_ALLELE_FREQUENCY, - version=GNOMAD_DATA_VERSION, - status=AnnotationStatus.SKIPPED, + created_count = preexisting_count = skipped_count = 0 + for allele_id, entry in allele_data.items(): + if allele_id in current_after: + action = "created" if allele_id in changed_allele_ids else "preexisting" + if action == "created": + created_count += 1 + else: + preexisting_count += 1 + _annotate_gnomad( + annotation_manager, + entry.authoritative_variant_ids, + AnnotationStatus.SUCCESS, + metadata={"clingen_allele_id": entry.payload, "action": action}, + ) + else: + skipped_count += 1 + _annotate_gnomad( + annotation_manager, + entry.authoritative_variant_ids, + AnnotationStatus.SKIPPED, failure_category=AnnotationFailureCategory.EXTERNAL_REFERENCE_NOT_FOUND, - annotation_data={ - "error_message": "No gnomAD variant could be linked for this mapped variant.", - }, - current=True, + error_message="No gnomAD variant could be linked for this allele.", + metadata={"clingen_allele_id": entry.payload}, ) annotation_manager.flush() - # Save final context and progress - job_manager.save_to_context({"num_mapped_variants_linked_to_gnomad_variants": num_linked_gnomad_variants}) - logger.info(msg="Done linking gnomAD variants to mapped variants.", extra=job_manager.logging_context()) + outcome_data = { + "created_allele_count": created_count, + "preexisting_allele_count": preexisting_count, + "skipped_allele_count": skipped_count, + } + job_manager.save_to_context(outcome_data) + logger.info(msg="Done linking gnomAD variants to alleles.", extra=job_manager.logging_context()) job_manager.db.flush() - return JobExecutionOutcome.succeeded( - data={ - "linked_count": num_linked_gnomad_variants, - "skipped_count": num_variant_caids - num_linked_gnomad_variants, - } - ) + return JobExecutionOutcome.succeeded(data=outcome_data) diff --git a/tests/lib/clingen/test_alleles.py b/tests/lib/clingen/test_alleles.py new file mode 100644 index 000000000..fbcfaa986 --- /dev/null +++ b/tests/lib/clingen/test_alleles.py @@ -0,0 +1,69 @@ +"""Unit tests for the pure allele-grouping primitive shared by the annotation jobs.""" + +from mavedb.lib.clingen.alleles import ScoreSetAlleleRow, group_alleles_for_annotation + + +def _row(allele_id, variant_id, *, is_authoritative, caid="CA1"): + return ScoreSetAlleleRow( + allele_id=allele_id, + post_mapped={"type": "Allele"}, + clingen_allele_id=caid, + variant_id=variant_id, + is_authoritative=is_authoritative, + ) + + +def test_collapses_rows_to_one_group_per_allele(): + rows = [ + _row(1, 10, is_authoritative=True), + _row(1, 11, is_authoritative=True), + _row(2, 12, is_authoritative=True, caid="CA2"), + ] + + groups = group_alleles_for_annotation(rows, payload=lambda r: r.clingen_allele_id) + + assert set(groups) == {1, 2} + assert groups[1].payload == "CA1" + assert groups[1].authoritative_variant_ids == [10, 11] + assert groups[2].authoritative_variant_ids == [12] + + +def test_only_authoritative_links_contribute_variant_ids(): + """An allele that is authoritative for one variant and RT-derived for another fans status only to + the authoritative variant — the bandaid invariant.""" + rows = [ + _row(1, 10, is_authoritative=True), + _row(1, 11, is_authoritative=False), + ] + + groups = group_alleles_for_annotation(rows, payload=lambda r: r.clingen_allele_id) + + assert groups[1].authoritative_variant_ids == [10] + + +def test_purely_rt_derived_allele_is_grouped_with_empty_fan_out(): + """A non-authoritative-only allele is still grouped (so it gets linked/annotated at the allele + level) but contributes no per-variant VAS row.""" + rows = [_row(1, 10, is_authoritative=False)] + + groups = group_alleles_for_annotation(rows, payload=lambda r: r.clingen_allele_id) + + assert set(groups) == {1} + assert groups[1].authoritative_variant_ids == [] + + +def test_payload_returning_none_skips_the_allele(): + """Returning None from payload drops the allele entirely — replacing each job's ad-hoc + 'no CAID / no HGVS -> continue' filter.""" + rows = [ + _row(1, 10, is_authoritative=True, caid=None), + _row(2, 11, is_authoritative=True, caid="CA2"), + ] + + groups = group_alleles_for_annotation(rows, payload=lambda r: r.clingen_allele_id) + + assert set(groups) == {2} + + +def test_empty_rows_yield_empty_grouping(): + assert group_alleles_for_annotation([], payload=lambda r: r.clingen_allele_id) == {} diff --git a/tests/lib/test_gnomad.py b/tests/lib/test_gnomad.py index 14dde9527..e00b57e76 100644 --- a/tests/lib/test_gnomad.py +++ b/tests/lib/test_gnomad.py @@ -3,8 +3,7 @@ from unittest.mock import patch import pytest - -from mavedb.models.variant_annotation_status import VariantAnnotationStatus +from sqlalchemy import select pyathena = pytest.importorskip("pyathena") fastapi = pytest.importorskip("fastapi") @@ -13,14 +12,15 @@ allele_list_from_list_like_string, gnomad_identifier, gnomad_table_name, - link_gnomad_variants_to_mapped_variants, + link_gnomad_variants_to_alleles, + normalize_caid, ) +from mavedb.models.allele import Allele +from mavedb.models.gnomad_allele_link import GnomadAlleleLink from mavedb.models.gnomad_variant import GnomADVariant -from mavedb.models.mapped_variant import MappedVariant from tests.helpers.constants import ( TEST_GNOMAD_DATA_VERSION, TEST_GNOMAD_VARIANT, - TEST_MINIMAL_MAPPED_VARIANT, ) ### Tests for gnomad_identifier function ### @@ -77,6 +77,23 @@ def test_gnomad_table_name_raises_if_env_not_set(): gnomad_table_name() +### Tests for normalize_caid function ### + + +@pytest.mark.parametrize( + "raw, expected", + [ + ("CA025094", "CA25094"), # the #722 example: gnomAD dump drops the leading zero + ("CA000123", "CA123"), # multiple leading zeros collapse + ("CA341478553", "CA341478553"), # already unpadded — unchanged + ("CA0", "CA0"), # keep the final digit even if it is a zero + ("not-a-caid", "not-a-caid"), # unrecognized input is passed through + ], +) +def test_normalize_caid(raw, expected): + assert normalize_caid(raw) == expected + + ### Tests for allele_list_from_list_like_string function ### @@ -118,217 +135,185 @@ def test_allele_list_from_list_like_string_invalid_format_not_list(): # If the package is working correctly, this function should work as expected. -### Tests for link_gnomad_variants_to_mapped_variants function ### +### Tests for link_gnomad_variants_to_alleles function ### -def _verify_annotation_status(session, mapped_variants, expected_version): - annotations = session.query(VariantAnnotationStatus).all() - assert len(annotations) == len(mapped_variants) +def _make_allele(session, caid, *, vrs_digest, level="genomic"): + """Create and persist a deduplicated Allele carrying a CAID.""" + allele = Allele(vrs_digest=vrs_digest, level=level, clingen_allele_id=caid) + session.add(allele) + session.commit() + session.refresh(allele) + return allele - for mapped_variant, annotation in zip(mapped_variants, annotations): - assert annotation.variant_id == mapped_variant.variant_id - assert annotation.annotation_type == "gnomad_allele_frequency" - assert annotation.version == expected_version +def _live_links_for(session, allele_id): + return session.scalars( + select(GnomadAlleleLink).where( + GnomadAlleleLink.allele_id == allele_id, + GnomadAlleleLink.current, + ) + ).all() -def test_links_new_gnomad_variant_to_mapped_variant( - session, mocked_gnomad_variant_row, setup_lib_db_with_mapped_variant -): - mapped_variant = setup_lib_db_with_mapped_variant - mapped_variant.clingen_allele_id = mocked_gnomad_variant_row.caid - session.add(mapped_variant) - session.commit() - with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): - result = link_gnomad_variants_to_mapped_variants(session, [mocked_gnomad_variant_row]) - assert result == 1 - session.commit() +def _assert_gnomad_variant_matches(gnomad_variant, **overrides): + expected = TEST_GNOMAD_VARIANT.copy() + expected.pop("creation_date") + expected.pop("modification_date") + expected.update(overrides) + for attr, value in expected.items(): + assert getattr(gnomad_variant, attr) == value - session.refresh(mapped_variant) - edited_saved_gnomad_variant = TEST_GNOMAD_VARIANT.copy() - edited_saved_gnomad_variant.pop("creation_date") - edited_saved_gnomad_variant.pop("modification_date") +def test_links_new_gnomad_variant_to_allele(session, mocked_gnomad_variant_row): + allele = _make_allele(session, mocked_gnomad_variant_row.caid, vrs_digest="vrs-1") - assert len(mapped_variant.gnomad_variants) == 1 - for attr in edited_saved_gnomad_variant: - assert getattr(mapped_variant.gnomad_variants[0], attr) == edited_saved_gnomad_variant[attr] + with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): + result = link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) + assert result == {allele.id} + session.commit() - _verify_annotation_status(session, [mapped_variant], TEST_GNOMAD_DATA_VERSION) + live_links = _live_links_for(session, allele.id) + assert len(live_links) == 1 + _assert_gnomad_variant_matches(live_links[0].gnomad_variant) -def test_can_link_gnomad_variants_with_none_type_faf_fields( - session, mocked_gnomad_variant_row, setup_lib_db_with_mapped_variant -): - mapped_variant = setup_lib_db_with_mapped_variant - mapped_variant.clingen_allele_id = mocked_gnomad_variant_row.caid - session.add(mapped_variant) - session.commit() +def test_can_link_gnomad_variants_with_none_type_faf_fields(session, mocked_gnomad_variant_row): + allele = _make_allele(session, mocked_gnomad_variant_row.caid, vrs_digest="vrs-1") mocked_gnomad_variant_row.__setattr__("joint.fafmax.faf95_max_gen_anc", None) mocked_gnomad_variant_row.__setattr__("joint.fafmax.faf95_max", None) with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): - result = link_gnomad_variants_to_mapped_variants(session, [mocked_gnomad_variant_row]) - assert result == 1 + result = link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) + assert result == {allele.id} session.commit() - gnomad_variant_comparator = TEST_GNOMAD_VARIANT.copy() - gnomad_variant_comparator.pop("creation_date") - gnomad_variant_comparator.pop("modification_date") - gnomad_variant_comparator["faf95_max"] = None - gnomad_variant_comparator["faf95_max_ancestry"] = None - - assert len(mapped_variant.gnomad_variants) == 1 - for attr in gnomad_variant_comparator: - assert getattr(mapped_variant.gnomad_variants[0], attr) == gnomad_variant_comparator[attr] - - _verify_annotation_status(session, [mapped_variant], TEST_GNOMAD_DATA_VERSION) + live_links = _live_links_for(session, allele.id) + assert len(live_links) == 1 + _assert_gnomad_variant_matches(live_links[0].gnomad_variant, faf95_max=None, faf95_max_ancestry=None) -def test_links_existing_gnomad_variant(session, mocked_gnomad_variant_row, setup_lib_db_with_mapped_variant): +def test_links_existing_gnomad_variant(session, mocked_gnomad_variant_row): gnomad_variant = GnomADVariant(**TEST_GNOMAD_VARIANT) - mapped_variant = setup_lib_db_with_mapped_variant - mapped_variant.clingen_allele_id = mocked_gnomad_variant_row.caid - session.add(mapped_variant) session.add(gnomad_variant) session.commit() + allele = _make_allele(session, mocked_gnomad_variant_row.caid, vrs_digest="vrs-1") with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): - result = link_gnomad_variants_to_mapped_variants(session, [mocked_gnomad_variant_row]) - assert result == 1 + result = link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) + assert result == {allele.id} session.commit() - session.refresh(mapped_variant) - - edited_saved_gnomad_variant = TEST_GNOMAD_VARIANT.copy() - edited_saved_gnomad_variant.pop("creation_date") - edited_saved_gnomad_variant.pop("modification_date") - - assert len(mapped_variant.gnomad_variants) == 1 - for attr in edited_saved_gnomad_variant: - assert getattr(mapped_variant.gnomad_variants[0], attr) == edited_saved_gnomad_variant[attr] - - _verify_annotation_status(session, [mapped_variant], TEST_GNOMAD_DATA_VERSION) - + # Reused the existing gnomAD variant rather than creating a second. + assert len(session.scalars(select(GnomADVariant)).all()) == 1 + live_links = _live_links_for(session, allele.id) + assert len(live_links) == 1 + assert live_links[0].gnomad_variant_id == gnomad_variant.id -def test_adding_existing_gnomad_variant_with_same_version_does_not_result_in_duplication( - session, mocked_gnomad_variant_row, setup_lib_db_with_mapped_variant -): - mapped_variant = setup_lib_db_with_mapped_variant - mapped_variant.clingen_allele_id = mocked_gnomad_variant_row.caid - session.add(mapped_variant) - session.commit() - with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): - result = link_gnomad_variants_to_mapped_variants(session, [mocked_gnomad_variant_row]) - assert result == 1 +def test_re_running_unchanged_data_is_idempotent(session, mocked_gnomad_variant_row): + """Supersede only on change: a second run with identical data writes nothing — one live link, + no retired rows, so the valid-time history records no spurious boundary.""" + allele = _make_allele(session, mocked_gnomad_variant_row.caid, vrs_digest="vrs-1") with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): - result = link_gnomad_variants_to_mapped_variants(session, [mocked_gnomad_variant_row]) - assert result == 1 + assert link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) == {allele.id} + session.commit() + # Second run sees the live link already points to this gnomAD variant → no change reported. + assert link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) == set() session.commit() - session.refresh(mapped_variant) - - edited_saved_gnomad_variant = TEST_GNOMAD_VARIANT.copy() - edited_saved_gnomad_variant.pop("creation_date") - edited_saved_gnomad_variant.pop("modification_date") + # One link, still live, never retired — the re-run did not churn the history. + all_links = session.scalars(select(GnomadAlleleLink).where(GnomadAlleleLink.allele_id == allele.id)).all() + assert len(all_links) == 1 + assert all_links[0].valid_to is None + assert len(session.scalars(select(GnomADVariant)).all()) == 1 - assert len(mapped_variant.gnomad_variants) == 1 - for attr in edited_saved_gnomad_variant: - assert getattr(mapped_variant.gnomad_variants[0], attr) == edited_saved_gnomad_variant[attr] - _verify_annotation_status(session, [mapped_variant, mapped_variant], TEST_GNOMAD_DATA_VERSION) +def test_version_bump_supersedes_to_single_live_link(session, mocked_gnomad_variant_row): + """A new gnomAD version retires the prior link and installs the new one — exactly one live link + per allele (not one per version), with the old version preserved as a retired row.""" + allele = _make_allele(session, mocked_gnomad_variant_row.caid, vrs_digest="vrs-1") + with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", "v1.old"): + assert link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) == {allele.id} + session.commit() -def test_links_multiple_rows_and_variants(session, mocked_gnomad_variant_row, setup_lib_db_with_mapped_variant): - mapped_variant1 = setup_lib_db_with_mapped_variant - mapped_variant2 = MappedVariant(**TEST_MINIMAL_MAPPED_VARIANT, variant_id=mapped_variant1.variant_id) + with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", "v2.new"): + assert link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) == {allele.id} + session.commit() - mapped_variant1.clingen_allele_id = mocked_gnomad_variant_row.caid - mapped_variant2.clingen_allele_id = mocked_gnomad_variant_row.caid - session.add(mapped_variant1) - session.add(mapped_variant2) + live_links = _live_links_for(session, allele.id) + assert len(live_links) == 1 + assert live_links[0].gnomad_variant.db_version == "v2.new" + # Old-version link retired, not deleted; both gnomAD variant rows persist. + all_links = session.scalars(select(GnomadAlleleLink).where(GnomadAlleleLink.allele_id == allele.id)).all() + assert len(all_links) == 2 + assert len([link for link in all_links if link.valid_to is not None]) == 1 + assert len(session.scalars(select(GnomADVariant)).all()) == 2 + + +def test_same_version_different_identifier_supersedes_newest_wins(session, mocked_gnomad_variant_row): + """A CAID re-resolving to a different identifier within the same version is an anomaly: log and + supersede newest-wins rather than raise — one odd allele must not abort the batch.""" + allele = _make_allele(session, mocked_gnomad_variant_row.caid, vrs_digest="vrs-1") + # Prior live link at the current version, but to a different identifier than the row resolves to. + stale = GnomADVariant( + db_name="gnomAD", + db_identifier="9-99999-C-T", + db_version=TEST_GNOMAD_DATA_VERSION, + allele_count=1, + allele_number=2, + allele_frequency=0.5, + ) + session.add(stale) + session.commit() + session.add(GnomadAlleleLink(allele_id=allele.id, gnomad_variant_id=stale.id)) session.commit() with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): - result = link_gnomad_variants_to_mapped_variants(session, [mocked_gnomad_variant_row]) - assert result == 2 + assert link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) == {allele.id} session.commit() - gnomad_variant_comparator = TEST_GNOMAD_VARIANT.copy() - gnomad_variant_comparator.pop("creation_date") - gnomad_variant_comparator.pop("modification_date") - - assert len(mapped_variant1.gnomad_variants) == 1 - assert len(mapped_variant2.gnomad_variants) == 1 - for mv in [mapped_variant1, mapped_variant2]: - for attr in gnomad_variant_comparator: - assert getattr(mv.gnomad_variants[0], attr) == gnomad_variant_comparator[attr] - - _verify_annotation_status(session, [mapped_variant1, mapped_variant2], TEST_GNOMAD_DATA_VERSION) - + live_links = _live_links_for(session, allele.id) + assert len(live_links) == 1 + assert live_links[0].gnomad_variant.db_identifier != "9-99999-C-T" # newest wins -def test_returns_zero_when_no_mapped_variants(session, mocked_gnomad_variant_row): - result = link_gnomad_variants_to_mapped_variants(session, [mocked_gnomad_variant_row]) - assert result == 0 - _verify_annotation_status(session, [], TEST_GNOMAD_DATA_VERSION) - - -def test_only_current_flag_filters_variants(session, mocked_gnomad_variant_row, setup_lib_db_with_mapped_variant): - mapped_variant1 = setup_lib_db_with_mapped_variant - mapped_variant2 = MappedVariant(**TEST_MINIMAL_MAPPED_VARIANT, variant_id=mapped_variant1.variant_id) - - mapped_variant1.current = False - mapped_variant1.clingen_allele_id = mocked_gnomad_variant_row.caid - mapped_variant2.clingen_allele_id = mocked_gnomad_variant_row.caid - session.add(mapped_variant1) - session.add(mapped_variant2) - session.commit() +def test_links_one_gnomad_variant_to_multiple_alleles_sharing_a_caid(session, mocked_gnomad_variant_row): + """A CAID shared by multiple alleles (cross-score-set dedup) fans the gnomAD variant to each.""" + allele1 = _make_allele(session, mocked_gnomad_variant_row.caid, vrs_digest="vrs-1") + allele2 = _make_allele(session, mocked_gnomad_variant_row.caid, vrs_digest="vrs-2", level="cdna") with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): - result = link_gnomad_variants_to_mapped_variants(session, [mocked_gnomad_variant_row]) - assert result == 1 + result = link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) + assert result == {allele1.id, allele2.id} session.commit() - gnomad_variant_comparator = TEST_GNOMAD_VARIANT.copy() - gnomad_variant_comparator.pop("creation_date") - gnomad_variant_comparator.pop("modification_date") - - assert len(mapped_variant1.gnomad_variants) == 0 - assert len(mapped_variant2.gnomad_variants) == 1 - for attr in gnomad_variant_comparator: - assert getattr(mapped_variant2.gnomad_variants[0], attr) == gnomad_variant_comparator[attr] + for allele in (allele1, allele2): + assert len(_live_links_for(session, allele.id)) == 1 + # Both links point at the single get-or-created gnomAD variant. + assert len(session.scalars(select(GnomADVariant)).all()) == 1 - _verify_annotation_status(session, [mapped_variant2], TEST_GNOMAD_DATA_VERSION) - -def test_only_current_flag_is_false_operates_on_all_variants( - session, mocked_gnomad_variant_row, setup_lib_db_with_mapped_variant -): - mapped_variant1 = setup_lib_db_with_mapped_variant - mapped_variant2 = MappedVariant(**TEST_MINIMAL_MAPPED_VARIANT, variant_id=mapped_variant1.variant_id) - - mapped_variant1.current = False - mapped_variant1.clingen_allele_id = mocked_gnomad_variant_row.caid - mapped_variant2.clingen_allele_id = mocked_gnomad_variant_row.caid - session.add(mapped_variant1) - session.add(mapped_variant2) - session.commit() +def test_links_allele_when_dump_strips_leading_zero_from_caid(session, mocked_gnomad_variant_row): + """The gnomAD dump records CAIDs without leading zeros (#722). An allele stored with the + zero-padded CAID must still match the dump's stripped form across the join.""" + allele = _make_allele(session, "CA025094", vrs_digest="vrs-1") + mocked_gnomad_variant_row.caid = "CA25094" # dump form: leading zero stripped with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): - result = link_gnomad_variants_to_mapped_variants(session, [mocked_gnomad_variant_row], False) - assert result == 2 + result = link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) + assert result == {allele.id} session.commit() - gnomad_variant_comparator = TEST_GNOMAD_VARIANT.copy() - gnomad_variant_comparator.pop("creation_date") - gnomad_variant_comparator.pop("modification_date") - for mv in [mapped_variant1, mapped_variant2]: - assert len(mv.gnomad_variants) == 1 - for attr in gnomad_variant_comparator: - assert getattr(mv.gnomad_variants[0], attr) == gnomad_variant_comparator[attr] + assert len(_live_links_for(session, allele.id)) == 1 + - _verify_annotation_status(session, [mapped_variant1, mapped_variant2], TEST_GNOMAD_DATA_VERSION) +def test_returns_empty_set_when_no_alleles_match(session, mocked_gnomad_variant_row): + result = link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) + assert result == set() + assert len(session.scalars(select(GnomadAlleleLink)).all()) == 0 + # No gnomAD variant is created when nothing matches the CAID. + assert len(session.scalars(select(GnomADVariant)).all()) == 0 diff --git a/tests/worker/jobs/conftest.py b/tests/worker/jobs/conftest.py index eac380862..d377f3fda 100644 --- a/tests/worker/jobs/conftest.py +++ b/tests/worker/jobs/conftest.py @@ -1,9 +1,13 @@ import pytest +from sqlalchemy import select +from mavedb.models.allele import Allele from mavedb.models.enums.job_pipeline import DependencyType from mavedb.models.job_dependency import JobDependency from mavedb.models.job_run import JobRun from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.pipeline import Pipeline from mavedb.models.score_set import ScoreSet from mavedb.models.variant import Variant @@ -281,6 +285,87 @@ def setup_sample_variants_with_caid( return variant, mapped_variant +@pytest.fixture +def setup_sample_alleles_with_caid(session, with_populated_domain_data, sample_link_gnomad_variants_run): + """Set up new-model rows (Variant + live MappingRecord + authoritative MappingRecordAllele + Allele) + for the gnomAD linkage job. The allele carries the CAID matched by the mocked Athena row, and the + allele is the authoritative measurement for the variant so the bandaid seam writes its VAS row. + """ + score_set = session.get(ScoreSet, sample_link_gnomad_variants_run.job_params["score_set_id"]) + + variant = Variant( + urn="urn:variant:test-variant-with-allele-caid", + score_set_id=score_set.id, + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={"hgvs_c": "NM_000000.1:c.1A>G", "hgvs_p": "NP_000000.1:p.Met1Val"}, + ) + allele = Allele( + vrs_digest="test-allele-vrs-digest", + level="genomic", + clingen_allele_id=VALID_CAID, + post_mapped={"type": "Allele", "expressions": [{"value": "NM_000000.1:c.1A>G", "syntax": "hgvs.c"}]}, + ) + session.add_all([variant, allele]) + session.commit() + + mapping_record = MappingRecord( + variant_id=variant.id, + assay_level="genomic", + mapping_api_version="pytest.0.0", + ) + session.add(mapping_record) + session.commit() + + session.add( + MappingRecordAllele( + mapping_record_id=mapping_record.id, + allele_id=allele.id, + is_authoritative=True, + ) + ) + session.commit() + return variant, allele + + +@pytest.fixture +def setup_rt_derived_allele_with_caid(session, setup_sample_alleles_with_caid): + """Add a NON-authoritative (RT-derived) allele to the variant's current mapping record, carrying + the CAID the mocked gnomAD row matches. The authoritative allele is given a CAID with no gnomAD + match, so only the RT-derived allele can link. This isolates the requirement that gnomAD linkage + must cover the full allele set (authoritative + RT-derived), not just authoritative links — for + protein/coding score sets the genomic allele gnomAD knows is the RT-derived one. + """ + variant, authoritative_allele = setup_sample_alleles_with_caid + + # Authoritative allele's CAID intentionally has no gnomAD match, so it cannot be what links. + authoritative_allele.clingen_allele_id = "CA_NO_GNOMAD_MATCH" + session.add(authoritative_allele) + + mapping_record = session.scalars( + select(MappingRecord).where(MappingRecord.variant_id == variant.id, MappingRecord.current) + ).one() + + rt_allele = Allele( + vrs_digest="test-rt-derived-allele-vrs-digest", + level="genomic", + clingen_allele_id=VALID_CAID, + post_mapped={"type": "Allele", "expressions": [{"value": "NC_000001.11:g.12345G>A", "syntax": "hgvs.g"}]}, + ) + session.add(rt_allele) + session.commit() + + session.add( + MappingRecordAllele( + mapping_record_id=mapping_record.id, + allele_id=rt_allele.id, + is_authoritative=False, + ) + ) + session.commit() + return variant, authoritative_allele, rt_allele + + ## Uniprot Job Fixtures ## diff --git a/tests/worker/jobs/external_services/test_gnomad.py b/tests/worker/jobs/external_services/test_gnomad.py index fc8e211c0..23316adb8 100644 --- a/tests/worker/jobs/external_services/test_gnomad.py +++ b/tests/worker/jobs/external_services/test_gnomad.py @@ -4,12 +4,16 @@ pytest.importorskip("arq") -from unittest.mock import MagicMock, patch +from unittest.mock import patch +from sqlalchemy import select + +from mavedb.lib import gnomad as gnomad_lib +from mavedb.lib.gnomad import GNOMAD_DATA_VERSION from mavedb.lib.types.workflow import JobExecutionOutcome from mavedb.models.enums.job_pipeline import JobStatus, PipelineStatus +from mavedb.models.gnomad_allele_link import GnomadAlleleLink from mavedb.models.gnomad_variant import GnomADVariant -from mavedb.models.mapped_variant import MappedVariant from mavedb.models.variant_annotation_status import VariantAnnotationStatus from mavedb.worker.jobs.external_services.gnomad import link_gnomad_variants from mavedb.worker.lib.managers.job_manager import JobManager @@ -22,7 +26,7 @@ class TestLinkGnomadVariantsUnit: """Unit tests for the link_gnomad_variants job.""" - async def test_link_gnomad_variants_no_variants_with_caids( + async def test_link_gnomad_variants_no_alleles_with_caids( self, session, with_populated_domain_data, @@ -30,7 +34,7 @@ async def test_link_gnomad_variants_no_variants_with_caids( mock_worker_ctx, sample_link_gnomad_variants_run, ): - """Test linking gnomAD variants when no mapped variants have CAIDs.""" + """No authoritative alleles with CAIDs -> the job succeeds with nothing to do.""" result = await link_gnomad_variants( mock_worker_ctx, 1, @@ -47,7 +51,7 @@ async def test_link_gnomad_variants_no_gnomad_matches( with_gnomad_linking_job, mock_worker_ctx, sample_link_gnomad_variants_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, athena_engine, ): """Test linking gnomAD variants when no gnomAD variants match the CAIDs.""" @@ -55,7 +59,7 @@ async def test_link_gnomad_variants_no_gnomad_matches( with ( patch( "mavedb.worker.jobs.external_services.gnomad.gnomad_variant_data_for_caids", - return_value={}, + return_value=[], ), patch("mavedb.worker.jobs.external_services.gnomad.athena.engine", athena_engine), ): @@ -75,7 +79,7 @@ async def test_link_gnomad_variants_call_linking_method( with_gnomad_linking_job, mock_worker_ctx, sample_link_gnomad_variants_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, athena_engine, ): """Test that the linking method is called when gnomAD variants match CAIDs.""" @@ -83,11 +87,11 @@ async def test_link_gnomad_variants_call_linking_method( with ( patch( "mavedb.worker.jobs.external_services.gnomad.gnomad_variant_data_for_caids", - return_value=[MagicMock()], + return_value=[object()], ), patch( - "mavedb.worker.jobs.external_services.gnomad.link_gnomad_variants_to_mapped_variants", - return_value=1, + "mavedb.worker.jobs.external_services.gnomad.link_gnomad_variants_to_alleles", + return_value=set(), ) as mock_linking_method, patch("mavedb.worker.jobs.external_services.gnomad.athena.engine", athena_engine), ): @@ -108,7 +112,7 @@ async def test_link_gnomad_variants_propagates_exceptions( with_gnomad_linking_job, mock_worker_ctx, sample_link_gnomad_variants_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, athena_engine, ): """Test that exceptions during the linking process are propagated.""" @@ -134,7 +138,7 @@ async def test_link_gnomad_variants_propagates_exceptions( class TestLinkGnomadVariantsIntegration: """Integration tests for the link_gnomad_variants job.""" - async def test_link_gnomad_variants_no_variants_with_caids( + async def test_link_gnomad_variants_no_alleles_with_caids( self, session, with_populated_domain_data, @@ -142,17 +146,16 @@ async def test_link_gnomad_variants_no_variants_with_caids( mock_worker_ctx, sample_link_gnomad_variants_run, ): - """Test the end-to-end functionality of the link_gnomad_variants job when no variants have CAIDs.""" + """Test the end-to-end functionality of the link_gnomad_variants job when no alleles have CAIDs.""" result = await link_gnomad_variants(mock_worker_ctx, sample_link_gnomad_variants_run.id) assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - # Verify that no gnomAD variants were linked - gnomad_variants = session.query(GnomADVariant).all() - assert len(gnomad_variants) == 0 + # Verify that no allele links were created + assert len(session.scalars(select(GnomadAlleleLink)).all()) == 0 - # Verify no annotations were rendered (since there were no variants with CAIDs) + # Verify no annotations were rendered (since there were no alleles with CAIDs) annotation_statuses = session.query(VariantAnnotationStatus).all() assert len(annotation_statuses) == 0 @@ -167,13 +170,13 @@ async def test_link_gnomad_variants_no_matching_caids( with_gnomad_linking_job, mock_worker_ctx, sample_link_gnomad_variants_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, athena_engine, ): """Test the end-to-end functionality of the link_gnomad_variants job when no matching CAIDs are found.""" - # Update the created mapped variant to have a CAID that won't match any gnomAD data - mapped_variant = session.query(MappedVariant).first() - mapped_variant.clingen_allele_id = "NON_MATCHING_CAID" + # Update the allele to have a CAID that won't match any seeded gnomAD data + _, allele = setup_sample_alleles_with_caid + allele.clingen_allele_id = "NON_MATCHING_CAID" session.commit() # Patch the athena engine to use the mock athena_engine fixture @@ -183,11 +186,10 @@ async def test_link_gnomad_variants_no_matching_caids( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - # Verify that no gnomAD variants were linked - gnomad_variants = session.query(GnomADVariant).all() - assert len(gnomad_variants) == 0 + # Verify that no allele links were created + assert len(session.scalars(select(GnomadAlleleLink)).all()) == 0 - # Verify a skipped annotation status was rendered (since there were variants with CAIDs) + # Verify a skipped annotation status was rendered (since there was an allele with a CAID) annotation_statuses = session.query(VariantAnnotationStatus).all() assert len(annotation_statuses) == 1 assert annotation_statuses[0].status == "skipped" @@ -204,10 +206,11 @@ async def test_link_gnomad_variants_successful_linking_independent( with_gnomad_linking_job, mock_worker_ctx, sample_link_gnomad_variants_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, athena_engine, ): """Test the end-to-end functionality of the link_gnomad_variants job with successful linking.""" + _, allele = setup_sample_alleles_with_caid # Patch the athena engine to use the mock athena_engine fixture with patch("mavedb.worker.jobs.external_services.gnomad.athena.engine", athena_engine): @@ -216,9 +219,15 @@ async def test_link_gnomad_variants_successful_linking_independent( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - # Verify that gnomAD variants were linked - gnomad_variants = session.query(GnomADVariant).all() - assert len(gnomad_variants) > 0 + # Verify that a gnomAD variant was created and a live link to the allele established + assert len(session.scalars(select(GnomADVariant)).all()) > 0 + live_links = session.scalars( + select(GnomadAlleleLink).where( + GnomadAlleleLink.allele_id == allele.id, + GnomadAlleleLink.current, + ) + ).all() + assert len(live_links) == 1 # Verify annotation status was rendered annotation_statuses = session.query(VariantAnnotationStatus).all() @@ -230,6 +239,147 @@ async def test_link_gnomad_variants_successful_linking_independent( session.refresh(sample_link_gnomad_variants_run) assert sample_link_gnomad_variants_run.status == JobStatus.SUCCEEDED + async def test_link_gnomad_variants_links_rt_derived_allele_but_annotates_only_authoritative( + self, + session, + with_populated_domain_data, + with_gnomad_linking_job, + mock_worker_ctx, + sample_link_gnomad_variants_run, + setup_rt_derived_allele_with_caid, + athena_engine, + ): + """gnomAD linkage must cover the full allele set, not just authoritative links: the + RT-derived allele carries the CAID gnomAD matches and must be linked. Per-variant VAS status, + however, is written only for the authoritative link (the interim bandaid) — so exactly one + annotation row is produced, keyed to the variant, never a second row for the RT-derived + allele.""" + variant, authoritative_allele, rt_allele = setup_rt_derived_allele_with_caid + + with patch("mavedb.worker.jobs.external_services.gnomad.athena.engine", athena_engine): + result = await link_gnomad_variants(mock_worker_ctx, sample_link_gnomad_variants_run.id) + + assert result.status == JobStatus.SUCCEEDED + + # The RT-derived (non-authoritative) allele IS linked — the core fix. + rt_links = session.scalars( + select(GnomadAlleleLink).where( + GnomadAlleleLink.allele_id == rt_allele.id, + GnomadAlleleLink.current, + ) + ).all() + assert len(rt_links) == 1 + # The authoritative allele's CAID had no gnomAD match, so it gets no link. + assert ( + len( + session.scalars( + select(GnomadAlleleLink).where(GnomadAlleleLink.allele_id == authoritative_allele.id) + ).all() + ) + == 0 + ) + + # Annotation status is written only for the authoritative link: exactly one VAS row, for the + # variant. (Its status is "skipped" because the variant's authoritative allele had no match — + # cross-level resolution onto the RT-derived allele is deferred to the AnnotationEvent design.) + annotation_statuses = session.query(VariantAnnotationStatus).all() + assert len(annotation_statuses) == 1 + assert annotation_statuses[0].variant_id == variant.id + assert annotation_statuses[0].annotation_type == "gnomad_allele_frequency" + + async def test_link_gnomad_variants_skips_allele_already_current( + self, + session, + with_populated_domain_data, + with_gnomad_linking_job, + mock_worker_ctx, + sample_link_gnomad_variants_run, + setup_sample_alleles_with_caid, + ): + """An allele already linked at the current gnomAD version is skipped: no Athena query, the + status reports SUCCESS/preexisting, and the existing link is not churned.""" + _, allele = setup_sample_alleles_with_caid + + # Simulate a prior run: a live link at the current gnomAD version. + gnomad_variant = GnomADVariant( + db_name="gnomAD", + db_identifier="1-12345-G-A", + db_version=GNOMAD_DATA_VERSION, + allele_count=1, + allele_number=2, + allele_frequency=0.5, + ) + session.add(gnomad_variant) + session.commit() + session.add(GnomadAlleleLink(allele_id=allele.id, gnomad_variant_id=gnomad_variant.id)) + session.commit() + + with patch("mavedb.worker.jobs.external_services.gnomad.gnomad_variant_data_for_caids") as fetch_spy: + result = await link_gnomad_variants(mock_worker_ctx, sample_link_gnomad_variants_run.id) + + fetch_spy.assert_not_called() # version-keyed skip avoided the external query entirely + assert result.data["preexisting_allele_count"] == 1 + assert result.data["created_allele_count"] == 0 + + # Link not churned: still exactly one, still live. + links = session.scalars(select(GnomadAlleleLink).where(GnomadAlleleLink.allele_id == allele.id)).all() + assert len(links) == 1 + assert links[0].valid_to is None + + # Status is SUCCESS, marked preexisting. + annotation_statuses = session.query(VariantAnnotationStatus).all() + assert len(annotation_statuses) == 1 + assert annotation_statuses[0].status == "success" + assert annotation_statuses[0].annotation_metadata["action"] == "preexisting" + + async def test_link_gnomad_variants_force_refetches_without_churn( + self, + session, + with_populated_domain_data, + with_gnomad_linking_job, + mock_worker_ctx, + sample_link_gnomad_variants_run, + setup_sample_alleles_with_caid, + athena_engine, + ): + """force bypasses the skip and re-fetches, but the linker supersedes only on change, so a + forced re-run of unchanged data reports preexisting and does not churn the link.""" + _, allele = setup_sample_alleles_with_caid + + # Prior live link pointing at the same variant the Athena mock resolves to (1-12345-G-A). + gnomad_variant = GnomADVariant( + db_name="gnomAD", + db_identifier="1-12345-G-A", + db_version=GNOMAD_DATA_VERSION, + allele_count=23, + allele_number=32432423, + allele_frequency=23 / 32432423, + ) + session.add(gnomad_variant) + session.commit() + session.add(GnomadAlleleLink(allele_id=allele.id, gnomad_variant_id=gnomad_variant.id)) + session.commit() + + sample_link_gnomad_variants_run.job_params = {**sample_link_gnomad_variants_run.job_params, "force": True} + session.commit() + + with ( + patch("mavedb.worker.jobs.external_services.gnomad.athena.engine", athena_engine), + patch( + "mavedb.worker.jobs.external_services.gnomad.gnomad_variant_data_for_caids", + side_effect=gnomad_lib.gnomad_variant_data_for_caids, + ) as fetch_spy, + ): + result = await link_gnomad_variants(mock_worker_ctx, sample_link_gnomad_variants_run.id) + + fetch_spy.assert_called_once() # force bypassed the version-keyed skip + assert result.data["preexisting_allele_count"] == 1 + assert result.data["created_allele_count"] == 0 + # Unchanged → no churn. + links = session.scalars(select(GnomadAlleleLink).where(GnomadAlleleLink.allele_id == allele.id)).all() + assert len(links) == 1 + assert links[0].valid_to is None + async def test_link_gnomad_variants_successful_linking_pipeline( self, session, @@ -237,7 +387,7 @@ async def test_link_gnomad_variants_successful_linking_pipeline( mock_worker_ctx, sample_link_gnomad_variants_run_pipeline, sample_link_gnomad_variants_pipeline, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, athena_engine, ): """Test the end-to-end functionality of the link_gnomad_variants job with successful linking in a pipeline.""" @@ -249,9 +399,8 @@ async def test_link_gnomad_variants_successful_linking_pipeline( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - # Verify that gnomAD variants were linked - gnomad_variants = session.query(GnomADVariant).all() - assert len(gnomad_variants) > 0 + # Verify that allele links were created + assert len(session.scalars(select(GnomadAlleleLink).where(GnomadAlleleLink.current)).all()) > 0 # Verify annotation status was rendered annotation_statuses = session.query(VariantAnnotationStatus).all() @@ -274,7 +423,7 @@ async def test_link_gnomad_variants_exceptions_handled_by_decorators( with_gnomad_linking_job, mock_worker_ctx, sample_link_gnomad_variants_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, athena_engine, ): """Test that exceptions during the linking process are handled by decorators.""" @@ -317,7 +466,7 @@ async def test_link_gnomad_variants_with_arq_context_independent( with_gnomad_linking_job, athena_engine, sample_link_gnomad_variants_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, ): """Test that the link_gnomad_variants job works with the ARQ context fixture.""" @@ -328,9 +477,8 @@ async def test_link_gnomad_variants_with_arq_context_independent( await arq_worker.async_run() await arq_worker.run_check() - # Verify that gnomAD variants were linked - gnomad_variants = session.query(GnomADVariant).all() - assert len(gnomad_variants) > 0 + # Verify that allele links were created + assert len(session.scalars(select(GnomadAlleleLink).where(GnomadAlleleLink.current)).all()) > 0 # Verify annotation status was rendered annotation_statuses = session.query(VariantAnnotationStatus).all() @@ -351,7 +499,7 @@ async def test_link_gnomad_variants_with_arq_context_pipeline( athena_engine, sample_link_gnomad_variants_run_pipeline, sample_link_gnomad_variants_pipeline, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, ): """Test that the link_gnomad_variants job works with the ARQ context fixture in a pipeline.""" @@ -362,9 +510,8 @@ async def test_link_gnomad_variants_with_arq_context_pipeline( await arq_worker.async_run() await arq_worker.run_check() - # Verify that gnomAD variants were linked - gnomad_variants = session.query(GnomADVariant).all() - assert len(gnomad_variants) > 0 + # Verify that allele links were created + assert len(session.scalars(select(GnomadAlleleLink).where(GnomadAlleleLink.current)).all()) > 0 # Verify annotation status was rendered annotation_statuses = session.query(VariantAnnotationStatus).all() @@ -389,7 +536,7 @@ async def test_link_gnomad_variants_with_arq_context_exception_handling_independ with_gnomad_linking_job, athena_engine, sample_link_gnomad_variants_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, ): """Test that exceptions in the link_gnomad_variants job are handled with the ARQ context fixture.""" @@ -406,9 +553,8 @@ async def test_link_gnomad_variants_with_arq_context_exception_handling_independ await arq_worker.run_check() mock_send_slack_job_error.assert_called_once() - # Verify that no gnomAD variants were linked - gnomad_variants = session.query(GnomADVariant).all() - assert len(gnomad_variants) == 0 + # Verify that no allele links were created + assert len(session.scalars(select(GnomadAlleleLink)).all()) == 0 # Verify no annotations were rendered annotation_statuses = session.query(VariantAnnotationStatus).all() @@ -427,7 +573,7 @@ async def test_link_gnomad_variants_with_arq_context_exception_handling_pipeline athena_engine, sample_link_gnomad_variants_pipeline, sample_link_gnomad_variants_run_pipeline, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, ): """Test that exceptions in the link_gnomad_variants job are handled with the ARQ context fixture.""" @@ -444,9 +590,8 @@ async def test_link_gnomad_variants_with_arq_context_exception_handling_pipeline await arq_worker.run_check() mock_send_slack_job_error.assert_called_once() - # Verify that no gnomAD variants were linked - gnomad_variants = session.query(GnomADVariant).all() - assert len(gnomad_variants) == 0 + # Verify that no allele links were created + assert len(session.scalars(select(GnomadAlleleLink)).all()) == 0 # Verify no annotations were rendered annotation_statuses = session.query(VariantAnnotationStatus).all() From c77c21b0fc35d3546b181441f3e6eaec26b18f26 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Mon, 22 Jun 2026 14:34:01 -0700 Subject: [PATCH 030/252] feat: link VEP consequences to alleles with Ensembl-release versioning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Migrate the VEP functional-consequence job (Step 2 of the annotation infrastructure migration) off MappedVariant onto the deduplicated allele model. - New ValidTime vep_allele_consequences table: a single allele-keyed row collapses record + link (the consequence is a scalar with no shared external entity, unlike gnomAD/ClinVar). One live consequence per allele via a partial unique index. - Job runs over the score set's full allele set (authoritative + RT-derived) via get_alleles_for_score_set + the shared grouping primitive; VAS still fans only to authoritative variants (the bandaid seam). - Version-key the refresh on the Ensembl release (/info/software): skip alleles already current, abort if the release can't be fetched. Supersede is value-keyed, not version-keyed — an unchanged consequence at a new release bumps source_version/access_date in place to avoid churning history. force bypasses the skip (e.g. after editing VEP_CONSEQUENCES). - A no-result is treated as a non-answer, never a negative: held consequences are not retired on an empty/failed VEP fetch. - Annotation links stay one-directional to Allele (no reverse back-ref). Adds lib + job tests covering linkage, RT-derived scope, version skip, in-place bump, supersede-on-change, no-result handling, and release-fetch failure. --- ...7c4e9d2f1b8_add_vep_allele_consequences.py | 65 ++ src/mavedb/lib/clingen/alleles.py | 17 +- src/mavedb/lib/vep.py | 108 ++- src/mavedb/models/__init__.py | 1 + src/mavedb/models/allele.py | 4 + src/mavedb/models/vep_allele_consequence.py | 75 ++ .../worker/jobs/external_services/vep.py | 594 +++++++-------- tests/lib/test_vep.py | 163 +++- tests/worker/jobs/conftest.py | 114 ++- .../external_services/network/test_vep.py | 37 +- .../worker/jobs/external_services/test_vep.py | 704 ++++++++++-------- 11 files changed, 1193 insertions(+), 689 deletions(-) create mode 100644 alembic/versions/a7c4e9d2f1b8_add_vep_allele_consequences.py create mode 100644 src/mavedb/models/vep_allele_consequence.py diff --git a/alembic/versions/a7c4e9d2f1b8_add_vep_allele_consequences.py b/alembic/versions/a7c4e9d2f1b8_add_vep_allele_consequences.py new file mode 100644 index 000000000..8b655f625 --- /dev/null +++ b/alembic/versions/a7c4e9d2f1b8_add_vep_allele_consequences.py @@ -0,0 +1,65 @@ +"""add vep_allele_consequences table + +Revision ID: a7c4e9d2f1b8 +Revises: e5f7a9c1b3d4 +Create Date: 2026-06-22 + +New valid-time table holding a deduplicated allele's VEP functional consequence, replacing the frozen +vep_functional_consequence/vep_access_date columns on mapped_variants for new-model writes (Step 2 of +the annotation infrastructure migration, docs/design/annotation-infrastructure-migration.md). A row is +live while valid_to is NULL; the partial unique index enforces a single live consequence per allele +(VEP's most-severe consequence is one current value, so a change supersedes rather than accumulates). +functional_consequence is nullable (reserved for a future negative cache). source_version is the +Ensembl release the consequence was resolved under (coordinated software + transcript set + vocabulary), +which version-keys the refresh skip like gnomAD's db_version; access_date is retained as a "last +confirmed" audit stamp. The VEP columns on mapped_variants are left untouched (frozen serving). +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "a7c4e9d2f1b8" +down_revision = "e5f7a9c1b3d4" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "vep_allele_consequences", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("allele_id", sa.Integer(), nullable=False), + sa.Column("functional_consequence", sa.String(), nullable=True), + sa.Column("source_version", sa.String(), nullable=False), + sa.Column("access_date", sa.Date(), nullable=False), + sa.Column("valid_from", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + sa.Column("valid_to", sa.DateTime(timezone=True), nullable=True), + sa.ForeignKeyConstraint( + ["allele_id"], + ["alleles.id"], + name="fk_vep_allele_consequences_allele_id", + ondelete="RESTRICT", + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + "ix_vep_allele_consequences_allele_id", + "vep_allele_consequences", + ["allele_id"], + ) + # One live consequence per allele: VEP's most-severe consequence is a single current value, so a + # changed result supersedes the prior row rather than accumulating one live row per access. + op.create_index( + "uq_vep_allele_consequences_live", + "vep_allele_consequences", + ["allele_id"], + unique=True, + postgresql_where=sa.text("valid_to IS NULL"), + ) + + +def downgrade() -> None: + op.drop_index("uq_vep_allele_consequences_live", table_name="vep_allele_consequences") + op.drop_index("ix_vep_allele_consequences_allele_id", table_name="vep_allele_consequences") + op.drop_table("vep_allele_consequences") diff --git a/src/mavedb/lib/clingen/alleles.py b/src/mavedb/lib/clingen/alleles.py index 79baf47ff..a2a962765 100644 --- a/src/mavedb/lib/clingen/alleles.py +++ b/src/mavedb/lib/clingen/alleles.py @@ -25,6 +25,10 @@ class ScoreSetAlleleRow(NamedTuple): ``is_authoritative`` is a property of the link, not the allele: the same VRS allele can be the authoritative measurement for one variant and an RT-derived equivalence for another. + + ``hgvs_g``/``hgvs_c``/``hgvs_p`` are allele-level (stable by construction), carried here so the + VEP job can build its HGVS payload without a second query. They are optional with a ``None`` + default so payloads keying only on the CAID (gnomAD/ClinVar) need not name them. """ allele_id: int @@ -32,6 +36,9 @@ class ScoreSetAlleleRow(NamedTuple): clingen_allele_id: str | None variant_id: int is_authoritative: bool + hgvs_g: str | None = None + hgvs_c: str | None = None + hgvs_p: str | None = None def get_alleles_for_score_set(db: Session, score_set_id: int) -> list[ScoreSetAlleleRow]: @@ -51,6 +58,9 @@ def get_alleles_for_score_set(db: Session, score_set_id: int) -> list[ScoreSetAl Allele.clingen_allele_id, Variant.id.label("variant_id"), MappingRecordAllele.is_authoritative, + Allele.hgvs_g, + Allele.hgvs_c, + Allele.hgvs_p, ) .join(MappingRecordAllele, MappingRecordAllele.allele_id == Allele.id) .join(MappingRecord, MappingRecord.id == MappingRecordAllele.mapping_record_id) @@ -61,7 +71,12 @@ def get_alleles_for_score_set(db: Session, score_set_id: int) -> list[ScoreSetAl .where(Allele.post_mapped.is_not(None)) ).all() - return [ScoreSetAlleleRow(r.id, r.post_mapped, r.clingen_allele_id, r.variant_id, r.is_authoritative) for r in rows] + return [ + ScoreSetAlleleRow( + r.id, r.post_mapped, r.clingen_allele_id, r.variant_id, r.is_authoritative, r.hgvs_g, r.hgvs_c, r.hgvs_p + ) + for r in rows + ] @dataclass diff --git a/src/mavedb/lib/vep.py b/src/mavedb/lib/vep.py index a7d4e7b3c..4b8ac0342 100644 --- a/src/mavedb/lib/vep.py +++ b/src/mavedb/lib/vep.py @@ -4,11 +4,16 @@ import functools import logging import os -from typing import Optional, Sequence +from datetime import date +from typing import Mapping, Optional, Sequence import requests +from sqlalchemy import select +from sqlalchemy.orm import Session +from mavedb.lib.logging.context import logging_context, save_to_logging_context from mavedb.lib.utils import request_with_backoff +from mavedb.models.vep_allele_consequence import VepAlleleConsequence logger = logging.getLogger(__name__) @@ -185,3 +190,104 @@ async def get_functional_consequence(hgvs_strings: Sequence[str]) -> dict[str, O result[hgvs] = most_severe_consequence return result + + +async def get_ensembl_release() -> str: + """Return the current Ensembl release the REST API is serving, e.g. ``"116"`` (``/info/software``). + + An Ensembl release is coordinated — software, transcript set, and consequence vocabulary all bump + together under one number — so this single value version-keys VEP results the way gnomAD keys on its + data version. The job stamps it on each consequence and skips re-querying alleles already live at the + current release. Raises on failure: the version is load-bearing for the skip, so a job that cannot + determine it must abort rather than mis-version its writes. + """ + headers = {"Content-Type": "application/json", "Accept": "application/json"} + loop = asyncio.get_running_loop() + response = await loop.run_in_executor( + None, + functools.partial( + request_with_backoff, + method="GET", + url=f"{ENSEMBL_API_URL}/info/software", + headers=headers, + timeout=30, + ), + ) + return str(response.json()["release"]) + + +def link_vep_consequences_to_alleles( + db: Session, + consequence_by_allele_id: Mapping[int, Optional[str]], + *, + source_version: str, + access_date: date, +) -> set[int]: + """Store VEP consequences against deduplicated alleles, superseding only on change. + + ``consequence_by_allele_id`` maps each queried allele to the consequence VEP resolved this run + (``None`` when VEP + Variant Recoder found nothing). ``source_version`` is the Ensembl release the + run resolved against. Each allele holds at most one live :class:`VepAlleleConsequence`, handled per + allele: + + - **unchanged** (live row already carries this consequence): advance ``source_version`` and + ``access_date`` in place — no supersede. Supersede is value-keyed, not version-keyed: a new + release that resolves the same categorical consequence must not fabricate a transaction-time + boundary, which would churn history every release. + - **new or changed** (no live row, or a different consequence): supersede keyed on ``allele_id`` + (retire the old, insert the successor stamped with this ``source_version``/``access_date``). + - **None this run**: leave any live row in place — do not overwrite a held consequence with a null result. + Log a warning if VEP found no consequence for an allele which previously had a live consequence. + + Does not commit. Returns the allele ids whose consequence was created or superseded this run. + """ + save_to_logging_context({"num_alleles_to_link_vep": len(consequence_by_allele_id)}) + logger.debug(msg="Linking VEP consequences to alleles", extra=logging_context()) + + changed_allele_ids: set[int] = set() + for allele_id, consequence in consequence_by_allele_id.items(): + live = db.scalar( + select(VepAlleleConsequence).where( + VepAlleleConsequence.allele_id == allele_id, + VepAlleleConsequence.current, + ) + ) + + # VEP found nothing this run. Do not overwrite a held consequence with a null result. + if consequence is None: + if live is not None: + logger.warning( + f"VEP found no consequence for allele {allele_id} this run; leaving prior consequence " + f"'{live.functional_consequence}' in place.", + extra=logging_context(), + ) + + continue + + # Unchanged: advance version/freshness in place. + if live is not None and live.functional_consequence == consequence: + live.source_version = source_version + live.access_date = access_date + continue + + # New or changed consequence: retire any live row, insert the successor. + VepAlleleConsequence.supersede_live_where( + db, + [ + VepAlleleConsequence( + allele_id=allele_id, + functional_consequence=consequence, + source_version=source_version, + access_date=access_date, + ) + ], + VepAlleleConsequence.allele_id == allele_id, + ) + changed_allele_ids.add(allele_id) + + save_to_logging_context({"changed_allele_count": len(changed_allele_ids)}) + logger.info( + msg=f"Created or superseded {len(changed_allele_ids)} VEP allele consequences this run.", + extra=logging_context(), + ) + return changed_allele_ids diff --git a/src/mavedb/models/__init__.py b/src/mavedb/models/__init__.py index 5284da11b..b18e20ea7 100644 --- a/src/mavedb/models/__init__.py +++ b/src/mavedb/models/__init__.py @@ -43,4 +43,5 @@ "variant_annotation_status", "variant", "variant_translation", + "vep_allele_consequence", ] diff --git a/src/mavedb/models/allele.py b/src/mavedb/models/allele.py index 54ce07e50..a57c4c802 100644 --- a/src/mavedb/models/allele.py +++ b/src/mavedb/models/allele.py @@ -51,6 +51,10 @@ def _transcript_expression(cls): back_populates="allele", ) + # Annotation links (VEP, gnomAD, ClinVar) deliberately carry no reverse collection here — they are + # one-directional annotation->Allele, navigated set-wise from the link tables, not from an Allele + # instance. Keep new annotation links one-directional unless a read path needs the navigation. + __table_args__ = ( UniqueConstraint("vrs_digest", name="uq_alleles_vrs_digest"), Index("ix_alleles_vrs_digest", "vrs_digest"), diff --git a/src/mavedb/models/vep_allele_consequence.py b/src/mavedb/models/vep_allele_consequence.py new file mode 100644 index 000000000..de41230a5 --- /dev/null +++ b/src/mavedb/models/vep_allele_consequence.py @@ -0,0 +1,75 @@ +from datetime import date +from typing import TYPE_CHECKING + +from sqlalchemy import Column, Date, ForeignKey, Index, Integer, String, text +from sqlalchemy.orm import Mapped, relationship + +from mavedb.db.base import Base +from mavedb.db.mixins import ValidTime + +if TYPE_CHECKING: + from .allele import Allele + + +class VepAlleleConsequence(ValidTime, Base): + """Valid-time VEP functional-consequence result for a deduplicated :class:`Allele`. + + Replaces the frozen ``vep_functional_consequence``/``vep_access_date`` columns on + ``MappedVariant`` for new-model writes (Step 2 of the annotation infrastructure migration, + docs/design/annotation-infrastructure-migration.md). A row is live while ``valid_to`` is NULL; + the partial unique index enforces **a single live consequence per allele** — VEP's most-severe + consequence is one current value, so a changed result supersedes the prior row rather than + accumulating. This matches the gnomAD link shape, not ClinVar's multi-live shape. + + ``source_version`` is the Ensembl release the consequence was resolved under (e.g. ``"116"``, + from ``/info/software``). An Ensembl release is coordinated — software + transcript set + + consequence vocabulary all bump together under one number — so this single value version-keys the + upstream result exactly like gnomAD's ``db_version``. The job skips re-querying any allele already + live at the current release. What it does **not** capture is our own ``VEP_CONSEQUENCES`` severity + ordering (the list we pick "most severe" from); a change to that is a manual ``force`` re-run, not + an automatic supersede. + + Supersede is deliberately **value-keyed, not version-keyed** (the one divergence from gnomAD): a VEP + consequence is categorical and usually identical across releases, so superseding on every release + bump would churn history every quarter with rows recording "still missense, still missense." Instead + a new release that resolves the *same* consequence advances ``source_version``/``access_date`` in + place — no supersede — and only a *changed* consequence retires the old row and inserts a successor. + The trade-off: the live row's ``source_version`` is the latest release that confirmed the value, not + the release it first appeared; acceptable because it describes the currently-held value's + provenance, not when it became true. ``access_date`` is retained as a human-facing "last confirmed" + audit stamp; it is no longer load-bearing for the skip. + + ``functional_consequence`` is nullable to leave room for a future negative cache (NULL = "VEP ran + and found nothing"); the current job writes only non-null consequences and re-queries no-result + alleles each run, mirroring gnomAD's no-match handling. + """ + + __tablename__ = "vep_allele_consequences" + + id: Mapped[int] = Column(Integer, primary_key=True) + allele_id: Mapped[int] = Column( + Integer, + ForeignKey("alleles.id", ondelete="RESTRICT"), + nullable=False, + ) + functional_consequence = Column(String, nullable=True) + source_version: Mapped[str] = Column(String, nullable=False) + access_date: Mapped[date] = Column(Date, nullable=False) + + allele: Mapped["Allele"] = relationship("Allele") + + __table_args__ = ( + Index( + "ix_vep_allele_consequences_allele_id", + "allele_id", + ), + # At most one live consequence per allele. A changed result supersedes (retires the old, + # inserts the new) rather than accumulating; superseded rows stay for point-in-time queries. + # Only the live row participates in this constraint. + Index( + "uq_vep_allele_consequences_live", + "allele_id", + unique=True, + postgresql_where=text("valid_to IS NULL"), + ), + ) diff --git a/src/mavedb/worker/jobs/external_services/vep.py b/src/mavedb/worker/jobs/external_services/vep.py index ba577f230..02f6a4498 100644 --- a/src/mavedb/worker/jobs/external_services/vep.py +++ b/src/mavedb/worker/jobs/external_services/vep.py @@ -1,10 +1,8 @@ """VEP functional consequence jobs for variant effect prediction. -This module handles the submission and processing of variant effect predictions -using the Ensembl VEP API. - -The processing is asynchronous, requiring batch submission of HGVS strings -to the VEP API with fallback to Variant Recoder when necessary. +This module links deduplicated alleles to their Ensembl VEP functional consequence. Submission is +batched against the VEP API, with a Variant Recoder fallback for HGVS strings VEP cannot resolve +directly (notably protein HGVS). """ import asyncio @@ -15,14 +13,20 @@ from sqlalchemy import select from mavedb.lib.annotation_status_manager import AnnotationStatusManager +from mavedb.lib.clingen.alleles import ScoreSetAlleleRow, get_alleles_for_score_set, group_alleles_for_annotation from mavedb.lib.types.workflow import JobExecutionOutcome from mavedb.lib.utils import batched -from mavedb.lib.vep import VEP_CONSEQUENCES, get_functional_consequence, run_variant_recoder +from mavedb.lib.vep import ( + VEP_CONSEQUENCES, + get_ensembl_release, + get_functional_consequence, + link_vep_consequences_to_alleles, + run_variant_recoder, +) from mavedb.models.enums.annotation_type import AnnotationType from mavedb.models.enums.job_pipeline import AnnotationFailureCategory, AnnotationStatus -from mavedb.models.mapped_variant import MappedVariant from mavedb.models.score_set import ScoreSet -from mavedb.models.variant import Variant +from mavedb.models.vep_allele_consequence import VepAlleleConsequence from mavedb.worker.jobs.utils.setup import validate_job_params from mavedb.worker.lib.decorators.pipeline_management import with_pipeline_management from mavedb.worker.lib.managers.job_manager import JobManager @@ -34,17 +38,160 @@ _RECODER_CONCURRENCY = int(os.getenv("RECODER_CONCURRENCY", "5")) +def _annotate_vep( + annotation_manager: AnnotationStatusManager, + variant_ids: list[int], + status: AnnotationStatus, + *, + failure_category: AnnotationFailureCategory | None = None, + error_message: str | None = None, + metadata: dict | None = None, +) -> None: + """Fan a VEP_FUNCTIONAL_CONSEQUENCE annotation out to every variant served by an allele. + + AAS migration seam: the single choke point for VEP's per-variant VAS writes. At migration it + becomes an allele-keyed event writer; the per-variant fan-out goes away, and the variant + association narrows to provenance (which variant's mapping drove the linkage). See + docs/design/allele-annotation-status.md. + + VEP carries no version string, so the VAS row is written with ``version=None`` (unlike gnomAD, + which keys on its data version). + """ + annotation_data: dict = {"annotation_metadata": metadata or {}} + if error_message is not None: + annotation_data["error_message"] = error_message + + for variant_id in variant_ids: + annotation_manager.add_annotation( + variant_id=variant_id, + annotation_type=AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, + status=status, + failure_category=failure_category, + annotation_data=annotation_data, + current=True, + ) + + +def _vep_hgvs_payload(row: ScoreSetAlleleRow) -> str | None: + """Build VEP's HGVS input for an allele: each allele will only have one of these, so the first + non-null is the one to submit. + """ + return row.hgvs_g or row.hgvs_c or row.hgvs_p or None + + +async def _resolve_consequences(unique_hgvs: list[str], job_manager: JobManager) -> dict[str, str]: + """Resolve a set of HGVS strings to their most-severe VEP consequence. + + Phase 1 submits the HGVS strings to VEP. Phase 2 runs Variant Recoder on the misses (a VEP entry + with a null consequence is treated as a miss — VEP knew the variant but could not classify it). + Phase 3 re-submits the recoded genomic strings to VEP and maps the most-severe consequence back to + the original HGVS. Returns only the HGVS strings that resolved to a consequence; absent keys are + failures the caller treats as no-result. + """ + all_consequences: dict[str, str] = {} + batches = list(batched(unique_hgvs, _VEP_BATCH_SIZE)) + + # --- Phase 1: initial VEP pass --- + all_missing_hgvs: set[str] = set() + for batch_idx, batch in enumerate(batches): + consequences = await get_functional_consequence(list(batch)) + hit_consequences = {h: c for h, c in consequences.items() if c is not None} + all_consequences.update(hit_consequences) + all_missing_hgvs.update(set(batch) - set(hit_consequences.keys())) + + job_manager.update_progress( + int((batch_idx + 1) / len(batches) * 33), + 100, + f"Processed initial VEP batch {batch_idx + 1}/{len(batches)}", + ) + + logger.info( + msg=f"Completed initial VEP processing. {len(all_missing_hgvs)} HGVS strings require Variant Recoder fallback.", + extra=job_manager.logging_context(), + ) + + if not all_missing_hgvs: + return all_consequences + + # --- Phase 2: Variant Recoder fallback for HGVS strings VEP could not resolve --- + recoder_batch_list = list(batched(list(all_missing_hgvs), _RECODER_BATCH_SIZE)) + semaphore = asyncio.Semaphore(_RECODER_CONCURRENCY) + completed_recoder_batches = 0 + + async def _recoder_with_semaphore(batch: list[str], total: int) -> dict[str, list[str]]: + nonlocal completed_recoder_batches + async with semaphore: + result = await run_variant_recoder(batch) + completed_recoder_batches += 1 + job_manager.update_progress( + 33 + int(completed_recoder_batches / total * 33), + 100, + f"Completed Variant Recoder batch {completed_recoder_batches}/{total}", + ) + return result + + total_recoder_batches = len(recoder_batch_list) + recoder_results = await asyncio.gather( + *[_recoder_with_semaphore(list(b), total_recoder_batches) for b in recoder_batch_list], + return_exceptions=True, + ) + + first_exception = next((r for r in recoder_results if isinstance(r, BaseException)), None) + if first_exception is not None: + successful_batches = sum(1 for r in recoder_results if not isinstance(r, BaseException)) + logger.error( + msg=f"Variant Recoder error ({successful_batches}/{total_recoder_batches} batches succeeded): {first_exception}", + extra=job_manager.logging_context(), + ) + raise first_exception + + hgvs_to_genomic: dict[str, list[str]] = {} + for result in recoder_results: + hgvs_to_genomic.update(result) # type: ignore[arg-type] + + logger.info( + msg=f"Completed Variant Recoder processing. {len(hgvs_to_genomic)} HGVS strings successfully recoded.", + extra=job_manager.logging_context(), + ) + + # --- Phase 3: VEP pass on the recoded genomic HGVS strings --- + all_recoded_genomic_hgvs = list({g for genomic_list in hgvs_to_genomic.values() for g in genomic_list}) + recoded_vep_batch_list = list(batched(all_recoded_genomic_hgvs, _VEP_BATCH_SIZE)) + all_recoded_consequences: dict[str, str | None] = {} + + for recoded_idx, recoded_batch in enumerate(recoded_vep_batch_list): + all_recoded_consequences.update(await get_functional_consequence(list(recoded_batch))) + job_manager.update_progress( + 66 + int((recoded_idx + 1) / len(recoded_vep_batch_list) * 33), + 100, + f"Processed recoded VEP batch {recoded_idx + 1}/{len(recoded_vep_batch_list)}", + ) + + # Map the most-severe consequence from the recoded genomic strings back to the original HGVS. + for original_hgvs, recoded_hgvs_list in hgvs_to_genomic.items(): + recoded_consequences = [c for h in recoded_hgvs_list if (c := all_recoded_consequences.get(h))] + most_severe = next((c for c in VEP_CONSEQUENCES if c in recoded_consequences), None) + if most_severe: + all_consequences[original_hgvs] = most_severe + + return all_consequences + + @with_pipeline_management async def populate_vep_for_score_set(ctx: dict, job_id: int, job_manager: JobManager) -> JobExecutionOutcome: - """Populate VEP functional consequence predictions for all mapped variants in a ScoreSet. + """Link deduplicated alleles to their VEP functional consequence. - This function retrieves all mapped variants with a populated hgvs_assay_level field for a given - ScoreSet and submits them to the Ensembl VEP API in configurable batches. It handles fallback - to the Variant Recoder API for variants that cannot be processed by VEP directly. + Runs over the score set's current alleles (authoritative and RT-derived), submits each allele's + HGVS to VEP (with Variant Recoder fallback), and stores the most-severe consequence in a valid-time + :class:`VepAlleleConsequence`, superseding only on change. Job Parameters: - - score_set_id (int): The ID of the ScoreSet containing mapped variants. + - score_set_id (int): The ID of the ScoreSet whose alleles to process. - correlation_id (str): Correlation ID for tracing requests across services. + - force (bool, optional): Bypass the current-release skip and re-query every HGVS-bearing + allele. The linker still supersedes only on a value change, so a forced re-run of unchanged + data writes no new rows. Use for re-ingestion, to heal suspected corruption, or after editing + the VEP_CONSEQUENCES severity ordering (a change the release version cannot see). Args: ctx (dict): The job context dictionary. @@ -52,7 +199,7 @@ async def populate_vep_for_score_set(ctx: dict, job_id: int, job_manager: JobMan job_manager (JobManager): Manager for job lifecycle and DB operations. Returns: - JobExecutionOutcome: Outcome with counts of processed, successful, and failed variants. + JobExecutionOutcome: outcome with per-allele created/preexisting/skipped counts. """ job = job_manager.get_job() @@ -62,6 +209,7 @@ async def populate_vep_for_score_set(ctx: dict, job_id: int, job_manager: JobMan # Safely ignore mypy warnings here, as params were checked above. score_set = job_manager.db.scalars(select(ScoreSet).where(ScoreSet.id == job.job_params["score_set_id"])).one() # type: ignore correlation_id = job.job_params["correlation_id"] # type: ignore + force = bool(job.job_params.get("force", False)) # type: ignore[union-attr] job_manager.save_to_context( { @@ -71,348 +219,136 @@ async def populate_vep_for_score_set(ctx: dict, job_id: int, job_manager: JobMan "correlation_id": correlation_id, } ) - job_manager.update_progress(0, 100, "Starting VEP population.") - logger.info(msg="Started VEP population", extra=job_manager.logging_context()) - - mapped_variants = job_manager.db.scalars( - select(MappedVariant) - .join(Variant) - .where( - Variant.score_set_id == score_set.id, - MappedVariant.current.is_(True), - MappedVariant.post_mapped.isnot(None), - ) - ).all() + job_manager.update_progress(0, 100, "Starting VEP consequence linkage.") + logger.info(msg="Started VEP consequence linkage", extra=job_manager.logging_context()) + + # One work-unit per allele (payload = HGVS; alleles without one are skipped). Covers ALL the score + # set's alleles — authoritative and RT-derived — since the genomic allele VEP is most reliable on + # is often the RT-derived one; VAS still fans only to authoritative_variant_ids (the bandaid seam). + allele_data = group_alleles_for_annotation( + get_alleles_for_score_set(job_manager.db, score_set.id), + payload=_vep_hgvs_payload, + ) - if not mapped_variants: + num_alleles_with_hgvs = len(allele_data) + job_manager.save_to_context({"num_alleles_to_link_vep": num_alleles_with_hgvs}) + + if not allele_data: logger.warning( - msg=f"No mapped variants found for score set {score_set.urn}. Skipped VEP population.", + msg="No current alleles with HGVS were found for this score set. Skipping VEP linkage (nothing to do).", extra=job_manager.logging_context(), ) job_manager.db.flush() return JobExecutionOutcome.succeeded( - data={ - "variants_processed": 0, - "variants_with_consequences": 0, - "variants_without_consequences": 0, - "variants_recoder_failed": 0, - } - ) - - job_manager.save_to_context({"total_variants_to_process": len(mapped_variants)}) - logger.info( - msg=f"Found {len(mapped_variants)} mapped variants for VEP processing", - extra=job_manager.logging_context(), - ) - - annotation_manager = AnnotationStatusManager(job_manager.db, job_run_id=job_manager.job_id) - - mapped_variants_by_id = {mv.id: mv for mv in mapped_variants} - - # Extract HGVS strings; skip and annotate variants that have none. - hgvs_and_mapped_variant_id_pairs: list[tuple[str, int]] = [] - - for mapped_variant in mapped_variants: - if not mapped_variant.hgvs_assay_level: - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, - status=AnnotationStatus.SKIPPED, - failure_category=AnnotationFailureCategory.MISSING_IDENTIFIER, - annotation_data={"error_message": "Mapped variant does not have an associated HGVS string."}, - ) - logger.debug("Mapped variant does not have an associated HGVS string.", extra=job_manager.logging_context()) - continue - - hgvs_and_mapped_variant_id_pairs.append((mapped_variant.hgvs_assay_level, mapped_variant.id)) # type: ignore - - batches = list(batched(hgvs_and_mapped_variant_id_pairs, _VEP_BATCH_SIZE)) - - job_manager.save_to_context({"vep_batches": len(batches)}) - logger.debug( - msg=f"Prepared {len(batches)} VEP batches ({_VEP_BATCH_SIZE} variants/batch)", - extra=job_manager.logging_context(), - ) - - # --- Phase 1: Initial VEP pass --- - all_consequences: dict[str, str] = {} - all_missing_hgvs: set[str] = set() - - for batch_idx, batch in enumerate(batches): - logger.debug( - msg=f"Processing VEP batch {batch_idx + 1}/{len(batches)}", - extra=job_manager.logging_context(), - ) - - hgvs_strings, mapped_variant_ids = map(list, zip(*batch)) # type: ignore - - consequences = await get_functional_consequence(hgvs_strings) - logger.debug( - msg=f"Received consequences for {len(consequences)} variants in VEP batch {batch_idx + 1}", - extra=job_manager.logging_context(), + data={"created_allele_count": 0, "preexisting_allele_count": 0, "skipped_allele_count": 0} ) - # Only store variants where VEP returned an actual consequence string. A None value - # means VEP knew the variant but couldn't classify it — treat that the same as absent - # and route to Recoder so we have the best chance of getting a consequence. - hit_consequences = {h: c for h, c in consequences.items() if c is not None} - all_consequences.update(hit_consequences) - - missing_hgvs = set(hgvs_strings) - set(hit_consequences.keys()) - for hgvs, mapped_variant_id in zip(hgvs_strings, mapped_variant_ids): - if hgvs in missing_hgvs: - all_missing_hgvs.add(hgvs) - - progress_pct = int((batch_idx + 1) / len(batches) * 33) - job_manager.save_to_context( - { - "initial_vep_batches_processed": batch_idx + 1, - "missing_hgvs_count": len(all_missing_hgvs), - } - ) - job_manager.update_progress( - progress_pct, - 100, - f"Processed initial VEP batch {batch_idx + 1}/{len(batches)}", + all_allele_ids = set(allele_data.keys()) + + # The Ensembl release version-keys the run (coordinated software + transcript set + vocabulary). It + # is load-bearing for the skip below, so a failure here aborts the job rather than mis-versioning + # writes (the exception propagates to the job decorators). + ensembl_release = await get_ensembl_release() + job_manager.save_to_context({"ensembl_release": ensembl_release}) + + def alleles_at_current_release(allele_ids: set[int]) -> set[int]: + """Allele ids (within the given set) holding a live VEP consequence at the current Ensembl release.""" + if not allele_ids: + return set() + return set( + job_manager.db.scalars( + select(VepAlleleConsequence.allele_id) + .where(VepAlleleConsequence.allele_id.in_(allele_ids)) + .where(VepAlleleConsequence.current) + .where(VepAlleleConsequence.functional_consequence.isnot(None)) + .where(VepAlleleConsequence.source_version == ensembl_release) + ).all() ) - logger.info( - msg=f"Completed initial VEP processing. {len(all_missing_hgvs)} variants require Variant Recoder fallback.", - extra=job_manager.logging_context(), + # Cost: skip alleles already resolved at the current Ensembl release (they cannot change without a + # release bump). force re-queries all — including alleles unchanged upstream but whose VEP_CONSEQUENCES + # severity ordering we have since edited; the linker still supersedes only on a value change, so a + # forced no-op writes nothing. + already_current = set() if force else alleles_at_current_release(all_allele_ids) + hgvs_by_allele = {aid: allele_data[aid].payload for aid in allele_data if aid not in already_current} + unique_hgvs = sorted(set(hgvs_by_allele.values())) + job_manager.save_to_context( + { + "num_alleles_already_current": len(already_current), + "num_hgvs_to_query": len(unique_hgvs), + "force": force, + } ) - # --- Phase 2: Variant Recoder fallback for HGVS strings VEP could not resolve --- - hgvs_to_genomic: dict[str, list[str]] = {} - recoder_missing_hgvs: set[str] = set() - - if all_missing_hgvs: - logger.info( - msg=f"Running Variant Recoder for {len(all_missing_hgvs)} HGVS strings", - extra=job_manager.logging_context(), - ) - - recoder_batch_list = list(batched(list(all_missing_hgvs), _RECODER_BATCH_SIZE)) - - logger.debug( - msg=f"Running {len(recoder_batch_list)} Variant Recoder batches with concurrency {_RECODER_CONCURRENCY}", - extra=job_manager.logging_context(), - ) - - semaphore = asyncio.Semaphore(_RECODER_CONCURRENCY) - completed_recoder_batches = 0 - - async def _recoder_with_semaphore(batch: list[str], batch_idx: int, total: int) -> dict[str, list[str]]: - nonlocal completed_recoder_batches - async with semaphore: - logger.debug( - msg=f"Starting Variant Recoder batch {batch_idx + 1}/{total} ({len(batch)} HGVS strings)", - extra=job_manager.logging_context(), - ) - result = await run_variant_recoder(batch) - completed_recoder_batches += 1 - logger.debug( - msg=f"Completed Variant Recoder batch {completed_recoder_batches}/{total} ({len(result)} variants recoded)", - extra=job_manager.logging_context(), - ) - progress_pct = 33 + int(completed_recoder_batches / total * 33) - job_manager.update_progress( - progress_pct, - 100, - f"Completed Variant Recoder batch {completed_recoder_batches}/{total}", - ) - return result - - total_recoder_batches = len(recoder_batch_list) - recoder_results = await asyncio.gather( - *[ - _recoder_with_semaphore(list(recoder_batch), idx, total_recoder_batches) - for idx, recoder_batch in enumerate(recoder_batch_list) - ], - return_exceptions=True, - ) - - successful_batches = sum(1 for r in recoder_results if not isinstance(r, Exception)) - - first_exception = next((r for r in recoder_results if isinstance(r, Exception)), None) - if first_exception is not None: - logger.error( - msg=f"Variant Recoder error ({successful_batches}/{total_recoder_batches} batches succeeded): {str(first_exception)}", - extra=job_manager.logging_context(), - ) - raise first_exception - - for result in recoder_results: - hgvs_to_genomic.update(result) # type: ignore[arg-type] - - job_manager.save_to_context( - { - "variant_recoder_batches_processed": len(recoder_batch_list), - "recoded_variants_count": len(hgvs_to_genomic), - } - ) - logger.info( - msg=f"Completed Variant Recoder processing. {len(hgvs_to_genomic)} variants successfully recoded.", - extra=job_manager.logging_context(), + changed_allele_ids: set[int] = set() + if unique_hgvs: + job_manager.update_progress(10, 100, f"Querying VEP for {len(unique_hgvs)} HGVS strings.") + consequences_by_hgvs = await _resolve_consequences(unique_hgvs, job_manager) + consequence_by_allele_id = {aid: consequences_by_hgvs.get(hgvs) for aid, hgvs in hgvs_by_allele.items()} + changed_allele_ids = link_vep_consequences_to_alleles( + job_manager.db, consequence_by_allele_id, source_version=ensembl_release, access_date=date.today() ) - - # --- Phase 3: VEP pass on the recoded genomic HGVS strings --- - # hgvs_to_genomic maps original HGVS → list[str]; flatten to a deduplicated list of - # genomic strings before batching with VEP. - all_recoded_genomic_hgvs = list({g for genomic_list in hgvs_to_genomic.values() for g in genomic_list}) - recoded_vep_batch_list = list(batched(all_recoded_genomic_hgvs, _VEP_BATCH_SIZE)) - all_recoded_consequences: dict[str, str | None] = {} - - for recoded_vep_batch_idx, recoded_vep_batch in enumerate(recoded_vep_batch_list): - logger.debug( - msg=f"Processing recoded HGVS VEP batch {recoded_vep_batch_idx + 1}/{len(recoded_vep_batch_list)}", - extra=job_manager.logging_context(), - ) - - recoded_vep_consequences = await get_functional_consequence(recoded_vep_batch) - all_recoded_consequences.update(recoded_vep_consequences) - - progress_pct = 66 + int((recoded_vep_batch_idx + 1) / len(recoded_vep_batch_list) * 33) - job_manager.save_to_context( - { - "recoded_vep_batches_processed": recoded_vep_batch_idx + 1, - "recoded_consequences_count": len(all_recoded_consequences), - } - ) - job_manager.update_progress( - progress_pct, - 100, - f"Processed recoded VEP batch {recoded_vep_batch_idx + 1}/{len(recoded_vep_batch_list)}", - ) - + job_manager.db.flush() + else: logger.info( - msg=f"Completed recoded VEP processing. {len(all_recoded_consequences)} recoded consequences retrieved.", + msg="All HGVS-bearing alleles are already resolved at the current Ensembl release; skipping VEP query.", extra=job_manager.logging_context(), ) + job_manager.update_progress(99, 100, "All alleles already current at this Ensembl release.") + + # Status is an audit event, written every run: SUCCESS/created if linked this run, SUCCESS/preexisting + # if already current or re-confirmed unchanged, SKIPPED if no live consequence (VEP had no result). + live_now = job_manager.db.execute( + select(VepAlleleConsequence.allele_id, VepAlleleConsequence.functional_consequence) + .where(VepAlleleConsequence.allele_id.in_(all_allele_ids)) + .where(VepAlleleConsequence.current) + .where(VepAlleleConsequence.functional_consequence.isnot(None)) + ).all() + consequence_now = {r.allele_id: r.functional_consequence for r in live_now} - # Map most-severe consequence from recoded genomic HGVS back to the original HGVS. - for original_hgvs, recoded_hgvs_list in hgvs_to_genomic.items(): - recoded_consequences_for_variant = [ - c for recoded_hgvs in recoded_hgvs_list if (c := all_recoded_consequences.get(recoded_hgvs)) - ] - - if recoded_consequences_for_variant: - most_severe = next( - (c for c in VEP_CONSEQUENCES if c in recoded_consequences_for_variant), - None, - ) - if most_severe: - all_consequences[original_hgvs] = most_severe - logger.debug( - msg=f"Selected most severe consequence '{most_severe}' for {original_hgvs}", - extra=job_manager.logging_context(), - ) + annotation_manager = AnnotationStatusManager(job_manager.db, job_run_id=job_manager.job_id) + created_count = preexisting_count = skipped_count = 0 + for allele_id, entry in allele_data.items(): + if allele_id in consequence_now: + action = "created" if allele_id in changed_allele_ids else "preexisting" + if action == "created": + created_count += 1 else: - logger.debug( - msg=f"Could not retrieve functional consequences for any recoded variants of {original_hgvs}", - extra=job_manager.logging_context(), - ) - - recoder_missing_hgvs = all_missing_hgvs - set(hgvs_to_genomic.keys()) - - # --- Phase 4: Annotate outcomes and update mapped variants in a single pass --- - - # HGVS strings that went through both VEP passes but still have no consequence. - all_processed_hgvs = {h for h, _ in hgvs_and_mapped_variant_id_pairs} - vep_failed_hgvs = all_processed_hgvs - set(all_consequences.keys()) - recoder_missing_hgvs - - variants_processed = 0 - variants_with_consequences = 0 - variants_without_consequences = 0 - variants_recoder_failed = 0 - - for hgvs_string, mapped_variant_id in hgvs_and_mapped_variant_id_pairs: - mapped_variant = mapped_variants_by_id.get(mapped_variant_id) # type: ignore - if mapped_variant is None: - continue - - consequence = all_consequences.get(hgvs_string) - if consequence: - mapped_variant.vep_functional_consequence = consequence - mapped_variant.vep_access_date = date.today() - job_manager.db.add(mapped_variant) - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, - status=AnnotationStatus.SUCCESS, - annotation_data={"annotation_metadata": {"functional_consequence": consequence}}, - ) - variants_with_consequences += 1 - logger.debug( - msg=f"Set consequence '{consequence}' for mapped variant {mapped_variant_id} (HGVS: {hgvs_string})", - extra=job_manager.logging_context(), - ) - elif hgvs_string in vep_failed_hgvs: - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, - status=AnnotationStatus.FAILED, - failure_category=AnnotationFailureCategory.EXTERNAL_REFERENCE_NOT_FOUND, - annotation_data={ - "error_message": "VEP could not determine a functional consequence for this variant, even after Variant Recoder fallback.", + preexisting_count += 1 + _annotate_vep( + annotation_manager, + entry.authoritative_variant_ids, + AnnotationStatus.SUCCESS, + metadata={ + "functional_consequence": consequence_now[allele_id], + "hgvs": entry.payload, + "action": action, }, ) - variants_without_consequences += 1 - logger.debug( - msg=f"Recorded VEP failure for mapped_variant_id {mapped_variant_id} (HGVS: {hgvs_string})", - extra=job_manager.logging_context(), - ) - elif hgvs_string in recoder_missing_hgvs: - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, - status=AnnotationStatus.FAILED, - failure_category=AnnotationFailureCategory.EXTERNAL_REFERENCE_NOT_FOUND, - annotation_data={ - "error_message": "Variant Recoder could not recode this HGVS string to a genomic equivalent.", - }, - ) - variants_recoder_failed += 1 - logger.debug( - msg=f"Recorded Variant Recoder failure for mapped_variant_id {mapped_variant_id} (HGVS: {hgvs_string})", - extra=job_manager.logging_context(), - ) else: - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, - status=AnnotationStatus.FAILED, - failure_category=AnnotationFailureCategory.UNKNOWN, - annotation_data={ - "error_message": "Variant was not classified by any VEP outcome branch. This is a bug.", - }, - ) - variants_without_consequences += 1 - logger.warning( - msg=f"Unexpected state: mapped_variant_id {mapped_variant_id} (HGVS: {hgvs_string}) was not classified by any outcome branch.", - extra=job_manager.logging_context(), + skipped_count += 1 + _annotate_vep( + annotation_manager, + entry.authoritative_variant_ids, + AnnotationStatus.SKIPPED, + failure_category=AnnotationFailureCategory.EXTERNAL_REFERENCE_NOT_FOUND, + error_message="VEP could not determine a functional consequence for this allele, even after Variant Recoder fallback.", + metadata={"hgvs": entry.payload}, ) - variants_processed += 1 - annotation_manager.flush() - job_manager.db.flush() + outcome_data = { + "created_allele_count": created_count, + "preexisting_allele_count": preexisting_count, + "skipped_allele_count": skipped_count, + } + job_manager.save_to_context(outcome_data) job_manager.update_progress( 100, 100, - f"Completed VEP functional consequence prediction for {variants_with_consequences}/{variants_processed} variants.", + f"Completed VEP linkage: {created_count} created, {preexisting_count} preexisting, {skipped_count} skipped.", ) - logger.info( - msg=f"Completed VEP prediction: {variants_with_consequences} with consequences, {variants_without_consequences} without, {variants_recoder_failed} recoder failed", - extra=job_manager.logging_context(), - ) - + logger.info(msg="Done linking VEP consequences to alleles.", extra=job_manager.logging_context()) job_manager.db.flush() - return JobExecutionOutcome.succeeded( - data={ - "variants_processed": variants_processed, - "variants_with_consequences": variants_with_consequences, - "variants_without_consequences": variants_without_consequences, - "variants_recoder_failed": variants_recoder_failed, - } - ) + return JobExecutionOutcome.succeeded(data=outcome_data) diff --git a/tests/lib/test_vep.py b/tests/lib/test_vep.py index 2e9e1faef..0d9473f1b 100644 --- a/tests/lib/test_vep.py +++ b/tests/lib/test_vep.py @@ -4,11 +4,20 @@ logic correctly handles the actual Ensembl REST API response shapes. """ +from datetime import date, timedelta from unittest.mock import MagicMock, patch import pytest +from sqlalchemy import select -from mavedb.lib.vep import get_functional_consequence, run_variant_recoder +from mavedb.lib.vep import ( + get_ensembl_release, + get_functional_consequence, + link_vep_consequences_to_alleles, + run_variant_recoder, +) +from mavedb.models.allele import Allele +from mavedb.models.vep_allele_consequence import VepAlleleConsequence def _mock_response(data) -> MagicMock: @@ -240,3 +249,155 @@ async def test_raises_if_more_than_200_variants(self): """Passing more than 200 HGVS strings raises ValueError before any HTTP call.""" with pytest.raises(ValueError, match="maximum of 200"): await get_functional_consequence(["NM_007294.4:c.1A>T"] * 201) + + +### Tests for get_ensembl_release function ### + + +@pytest.mark.asyncio +async def test_get_ensembl_release_returns_release_as_string(): + """The /info/software release integer is returned as a string for use as source_version.""" + with patch("mavedb.lib.vep.request_with_backoff", return_value=_mock_response({"release": 116})): + assert await get_ensembl_release() == "116" + + +### Tests for link_vep_consequences_to_alleles function ### + + +def _make_allele(session, *, vrs_digest, level="genomic"): + """Create and persist a deduplicated Allele.""" + allele = Allele(vrs_digest=vrs_digest, level=level) + session.add(allele) + session.commit() + session.refresh(allele) + return allele + + +def _live_rows_for(session, allele_id): + return session.scalars( + select(VepAlleleConsequence).where( + VepAlleleConsequence.allele_id == allele_id, + VepAlleleConsequence.current, + ) + ).all() + + +def _all_rows_for(session, allele_id): + return session.scalars(select(VepAlleleConsequence).where(VepAlleleConsequence.allele_id == allele_id)).all() + + +def test_link_vep_creates_new_consequence(session): + """A consequence for an allele with no live row creates a single live row and is reported changed.""" + allele = _make_allele(session, vrs_digest="vrs-1") + + changed = link_vep_consequences_to_alleles( + session, {allele.id: "missense_variant"}, source_version="116", access_date=date.today() + ) + session.commit() + + assert changed == {allele.id} + live = _live_rows_for(session, allele.id) + assert len(live) == 1 + assert live[0].functional_consequence == "missense_variant" + assert live[0].source_version == "116" + assert live[0].access_date == date.today() + + +def test_link_vep_unchanged_bumps_version_and_date_in_place(session): + """Re-confirming an unchanged consequence at a new release advances source_version and access_date + in place — no supersede, no new valid-time boundary, and the allele is not reported changed.""" + allele = _make_allele(session, vrs_digest="vrs-1") + session.add( + VepAlleleConsequence( + allele_id=allele.id, + functional_consequence="missense_variant", + source_version="115", + access_date=date.today() - timedelta(days=90), + ) + ) + session.commit() + + changed = link_vep_consequences_to_alleles( + session, {allele.id: "missense_variant"}, source_version="116", access_date=date.today() + ) + session.commit() + + assert changed == set() + # One row, still live, never retired — version and access_date advanced in place. + all_rows = _all_rows_for(session, allele.id) + assert len(all_rows) == 1 + assert all_rows[0].valid_to is None + assert all_rows[0].source_version == "116" + assert all_rows[0].access_date == date.today() + + +def test_link_vep_changed_consequence_supersedes(session): + """A changed consequence retires the live row and inserts the successor — exactly one live row, + keyed on allele_id, with the old one preserved as retired history.""" + allele = _make_allele(session, vrs_digest="vrs-1") + session.add( + VepAlleleConsequence( + allele_id=allele.id, + functional_consequence="synonymous_variant", + source_version="115", + access_date=date.today() - timedelta(days=90), + ) + ) + session.commit() + + changed = link_vep_consequences_to_alleles( + session, {allele.id: "missense_variant"}, source_version="116", access_date=date.today() + ) + session.commit() + + assert changed == {allele.id} + live = _live_rows_for(session, allele.id) + assert len(live) == 1 + assert live[0].functional_consequence == "missense_variant" + assert live[0].source_version == "116" + + all_rows = _all_rows_for(session, allele.id) + assert len(all_rows) == 2 + assert len([r for r in all_rows if r.valid_to is not None]) == 1 + + +def test_link_vep_none_leaves_live_row_untouched(session): + """A transient None result must not overwrite a held consequence: the live row is left intact + (value, version, and date) and the allele is not reported changed.""" + allele = _make_allele(session, vrs_digest="vrs-1") + session.add( + VepAlleleConsequence( + allele_id=allele.id, + functional_consequence="missense_variant", + source_version="115", + access_date=date.today() - timedelta(days=90), + ) + ) + session.commit() + + changed = link_vep_consequences_to_alleles( + session, {allele.id: None}, source_version="116", access_date=date.today() + ) + session.commit() + + assert changed == set() + live = _live_rows_for(session, allele.id) + assert len(live) == 1 + assert live[0].functional_consequence == "missense_variant" + # Not re-confirmed -> neither version nor access_date advanced. + assert live[0].source_version == "115" + assert live[0].access_date == date.today() - timedelta(days=90) + + +def test_link_vep_none_with_no_live_row_writes_nothing(session): + """A None result for an allele with no live row writes nothing (the allele is re-queried next run), + mirroring gnomAD's no-match handling.""" + allele = _make_allele(session, vrs_digest="vrs-1") + + changed = link_vep_consequences_to_alleles( + session, {allele.id: None}, source_version="116", access_date=date.today() + ) + session.commit() + + assert changed == set() + assert len(_all_rows_for(session, allele.id)) == 0 diff --git a/tests/worker/jobs/conftest.py b/tests/worker/jobs/conftest.py index d377f3fda..6b7309fd2 100644 --- a/tests/worker/jobs/conftest.py +++ b/tests/worker/jobs/conftest.py @@ -1340,8 +1340,14 @@ def sample_populate_vep_run_pipeline( @pytest.fixture -def setup_sample_variants_for_vep(session, with_populated_domain_data, mock_worker_ctx, sample_populate_vep_run): - """Setup a variant and mapped variant with hgvs_assay_level for VEP testing.""" +def setup_sample_alleles_for_vep(session, with_populated_domain_data, mock_worker_ctx, sample_populate_vep_run): + """Set up new-model rows (Variant + live MappingRecord + authoritative MappingRecordAllele + Allele) + for the VEP consequence job. The allele carries an HGVS the job submits to VEP and is the + authoritative measurement for the variant, so the bandaid seam writes its per-variant VAS row. + + The HGVS lives on ``Allele.hgvs_c`` (a coding HGVS VEP resolves directly) — the new job reads its + submission string from the allele, not from ``MappedVariant.hgvs_assay_level``. + """ score_set = session.get(ScoreSet, sample_populate_vep_run.job_params["score_set_id"]) variant = Variant( @@ -1351,28 +1357,42 @@ def setup_sample_variants_for_vep(session, with_populated_domain_data, mock_work hgvs_pro="NP_009225.1:p.Cys2Tyr", data={"hgvs_c": "NM_007294.4:c.5A>G", "hgvs_p": "NP_009225.1:p.Cys2Tyr"}, ) - session.add(variant) + allele = Allele( + vrs_digest="test-vep-allele-vrs-digest", + level="cdna", + hgvs_c="NM_007294.4:c.5A>G", + post_mapped={"type": "Allele", "expressions": [{"value": "NM_007294.4:c.5A>G", "syntax": "hgvs.c"}]}, + ) + session.add_all([variant, allele]) session.commit() - mapped_variant = MappedVariant( + + mapping_record = MappingRecord( variant_id=variant.id, - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - post_mapped={"type": "Allele", "expressions": [{"value": "NM_007294.4:c.5A>G", "syntax": "hgvs.c"}]}, - hgvs_assay_level="NM_007294.4:c.5A>G", + assay_level="cdna", + mapping_api_version="pytest.0.0", ) - session.add(mapped_variant) + session.add(mapping_record) session.commit() - return variant, mapped_variant + + session.add( + MappingRecordAllele( + mapping_record_id=mapping_record.id, + allele_id=allele.id, + is_authoritative=True, + ) + ) + session.commit() + return variant, allele @pytest.fixture -def setup_sample_protein_variant_for_vep(session, with_populated_domain_data, mock_worker_ctx, sample_populate_vep_run): - """Setup a protein HGVS variant (NP_ accession) that VEP cannot resolve directly. +def setup_sample_protein_allele_for_vep(session, with_populated_domain_data, mock_worker_ctx, sample_populate_vep_run): + """Set up an allele whose only HGVS is a protein HGVS (NP_ accession) that VEP cannot resolve + directly. - VEP's /vep/human/hgvs endpoint does not return results for protein HGVS strings like - NP_009225.1:p.Val1696His, so these must be recoded via Variant Recoder first. This fixture - exercises the recoder fallback path end-to-end. + VEP's /vep/human/hgvs endpoint returns no consequence for protein HGVS like + NP_009225.1:p.Val1696His, so the job must fall back to Variant Recoder. ``hgvs_g``/``hgvs_c`` are + left unset so the VEP payload resolves to ``hgvs_p``, exercising the recoder fallback path. """ score_set = session.get(ScoreSet, sample_populate_vep_run.job_params["score_set_id"]) @@ -1382,17 +1402,61 @@ def setup_sample_protein_variant_for_vep(session, with_populated_domain_data, mo hgvs_pro="NP_009225.1:p.Val1696His", data={"hgvs_p": "NP_009225.1:p.Val1696His"}, ) - session.add(variant) + allele = Allele( + vrs_digest="test-vep-protein-allele-vrs-digest", + level="protein", + hgvs_p="NP_009225.1:p.Val1696His", + post_mapped={"type": "Allele", "expressions": [{"value": "NP_009225.1:p.Val1696His", "syntax": "hgvs.p"}]}, + ) + session.add_all([variant, allele]) session.commit() - mapped_variant = MappedVariant( + mapping_record = MappingRecord( variant_id=variant.id, - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - post_mapped={"type": "Allele", "expressions": [{"value": "NP_009225.1:p.Val1696His", "syntax": "hgvs.p"}]}, - hgvs_assay_level="NP_009225.1:p.Val1696His", + assay_level="protein", + mapping_api_version="pytest.0.0", ) - session.add(mapped_variant) + session.add(mapping_record) session.commit() - return variant, mapped_variant + + session.add( + MappingRecordAllele( + mapping_record_id=mapping_record.id, + allele_id=allele.id, + is_authoritative=True, + ) + ) + session.commit() + return variant, allele + + +@pytest.fixture +def setup_rt_derived_allele_for_vep(session, setup_sample_alleles_for_vep): + """Add a NON-authoritative (RT-derived) allele to the variant's current mapping record, carrying a + genomic HGVS of its own. Isolates the requirement that VEP linkage covers the full allele set + (authoritative + RT-derived), while the per-variant VAS fan-out stays authoritative-only. + """ + variant, authoritative_allele = setup_sample_alleles_for_vep + + mapping_record = session.scalars( + select(MappingRecord).where(MappingRecord.variant_id == variant.id, MappingRecord.current) + ).one() + + rt_allele = Allele( + vrs_digest="test-rt-derived-vep-allele-vrs-digest", + level="genomic", + hgvs_g="NC_000017.11:g.43124027T>C", + post_mapped={"type": "Allele", "expressions": [{"value": "NC_000017.11:g.43124027T>C", "syntax": "hgvs.g"}]}, + ) + session.add(rt_allele) + session.commit() + + session.add( + MappingRecordAllele( + mapping_record_id=mapping_record.id, + allele_id=rt_allele.id, + is_authoritative=False, + ) + ) + session.commit() + return variant, authoritative_allele, rt_allele diff --git a/tests/worker/jobs/external_services/network/test_vep.py b/tests/worker/jobs/external_services/network/test_vep.py index 61071fc42..0a52012af 100644 --- a/tests/worker/jobs/external_services/network/test_vep.py +++ b/tests/worker/jobs/external_services/network/test_vep.py @@ -9,6 +9,7 @@ from mavedb.models.enums.annotation_type import AnnotationType from mavedb.models.enums.job_pipeline import AnnotationStatus, JobStatus, PipelineStatus from mavedb.models.variant_annotation_status import VariantAnnotationStatus +from mavedb.models.vep_allele_consequence import VepAlleleConsequence pytestmark = pytest.mark.usefixtures("patch_db_session_ctxmgr") @@ -27,10 +28,10 @@ async def test_populate_vep_e2e( arq_worker, sample_populate_vep_run_pipeline, sample_populate_vep_pipeline, - setup_sample_variants_for_vep, + setup_sample_alleles_for_vep, ): - """Enqueue the VEP job, run the worker, and verify consequence and annotation are populated.""" - _, mapped_variant = setup_sample_variants_for_vep + """Enqueue the VEP job, run the worker, and verify the allele's consequence and annotation.""" + variant, allele = setup_sample_alleles_for_vep await arq_redis.enqueue_job("populate_vep_for_score_set", sample_populate_vep_run_pipeline.id) await arq_worker.async_run() @@ -42,13 +43,18 @@ async def test_populate_vep_e2e( session.refresh(sample_populate_vep_pipeline) assert sample_populate_vep_pipeline.status == PipelineStatus.SUCCEEDED - session.refresh(mapped_variant) - assert mapped_variant.vep_functional_consequence is not None - assert mapped_variant.vep_access_date is not None + live = session.scalars( + select(VepAlleleConsequence).where( + VepAlleleConsequence.allele_id == allele.id, + VepAlleleConsequence.current, + ) + ).one() + assert live.functional_consequence is not None + assert live.access_date is not None annotation = session.scalars( select(VariantAnnotationStatus).where( - VariantAnnotationStatus.variant_id == mapped_variant.variant_id, + VariantAnnotationStatus.variant_id == variant.id, VariantAnnotationStatus.annotation_type == AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, VariantAnnotationStatus.current.is_(True), ) @@ -62,7 +68,7 @@ async def test_populate_vep_e2e_with_recoder_path( arq_worker, sample_populate_vep_run_pipeline, sample_populate_vep_pipeline, - setup_sample_protein_variant_for_vep, + setup_sample_protein_allele_for_vep, ): """VEP job uses Variant Recoder for a protein HGVS (NP_ accession) that VEP cannot resolve directly. @@ -70,7 +76,7 @@ async def test_populate_vep_e2e_with_recoder_path( does not return a consequence for. The job must fall back to Variant Recoder, recode it to a genomic HGVS, and then re-query VEP with the recoded string. """ - _, mapped_variant = setup_sample_protein_variant_for_vep + variant, allele = setup_sample_protein_allele_for_vep await arq_redis.enqueue_job("populate_vep_for_score_set", sample_populate_vep_run_pipeline.id) await arq_worker.async_run() @@ -82,13 +88,18 @@ async def test_populate_vep_e2e_with_recoder_path( session.refresh(sample_populate_vep_pipeline) assert sample_populate_vep_pipeline.status == PipelineStatus.SUCCEEDED - session.refresh(mapped_variant) - assert mapped_variant.vep_functional_consequence is not None - assert mapped_variant.vep_access_date is not None + live = session.scalars( + select(VepAlleleConsequence).where( + VepAlleleConsequence.allele_id == allele.id, + VepAlleleConsequence.current, + ) + ).one() + assert live.functional_consequence is not None + assert live.access_date is not None annotation = session.scalars( select(VariantAnnotationStatus).where( - VariantAnnotationStatus.variant_id == mapped_variant.variant_id, + VariantAnnotationStatus.variant_id == variant.id, VariantAnnotationStatus.annotation_type == AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, VariantAnnotationStatus.current.is_(True), ) diff --git a/tests/worker/jobs/external_services/test_vep.py b/tests/worker/jobs/external_services/test_vep.py index 611c33894..7e33f24fa 100644 --- a/tests/worker/jobs/external_services/test_vep.py +++ b/tests/worker/jobs/external_services/test_vep.py @@ -4,29 +4,48 @@ pytest.importorskip("arq") +from datetime import date, timedelta from unittest.mock import patch from sqlalchemy import select from mavedb.lib.types.workflow import JobExecutionOutcome -from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationFailureCategory, AnnotationStatus, JobStatus -from mavedb.models.mapped_variant import MappedVariant -from mavedb.models.score_set import ScoreSet -from mavedb.models.variant import Variant +from mavedb.models.enums.job_pipeline import JobStatus, PipelineStatus from mavedb.models.variant_annotation_status import VariantAnnotationStatus +from mavedb.models.vep_allele_consequence import VepAlleleConsequence from mavedb.worker.jobs.external_services.vep import populate_vep_for_score_set from mavedb.worker.lib.managers.job_manager import JobManager pytestmark = pytest.mark.usefixtures("patch_db_session_ctxmgr") +_RESOLVE = "mavedb.worker.jobs.external_services.vep._resolve_consequences" +_RELEASE = "mavedb.worker.jobs.external_services.vep.get_ensembl_release" +_ENSEMBL_RELEASE = "116" + + +@pytest.fixture(autouse=True) +def mock_ensembl_release(): + """Stamp every job run with a fixed Ensembl release so tests version-key deterministically without + hitting /info/software. Tests exercising a release-fetch failure override this with an inner patch.""" + with patch(_RELEASE, return_value=_ENSEMBL_RELEASE): + yield + + +def _live_consequences_for(session, allele_id): + return session.scalars( + select(VepAlleleConsequence).where( + VepAlleleConsequence.allele_id == allele_id, + VepAlleleConsequence.current, + ) + ).all() + @pytest.mark.asyncio @pytest.mark.unit class TestPopulateVepForScoreSetUnit: - """Unit tests for populate_vep_for_score_set.""" + """Unit tests for the populate_vep_for_score_set job.""" - async def test_no_mapped_variants( + async def test_no_alleles_with_hgvs( self, session, with_populated_domain_data, @@ -34,7 +53,7 @@ async def test_no_mapped_variants( mock_worker_ctx, sample_populate_vep_run, ): - """Job succeeds with zero counts when no mapped variants exist.""" + """No alleles with HGVS -> the job succeeds with nothing to do.""" result = await populate_vep_for_score_set( mock_worker_ctx, 1, @@ -43,405 +62,411 @@ async def test_no_mapped_variants( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - assert result.data["variants_processed"] == 0 - assert result.data["variants_with_consequences"] == 0 - assert result.data["variants_recoder_failed"] == 0 + assert result.data["created_allele_count"] == 0 + assert result.data["preexisting_allele_count"] == 0 + assert result.data["skipped_allele_count"] == 0 - async def test_variant_without_hgvs_assay_level_skipped( + async def test_calls_resolver_when_alleles_present( self, session, with_populated_domain_data, with_populate_vep_job, mock_worker_ctx, sample_populate_vep_run, - setup_sample_variants_for_vep, + setup_sample_alleles_for_vep, ): - """A mapped variant with no hgvs_assay_level gets a SKIPPED annotation.""" - _, mapped_variant = setup_sample_variants_for_vep - mapped_variant.hgvs_assay_level = None - session.commit() - - result = await populate_vep_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_vep_run.id), - ) + """The VEP resolver is invoked once when the score set has HGVS-bearing alleles to query.""" + with patch(_RESOLVE, return_value={}) as mock_resolve: + result = await populate_vep_for_score_set( + mock_worker_ctx, + 1, + JobManager(session, mock_worker_ctx["redis"], sample_populate_vep_run.id), + ) assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - assert result.data["variants_processed"] == 0 + mock_resolve.assert_called_once() - annotation = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.variant_id == mapped_variant.variant_id, - VariantAnnotationStatus.annotation_type == AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, - ) - ).one() - assert annotation.status == AnnotationStatus.SKIPPED - assert annotation.failure_category == AnnotationFailureCategory.MISSING_IDENTIFIER - - async def test_vep_api_success_sets_consequence_and_annotation( + async def test_propagates_exceptions( self, session, with_populated_domain_data, with_populate_vep_job, mock_worker_ctx, sample_populate_vep_run, - setup_sample_variants_for_vep, + setup_sample_alleles_for_vep, ): - """VEP returns a consequence: mapped variant and SUCCESS annotation are updated.""" - _, mapped_variant = setup_sample_variants_for_vep - hgvs = mapped_variant.hgvs_assay_level - - with patch( - "mavedb.worker.jobs.external_services.vep.get_functional_consequence", - return_value={hgvs: "missense_variant"}, + """Exceptions raised while resolving consequences are propagated.""" + with patch(_RESOLVE, side_effect=Exception("Test exception")): + with pytest.raises(Exception) as exc_info: + await populate_vep_for_score_set( + mock_worker_ctx, + 1, + JobManager(session, mock_worker_ctx["redis"], sample_populate_vep_run.id), + ) + + assert str(exc_info.value) == "Test exception" + + async def test_aborts_when_release_unavailable( + self, + session, + with_populated_domain_data, + with_populate_vep_job, + mock_worker_ctx, + sample_populate_vep_run, + setup_sample_alleles_for_vep, + ): + """If the Ensembl release cannot be fetched the job aborts rather than mis-versioning its writes + — the version is load-bearing for the skip, so a failure must propagate, not be swallowed.""" + with ( + patch(_RELEASE, side_effect=Exception("info/software unavailable")), + patch(_RESOLVE) as mock_resolve, ): - result = await populate_vep_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_vep_run.id), - ) + with pytest.raises(Exception) as exc_info: + await populate_vep_for_score_set( + mock_worker_ctx, + 1, + JobManager(session, mock_worker_ctx["redis"], sample_populate_vep_run.id), + ) + + assert str(exc_info.value) == "info/software unavailable" + mock_resolve.assert_not_called() # never queried VEP without a version to stamp + +@pytest.mark.asyncio +@pytest.mark.integration +class TestPopulateVepForScoreSetIntegration: + """Integration tests for the populate_vep_for_score_set job.""" + + async def test_no_alleles_with_hgvs( + self, + session, + with_populated_domain_data, + with_populate_vep_job, + mock_worker_ctx, + sample_populate_vep_run, + ): + """End-to-end: no alleles -> no consequence rows, no annotations, job succeeds.""" + result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - assert result.data["variants_processed"] == 1 - assert result.data["variants_with_consequences"] == 1 - assert result.data["variants_recoder_failed"] == 0 - - session.refresh(mapped_variant) - assert mapped_variant.vep_functional_consequence == "missense_variant" - assert mapped_variant.vep_access_date is not None - - annotation = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.variant_id == mapped_variant.variant_id, - VariantAnnotationStatus.annotation_type == AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, - ) - ).one() - assert annotation.status == AnnotationStatus.SUCCESS - async def test_vep_missing_triggers_variant_recoder_fallback( + assert len(session.scalars(select(VepAlleleConsequence)).all()) == 0 + assert len(session.query(VariantAnnotationStatus).all()) == 0 + + session.refresh(sample_populate_vep_run) + assert sample_populate_vep_run.status == JobStatus.SUCCEEDED + + async def test_no_consequence_resolved( self, session, with_populated_domain_data, with_populate_vep_job, mock_worker_ctx, sample_populate_vep_run, - setup_sample_variants_for_vep, + setup_sample_alleles_for_vep, ): - """When VEP misses a variant, Variant Recoder is called and its result fed back to VEP.""" - _, mapped_variant = setup_sample_variants_for_vep - hgvs = mapped_variant.hgvs_assay_level - genomic_hgvs = "NC_000017.11:g.43094692C>T" + """An allele with HGVS that VEP cannot classify gets no consequence row and a SKIPPED VAS.""" + _, allele = setup_sample_alleles_for_vep - with ( - patch( - "mavedb.worker.jobs.external_services.vep.get_functional_consequence", - side_effect=[ - {}, # initial VEP pass returns nothing - {genomic_hgvs: "missense_variant"}, # second VEP pass on recoded HGVS - ], - ), - patch( - "mavedb.worker.jobs.external_services.vep.run_variant_recoder", - return_value={hgvs: [genomic_hgvs]}, - ), - ): - result = await populate_vep_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_vep_run.id), - ) + with patch(_RESOLVE, return_value={}): + result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) - assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - assert result.data["variants_with_consequences"] == 1 - assert result.data["variants_recoder_failed"] == 0 + assert len(_live_consequences_for(session, allele.id)) == 0 - session.refresh(mapped_variant) - assert mapped_variant.vep_functional_consequence == "missense_variant" + annotation_statuses = session.query(VariantAnnotationStatus).all() + assert len(annotation_statuses) == 1 + assert annotation_statuses[0].status == "skipped" + assert annotation_statuses[0].annotation_type == "vep_functional_consequence" - async def test_variant_recoder_failure_annotated_as_failed( + async def test_successful_linking_independent( self, session, with_populated_domain_data, with_populate_vep_job, mock_worker_ctx, sample_populate_vep_run, - setup_sample_variants_for_vep, + setup_sample_alleles_for_vep, ): - """Variant Recoder returning no result for an HGVS produces a FAILED annotation.""" - _, mapped_variant = setup_sample_variants_for_vep + """A resolved consequence creates a single live VepAlleleConsequence and a SUCCESS VAS row.""" + _, allele = setup_sample_alleles_for_vep - with ( - patch( - "mavedb.worker.jobs.external_services.vep.get_functional_consequence", - return_value={}, - ), - patch( - "mavedb.worker.jobs.external_services.vep.run_variant_recoder", - return_value={}, # recoder has no result - ), - ): - result = await populate_vep_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_vep_run.id), - ) + with patch(_RESOLVE, return_value={allele.hgvs_c: "missense_variant"}): + result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) - assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - assert result.data["variants_without_consequences"] == 0 - assert result.data["variants_recoder_failed"] == 1 + assert result.data["created_allele_count"] == 1 - annotation = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.variant_id == mapped_variant.variant_id, - VariantAnnotationStatus.annotation_type == AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, - ) - ).one() - assert annotation.status == AnnotationStatus.FAILED - assert annotation.failure_category == AnnotationFailureCategory.EXTERNAL_REFERENCE_NOT_FOUND + live = _live_consequences_for(session, allele.id) + assert len(live) == 1 + assert live[0].functional_consequence == "missense_variant" + assert live[0].source_version == _ENSEMBL_RELEASE + assert live[0].access_date == date.today() - async def test_vep_failure_after_recoder_annotated_as_failed( + annotation_statuses = session.query(VariantAnnotationStatus).all() + assert len(annotation_statuses) == 1 + assert annotation_statuses[0].status == "success" + assert annotation_statuses[0].annotation_type == "vep_functional_consequence" + assert annotation_statuses[0].annotation_metadata["action"] == "created" + + async def test_links_rt_derived_allele_but_annotates_only_authoritative( self, session, with_populated_domain_data, with_populate_vep_job, mock_worker_ctx, sample_populate_vep_run, - setup_sample_variants_for_vep, + setup_rt_derived_allele_for_vep, ): - """VEP returning no consequence even after Variant Recoder produces a FAILED annotation.""" - _, mapped_variant = setup_sample_variants_for_vep - hgvs = mapped_variant.hgvs_assay_level - genomic_hgvs = "NC_000017.11:g.43094692C>T" + """VEP linkage must cover the full allele set, not just authoritative links: the RT-derived + allele's genomic HGVS resolves and must get a consequence row. Per-variant VAS, however, is + written only for the authoritative link (the interim bandaid) — so exactly one annotation row + is produced, keyed to the variant, never a second row for the RT-derived allele.""" + variant, authoritative_allele, rt_allele = setup_rt_derived_allele_for_vep - with ( - patch( - "mavedb.worker.jobs.external_services.vep.get_functional_consequence", - return_value={}, # VEP returns nothing in both passes - ), - patch( - "mavedb.worker.jobs.external_services.vep.run_variant_recoder", - return_value={hgvs: [genomic_hgvs]}, - ), - ): - result = await populate_vep_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_vep_run.id), - ) + # VEP resolves only the RT-derived allele's genomic HGVS; the authoritative allele's coding + # HGVS yields nothing this run. + with patch(_RESOLVE, return_value={rt_allele.hgvs_g: "missense_variant"}): + result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) - assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - assert result.data["variants_without_consequences"] == 1 - assert result.data["variants_recoder_failed"] == 0 - annotation = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.variant_id == mapped_variant.variant_id, - VariantAnnotationStatus.annotation_type == AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, - ) - ).one() - assert annotation.status == AnnotationStatus.FAILED - assert annotation.failure_category == AnnotationFailureCategory.EXTERNAL_REFERENCE_NOT_FOUND - - async def test_vep_none_consequence_routes_to_recoder( + # The RT-derived (non-authoritative) allele IS linked — the core fix. + rt_live = _live_consequences_for(session, rt_allele.id) + assert len(rt_live) == 1 + assert rt_live[0].functional_consequence == "missense_variant" + # The authoritative allele's HGVS had no consequence, so it gets no row. + assert len(_live_consequences_for(session, authoritative_allele.id)) == 0 + + # Annotation status is written only for the authoritative link: exactly one VAS row, for the + # variant. (Its status is "skipped" because the variant's authoritative allele had no + # consequence — cross-level resolution onto the RT-derived allele is deferred to AnnotationEvent.) + annotation_statuses = session.query(VariantAnnotationStatus).all() + assert len(annotation_statuses) == 1 + assert annotation_statuses[0].variant_id == variant.id + assert annotation_statuses[0].annotation_type == "vep_functional_consequence" + + async def test_skips_allele_already_at_current_release( self, session, with_populated_domain_data, with_populate_vep_job, mock_worker_ctx, sample_populate_vep_run, - setup_sample_variants_for_vep, + setup_sample_alleles_for_vep, ): - """A None consequence from VEP Phase 1 is treated as a miss and routed through Recoder. + """An allele with a live consequence already at the current Ensembl release is skipped: no VEP + query, the status reports SUCCESS/preexisting, and the existing row is not churned.""" + _, allele = setup_sample_alleles_for_vep + + # Simulate a prior run at the current release. + session.add( + VepAlleleConsequence( + allele_id=allele.id, + functional_consequence="missense_variant", + source_version=_ENSEMBL_RELEASE, + access_date=date.today(), + ) + ) + session.commit() - VEP can return an entry with most_severe_consequence=None when it recognises a variant - but cannot classify it. This should not silently fall into the UNKNOWN outcome branch — - instead it should be treated identically to an absent entry and sent to Recoder. - """ - _, mapped_variant = setup_sample_variants_for_vep - hgvs = mapped_variant.hgvs_assay_level - genomic_hgvs = "NC_000017.11:g.43094692C>T" + with patch(_RESOLVE) as mock_resolve: + result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) - with ( - patch( - "mavedb.worker.jobs.external_services.vep.get_functional_consequence", - side_effect=[ - {hgvs: None}, # VEP knows the variant but returns no consequence - {genomic_hgvs: "missense_variant"}, # Phase 3 on recoded genomic string - ], - ), - patch( - "mavedb.worker.jobs.external_services.vep.run_variant_recoder", - return_value={hgvs: [genomic_hgvs]}, - ), - ): - result = await populate_vep_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_vep_run.id), - ) + mock_resolve.assert_not_called() # version-keyed skip avoided the external query entirely + assert result.data["preexisting_allele_count"] == 1 + assert result.data["created_allele_count"] == 0 - assert result.status == JobStatus.SUCCEEDED - assert result.data["variants_with_consequences"] == 1 + # Row not churned: still exactly one, still live. + rows = session.scalars(select(VepAlleleConsequence).where(VepAlleleConsequence.allele_id == allele.id)).all() + assert len(rows) == 1 + assert rows[0].valid_to is None - session.refresh(mapped_variant) - assert mapped_variant.vep_functional_consequence == "missense_variant" + annotation_statuses = session.query(VariantAnnotationStatus).all() + assert len(annotation_statuses) == 1 + assert annotation_statuses[0].status == "success" + assert annotation_statuses[0].annotation_metadata["action"] == "preexisting" - async def test_most_severe_consequence_selected_from_multiple_genomic_hgvs( + async def test_new_release_same_consequence_bumps_in_place( self, session, with_populated_domain_data, with_populate_vep_job, mock_worker_ctx, sample_populate_vep_run, - setup_sample_variants_for_vep, + setup_sample_alleles_for_vep, ): - """When Recoder returns multiple genomic strings with different consequences, the most severe wins.""" - _, mapped_variant = setup_sample_variants_for_vep - hgvs = mapped_variant.hgvs_assay_level - genomic_less_severe = "NC_000017.11:g.43094692C>T" # → missense_variant - genomic_more_severe = "NC_000017.11:g.43094692C>A" # → stop_gained - - with ( - patch( - "mavedb.worker.jobs.external_services.vep.get_functional_consequence", - side_effect=[ - {}, # Phase 1: VEP misses - { - genomic_less_severe: "missense_variant", - genomic_more_severe: "stop_gained", - }, # Phase 3: two different consequences - ], - ), - patch( - "mavedb.worker.jobs.external_services.vep.run_variant_recoder", - return_value={hgvs: [genomic_less_severe, genomic_more_severe]}, - ), - ): - result = await populate_vep_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_vep_run.id), + """An allele live at an older release is re-queried; an unchanged consequence advances + source_version (and access_date) in place — no supersede, so a new release does not churn + history for a categorical value that did not change.""" + _, allele = setup_sample_alleles_for_vep + + session.add( + VepAlleleConsequence( + allele_id=allele.id, + functional_consequence="missense_variant", + source_version="115", + access_date=date.today() - timedelta(days=90), ) + ) + session.commit() + + with patch(_RESOLVE, return_value={allele.hgvs_c: "missense_variant"}) as mock_resolve: + result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) - assert result.data["variants_with_consequences"] == 1 + mock_resolve.assert_called_once() # older release -> not skipped, re-queried + assert result.data["preexisting_allele_count"] == 1 + assert result.data["created_allele_count"] == 0 - session.refresh(mapped_variant) - assert mapped_variant.vep_functional_consequence == "stop_gained" + # Unchanged value -> no supersede: one row, still live, version + date advanced in place. + rows = session.scalars(select(VepAlleleConsequence).where(VepAlleleConsequence.allele_id == allele.id)).all() + assert len(rows) == 1 + assert rows[0].valid_to is None + assert rows[0].source_version == _ENSEMBL_RELEASE + assert rows[0].access_date == date.today() - async def test_multiple_variants_sharing_hgvs_all_get_consequence( + async def test_force_requeries_unchanged_without_churn( self, session, with_populated_domain_data, with_populate_vep_job, mock_worker_ctx, sample_populate_vep_run, - setup_sample_variants_for_vep, + setup_sample_alleles_for_vep, ): - """All mapped variants that share an HGVS string each receive the consequence.""" - _, mapped_variant_1 = setup_sample_variants_for_vep - hgvs = mapped_variant_1.hgvs_assay_level - - score_set = session.get(ScoreSet, sample_populate_vep_run.job_params["score_set_id"]) - variant_2 = Variant( - urn="urn:variant:test-variant-for-vep-2", - score_set_id=score_set.id, - hgvs_nt=hgvs, - data={"hgvs_c": hgvs}, + """force bypasses the current-release skip and re-queries, but the linker only supersedes on a + value change: a forced re-run that resolves the same consequence reports preexisting and does + not churn the row (version/access_date advanced in place).""" + _, allele = setup_sample_alleles_for_vep + + # Prior live consequence already at the current release (would be skipped without force). + session.add( + VepAlleleConsequence( + allele_id=allele.id, + functional_consequence="missense_variant", + source_version=_ENSEMBL_RELEASE, + access_date=date.today() - timedelta(days=1), + ) ) - session.add(variant_2) session.commit() - mapped_variant_2 = MappedVariant( - variant_id=variant_2.id, - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - post_mapped={"type": "Allele", "expressions": [{"value": hgvs, "syntax": "hgvs.c"}]}, - hgvs_assay_level=hgvs, - ) - session.add(mapped_variant_2) + + sample_populate_vep_run.job_params = {**sample_populate_vep_run.job_params, "force": True} session.commit() - with patch( - "mavedb.worker.jobs.external_services.vep.get_functional_consequence", - return_value={hgvs: "missense_variant"}, - ): - result = await populate_vep_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_vep_run.id), - ) + with patch(_RESOLVE, return_value={allele.hgvs_c: "missense_variant"}) as mock_resolve: + result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) - assert result.data["variants_processed"] == 2 - assert result.data["variants_with_consequences"] == 2 + mock_resolve.assert_called_once() # force bypassed the current-release skip + assert result.data["preexisting_allele_count"] == 1 + assert result.data["created_allele_count"] == 0 - session.refresh(mapped_variant_1) - session.refresh(mapped_variant_2) - assert mapped_variant_1.vep_functional_consequence == "missense_variant" - assert mapped_variant_2.vep_functional_consequence == "missense_variant" + # Unchanged consequence -> no supersede: one row, still live, access_date touched in place. + rows = session.scalars(select(VepAlleleConsequence).where(VepAlleleConsequence.allele_id == allele.id)).all() + assert len(rows) == 1 + assert rows[0].valid_to is None + assert rows[0].access_date == date.today() - async def test_vep_batch_api_exception_raises( + async def test_new_release_changed_consequence_supersedes( self, session, with_populated_domain_data, with_populate_vep_job, mock_worker_ctx, sample_populate_vep_run, - setup_sample_variants_for_vep, + setup_sample_alleles_for_vep, ): - """An unexpected exception from the VEP API propagates to the job management decorator.""" - with ( - patch( - "mavedb.worker.jobs.external_services.vep.get_functional_consequence", - side_effect=RuntimeError("VEP API unreachable"), - ), - pytest.raises(RuntimeError, match="VEP API unreachable"), - ): - await populate_vep_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_vep_run.id), + """An allele live at an older release is re-queried; a changed result supersedes it — one live + row carrying the new consequence/release and one retired row preserving the old.""" + _, allele = setup_sample_alleles_for_vep + + # Prior live consequence at an older release -> not skipped, eligible for re-query. + session.add( + VepAlleleConsequence( + allele_id=allele.id, + functional_consequence="synonymous_variant", + source_version="115", + access_date=date.today() - timedelta(days=90), ) + ) + session.commit() + + with patch(_RESOLVE, return_value={allele.hgvs_c: "missense_variant"}): + result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) + + assert result.data["created_allele_count"] == 1 + + live = _live_consequences_for(session, allele.id) + assert len(live) == 1 + assert live[0].functional_consequence == "missense_variant" + assert live[0].source_version == _ENSEMBL_RELEASE - async def test_variant_recoder_api_exception_raises( + # Old consequence retired, not deleted. + all_rows = session.scalars( + select(VepAlleleConsequence).where(VepAlleleConsequence.allele_id == allele.id) + ).all() + assert len(all_rows) == 2 + assert len([r for r in all_rows if r.valid_to is not None]) == 1 + + async def test_successful_linking_pipeline( + self, + session, + with_populated_domain_data, + mock_worker_ctx, + sample_populate_vep_run_pipeline, + sample_populate_vep_pipeline, + setup_sample_alleles_for_vep, + ): + """End-to-end successful linking within a pipeline updates both job and pipeline status.""" + _, allele = setup_sample_alleles_for_vep + + with patch(_RESOLVE, return_value={allele.hgvs_c: "missense_variant"}): + result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run_pipeline.id) + + assert result.status == JobStatus.SUCCEEDED + assert len(session.scalars(select(VepAlleleConsequence).where(VepAlleleConsequence.current)).all()) == 1 + + annotation_statuses = session.query(VariantAnnotationStatus).all() + assert len(annotation_statuses) == 1 + assert annotation_statuses[0].status == "success" + assert annotation_statuses[0].annotation_type == "vep_functional_consequence" + + session.refresh(sample_populate_vep_run_pipeline) + assert sample_populate_vep_run_pipeline.status == JobStatus.SUCCEEDED + session.refresh(sample_populate_vep_pipeline) + assert sample_populate_vep_pipeline.status == PipelineStatus.SUCCEEDED + + async def test_exceptions_handled_by_decorators( self, session, with_populated_domain_data, with_populate_vep_job, mock_worker_ctx, sample_populate_vep_run, - setup_sample_variants_for_vep, + setup_sample_alleles_for_vep, ): - """An unexpected exception from the Variant Recoder API propagates to the job management decorator.""" + """Exceptions during resolution are handled by the job decorators (Slack alert, ERRORED).""" with ( - patch( - "mavedb.worker.jobs.external_services.vep.get_functional_consequence", - return_value={}, - ), - patch( - "mavedb.worker.jobs.external_services.vep.run_variant_recoder", - side_effect=RuntimeError("Recoder API unreachable"), - ), - pytest.raises(RuntimeError, match="Recoder API unreachable"), + patch(_RESOLVE, side_effect=Exception("Test exception")), + patch("mavedb.worker.lib.decorators.job_management.send_slack_job_error") as mock_send_slack_job_error, ): - await populate_vep_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_vep_run.id), - ) + result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) + + mock_send_slack_job_error.assert_called_once() + assert isinstance(result, JobExecutionOutcome) + assert result.status == JobStatus.ERRORED + assert isinstance(result.exception, Exception) + + session.refresh(sample_populate_vep_run) + assert sample_populate_vep_run.status == JobStatus.ERRORED @pytest.mark.asyncio @pytest.mark.integration -class TestPopulateVepForScoreSetIntegration: - """Integration tests for populate_vep_for_score_set run through an ARQ worker context.""" +class TestPopulateVepForScoreSetArqContext: + """Tests for the populate_vep_for_score_set job using the ARQ context fixture.""" - async def test_populate_vep_with_arq_context( + async def test_populate_vep_with_arq_context_independent( self, arq_redis, arq_worker, @@ -449,35 +474,27 @@ async def test_populate_vep_with_arq_context( with_populated_domain_data, with_populate_vep_job, sample_populate_vep_run, - setup_sample_variants_for_vep, + setup_sample_alleles_for_vep, ): - """Job completes successfully within an ARQ worker context.""" - _, mapped_variant = setup_sample_variants_for_vep - hgvs = mapped_variant.hgvs_assay_level + """The VEP job links a consequence and records a SUCCESS annotation through the ARQ worker.""" + _, allele = setup_sample_alleles_for_vep - with patch( - "mavedb.worker.jobs.external_services.vep.get_functional_consequence", - return_value={hgvs: "missense_variant"}, - ): + with patch(_RESOLVE, return_value={allele.hgvs_c: "missense_variant"}): await arq_redis.enqueue_job("populate_vep_for_score_set", sample_populate_vep_run.id) await arq_worker.async_run() await arq_worker.run_check() - session.refresh(sample_populate_vep_run) - assert sample_populate_vep_run.status == JobStatus.SUCCEEDED + assert len(session.scalars(select(VepAlleleConsequence).where(VepAlleleConsequence.current)).all()) == 1 - session.refresh(mapped_variant) - assert mapped_variant.vep_functional_consequence == "missense_variant" + annotation_statuses = session.query(VariantAnnotationStatus).all() + assert len(annotation_statuses) == 1 + assert annotation_statuses[0].status == "success" + assert annotation_statuses[0].annotation_type == "vep_functional_consequence" - annotation = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.variant_id == mapped_variant.variant_id, - VariantAnnotationStatus.annotation_type == AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, - ) - ).one() - assert annotation.status == AnnotationStatus.SUCCESS + session.refresh(sample_populate_vep_run) + assert sample_populate_vep_run.status == JobStatus.SUCCEEDED - async def test_populate_vep_in_pipeline_context( + async def test_populate_vep_with_arq_context_pipeline( self, arq_redis, arq_worker, @@ -485,24 +502,73 @@ async def test_populate_vep_in_pipeline_context( with_populated_domain_data, sample_populate_vep_run_pipeline, sample_populate_vep_pipeline, - setup_sample_variants_for_vep, + setup_sample_alleles_for_vep, ): - """Job completes and advances the pipeline when run in a pipeline context.""" - from mavedb.models.enums.job_pipeline import PipelineStatus + """The VEP job completes and advances the pipeline through the ARQ worker.""" + _, allele = setup_sample_alleles_for_vep - _, mapped_variant = setup_sample_variants_for_vep - hgvs = mapped_variant.hgvs_assay_level - - with patch( - "mavedb.worker.jobs.external_services.vep.get_functional_consequence", - return_value={hgvs: "synonymous_variant"}, - ): + with patch(_RESOLVE, return_value={allele.hgvs_c: "missense_variant"}): await arq_redis.enqueue_job("populate_vep_for_score_set", sample_populate_vep_run_pipeline.id) await arq_worker.async_run() await arq_worker.run_check() + assert len(session.scalars(select(VepAlleleConsequence).where(VepAlleleConsequence.current)).all()) == 1 + session.refresh(sample_populate_vep_run_pipeline) assert sample_populate_vep_run_pipeline.status == JobStatus.SUCCEEDED - session.refresh(sample_populate_vep_pipeline) assert sample_populate_vep_pipeline.status == PipelineStatus.SUCCEEDED + + async def test_populate_vep_with_arq_context_exception_handling_independent( + self, + arq_redis, + arq_worker, + session, + with_populated_domain_data, + with_populate_vep_job, + sample_populate_vep_run, + setup_sample_alleles_for_vep, + ): + """Exceptions in the VEP job are handled with the ARQ context fixture.""" + with ( + patch(_RESOLVE, side_effect=Exception("Test exception")), + patch("mavedb.worker.lib.decorators.job_management.send_slack_job_error") as mock_send_slack_job_error, + ): + await arq_redis.enqueue_job("populate_vep_for_score_set", sample_populate_vep_run.id) + await arq_worker.async_run() + await arq_worker.run_check() + + mock_send_slack_job_error.assert_called_once() + assert len(session.scalars(select(VepAlleleConsequence)).all()) == 0 + assert len(session.query(VariantAnnotationStatus).all()) == 0 + + session.refresh(sample_populate_vep_run) + assert sample_populate_vep_run.status == JobStatus.ERRORED + + async def test_populate_vep_with_arq_context_exception_handling_pipeline( + self, + arq_redis, + arq_worker, + session, + with_populated_domain_data, + sample_populate_vep_pipeline, + sample_populate_vep_run_pipeline, + setup_sample_alleles_for_vep, + ): + """Exceptions in the VEP job fail the pipeline with the ARQ context fixture.""" + with ( + patch(_RESOLVE, side_effect=Exception("Test exception")), + patch("mavedb.worker.lib.decorators.job_management.send_slack_job_error") as mock_send_slack_job_error, + ): + await arq_redis.enqueue_job("populate_vep_for_score_set", sample_populate_vep_run_pipeline.id) + await arq_worker.async_run() + await arq_worker.run_check() + + mock_send_slack_job_error.assert_called_once() + assert len(session.scalars(select(VepAlleleConsequence)).all()) == 0 + assert len(session.query(VariantAnnotationStatus).all()) == 0 + + session.refresh(sample_populate_vep_run_pipeline) + assert sample_populate_vep_run_pipeline.status == JobStatus.ERRORED + session.refresh(sample_populate_vep_pipeline) + assert sample_populate_vep_pipeline.status == PipelineStatus.FAILED From 62476807d001c7dffee473331cedc5d68c9bb362 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Mon, 22 Jun 2026 16:52:47 -0700 Subject: [PATCH 031/252] feat(clinvar): migrate ClinVar annotation onto the allele model Step 3 of the #742 external-annotation migration. ClinVar linkage moves off MappedVariant onto the deduplicated allele model. - Rename clinical_controls -> clinvar_controls (ORM model + table + unique constraint). Internal only: the ClinicalControl* view models, the /clinical-controls serving endpoints, and the frozen mapped_variants_clinical_controls association are unchanged, since the view-model name is both the OpenAPI schema name and the record_type discriminator consumed by the UI. - New clinvar_allele_links ValidTime table + ClinvarAlleleLink model. Multi-live: partial unique index (allele_id, clinvar_control_id) WHERE valid_to IS NULL, so an allele accumulates one live link per release. - Refactor refresh_clinvar_controls onto get_alleles_for_score_set + group_alleles_for_annotation (payload = CAID, full allele scope). Links are get-or-create; a same-version re-resolution to a different control supersedes newest-wins (gap-free retire+insert) rather than leaving two live links. VAS writes funnel through the _annotate_clinvar choke point, fanned only to authoritative_variant_ids and version-scoped. - Additively capture ClinVar's VariationID: nullable clinvar_variation_id column populated forward from the variant_summary TSV (parse degrades to None on archival schemas lacking the column). Unserved; the dedicated clinvar_variants remodel is deferred to the read-cutover. Tests rewritten for the allele model, covering the multi-live link writes, the version-scoped supersede guard, and the authoritative-only VAS fan-out. --- ...e_clinical_controls_to_clinvar_controls.py | 39 + .../c3d4e5f6a7b8_add_clinvar_allele_links.py | 73 + .../d4e5f6a7b8c9_add_clinvar_variation_id.py | 29 + src/mavedb/lib/clinvar/constants.py | 6 +- src/mavedb/lib/clinvar/utils.py | 5 +- src/mavedb/lib/score_sets.py | 24 +- src/mavedb/models/__init__.py | 1 + src/mavedb/models/clinical_control.py | 20 +- .../models/clinical_control_mapped_variant.py | 2 +- src/mavedb/models/clinvar_allele_link.py | 65 + src/mavedb/models/mapped_variant.py | 6 +- src/mavedb/routers/score_sets.py | 20 +- .../worker/jobs/external_services/clinvar.py | 346 ++-- tests/helpers/util/score_set.py | 2 +- tests/routers/conftest.py | 6 +- .../external_services/network/test_clinvar.py | 13 +- .../jobs/external_services/test_clinvar.py | 1413 +++-------------- 17 files changed, 734 insertions(+), 1336 deletions(-) create mode 100644 alembic/versions/b2c3d4e5f6a7_rename_clinical_controls_to_clinvar_controls.py create mode 100644 alembic/versions/c3d4e5f6a7b8_add_clinvar_allele_links.py create mode 100644 alembic/versions/d4e5f6a7b8c9_add_clinvar_variation_id.py create mode 100644 src/mavedb/models/clinvar_allele_link.py diff --git a/alembic/versions/b2c3d4e5f6a7_rename_clinical_controls_to_clinvar_controls.py b/alembic/versions/b2c3d4e5f6a7_rename_clinical_controls_to_clinvar_controls.py new file mode 100644 index 000000000..19cca0789 --- /dev/null +++ b/alembic/versions/b2c3d4e5f6a7_rename_clinical_controls_to_clinvar_controls.py @@ -0,0 +1,39 @@ +"""rename clinical_controls to clinvar_controls + +Revision ID: b2c3d4e5f6a7 +Revises: a7c4e9d2f1b8 +Create Date: 2026-06-22 + +Renames the clinical_controls entity table to clinvar_controls, and renames the unique +constraint to match. The frozen association table (mapped_variants_clinical_controls) +and its FK to the renamed table are left structurally intact — PostgreSQL updates the FK +target automatically on table rename. The Python model is renamed ClinicalControl → +ClinvarControl in the same changeset (no data migration). +""" + +from alembic import op + +revision = "b2c3d4e5f6a7" +down_revision = "a7c4e9d2f1b8" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.rename_table("clinical_controls", "clinvar_controls") + # PostgreSQL does not auto-rename constraints on table rename; rename explicitly so the + # on_conflict_do_update(constraint=...) in the job references the correct name. + op.execute( + "ALTER TABLE clinvar_controls RENAME CONSTRAINT " + "uq_clinical_controls_db_name_identifier_version " + "TO uq_clinvar_controls_db_name_identifier_version" + ) + + +def downgrade() -> None: + op.execute( + "ALTER TABLE clinvar_controls RENAME CONSTRAINT " + "uq_clinvar_controls_db_name_identifier_version " + "TO uq_clinical_controls_db_name_identifier_version" + ) + op.rename_table("clinvar_controls", "clinical_controls") diff --git a/alembic/versions/c3d4e5f6a7b8_add_clinvar_allele_links.py b/alembic/versions/c3d4e5f6a7b8_add_clinvar_allele_links.py new file mode 100644 index 000000000..4a20f008f --- /dev/null +++ b/alembic/versions/c3d4e5f6a7b8_add_clinvar_allele_links.py @@ -0,0 +1,73 @@ +"""add clinvar_allele_links table + +Revision ID: c3d4e5f6a7b8 +Revises: b2c3d4e5f6a7 +Create Date: 2026-06-22 + +New valid-time link table connecting deduplicated alleles to ClinvarControl rows, replacing +the frozen mapped_variants_clinical_controls association for new-model writes. + +ClinVar's link shape is deliberately multi-live: the partial unique index is +(allele_id, clinvar_control_id) WHERE valid_to IS NULL, so an allele accumulates one live +link per ClinVar release rather than superseding as in gnomAD/VEP. Each ClinVar release is a +distinct ClinvarControl row, so different releases stack as independent live links. A link is +retired (valid_to closed) only if ClinVar later removes the variant from a release, which +would surface as a re-run finding no data for that release and retiring the corresponding +link — archival data never changes, so this path is theoretical. + +The existing mapped_variants_clinical_controls association table is left untouched (frozen +for serving existing data). +""" + +import sqlalchemy as sa + +from alembic import op + +revision = "c3d4e5f6a7b8" +down_revision = "b2c3d4e5f6a7" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "clinvar_allele_links", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("allele_id", sa.Integer(), nullable=False), + sa.Column("clinvar_control_id", sa.Integer(), nullable=False), + sa.Column("valid_from", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + sa.Column("valid_to", sa.DateTime(timezone=True), nullable=True), + sa.ForeignKeyConstraint( + ["allele_id"], + ["alleles.id"], + name="fk_clinvar_allele_links_allele_id", + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["clinvar_control_id"], + ["clinvar_controls.id"], + name="fk_clinvar_allele_links_clinvar_control_id", + ondelete="RESTRICT", + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index("ix_clinvar_allele_links_allele_id", "clinvar_allele_links", ["allele_id"]) + op.create_index("ix_clinvar_allele_links_clinvar_control_id", "clinvar_allele_links", ["clinvar_control_id"]) + # Multi-live: one live link per (allele, release). An allele accumulates one live link per + # ClinVar release rather than superseding — unlike gnomAD/VEP which enforce one live link + # per allele across all versions. Superseded rows (valid_to IS NOT NULL) are preserved for + # point-in-time queries. + op.create_index( + "uq_clinvar_allele_links_live", + "clinvar_allele_links", + ["allele_id", "clinvar_control_id"], + unique=True, + postgresql_where=sa.text("valid_to IS NULL"), + ) + + +def downgrade() -> None: + op.drop_index("uq_clinvar_allele_links_live", table_name="clinvar_allele_links") + op.drop_index("ix_clinvar_allele_links_clinvar_control_id", table_name="clinvar_allele_links") + op.drop_index("ix_clinvar_allele_links_allele_id", table_name="clinvar_allele_links") + op.drop_table("clinvar_allele_links") diff --git a/alembic/versions/d4e5f6a7b8c9_add_clinvar_variation_id.py b/alembic/versions/d4e5f6a7b8c9_add_clinvar_variation_id.py new file mode 100644 index 000000000..de5758b4c --- /dev/null +++ b/alembic/versions/d4e5f6a7b8c9_add_clinvar_variation_id.py @@ -0,0 +1,29 @@ +"""add clinvar_variation_id to clinvar_controls + +Revision ID: d4e5f6a7b8c9 +Revises: c3d4e5f6a7b8 +Create Date: 2026-06-22 + +Additive, non-breaking column for ClinVar's canonical public identifier (VariationID), captured forward +from the variant_summary TSV. db_identifier continues to hold the AlleleID (the allele-level handle used +for gnomAD cross-references); this carries the VariationID beside it for eventual external ClinVar links +(clinvar/variation/{id}). Nullable and not yet served — the dedicated clinvar_variants remodel (explicit +fields replacing the generic db_* shape, the serving/UI cutover, and backfill of existing rows) is +deferred. +""" + +import sqlalchemy as sa +from alembic import op + +revision = "d4e5f6a7b8c9" +down_revision = "c3d4e5f6a7b8" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column("clinvar_controls", sa.Column("clinvar_variation_id", sa.String(), nullable=True)) + + +def downgrade() -> None: + op.drop_column("clinvar_controls", "clinvar_variation_id") diff --git a/src/mavedb/lib/clinvar/constants.py b/src/mavedb/lib/clinvar/constants.py index e70c4fee2..fd80212be 100644 --- a/src/mavedb/lib/clinvar/constants.py +++ b/src/mavedb/lib/clinvar/constants.py @@ -26,6 +26,10 @@ backoff for throttling is unnecessary — a modest retry with short backoff suffices. """ -CLINVAR_FIELDS_TO_KEEP = ("GeneSymbol", "ClinicalSignificance", "ReviewStatus") +CLINVAR_FIELDS_TO_KEEP = ("GeneSymbol", "ClinicalSignificance", "ReviewStatus", "VariationID") """Only these fields are extracted from each ClinVar TSV row and cached. The full TSV has ~30 columns; trimming to only what we need shrinks the cached pickle from hundreds of MB to tens of MB and speeds up load times. + +VariationID is ClinVar's canonical public identifier (anchors the web UI / variation links); we keep it +alongside the AlleleID (the row key) so the link record can carry both. A row missing the column on an +older archival TSV degrades to None rather than failing the whole version's parse. """ diff --git a/src/mavedb/lib/clinvar/utils.py b/src/mavedb/lib/clinvar/utils.py index 689e369ea..d72968da8 100644 --- a/src/mavedb/lib/clinvar/utils.py +++ b/src/mavedb/lib/clinvar/utils.py @@ -175,8 +175,11 @@ def _fetch_parse_and_cache( # as a list (which would be 1.5–2 GB for a modern TSV). with gzip.open(filename=buf, mode="rt") as f: reader = csv.DictReader(f, delimiter="\t") # type: ignore + # row.get (not row[field]) so a field absent from an older archival TSV schema yields + # None for that row rather than raising and discarding the whole version's parse. data: Dict[str, Dict[str, str]] = { - str(row["#AlleleID"]): {field: row[field] for field in CLINVAR_FIELDS_TO_KEEP} for row in reader + str(row["#AlleleID"]): {field: row.get(field) for field in CLINVAR_FIELDS_TO_KEEP} # type: ignore[misc] + for row in reader } finally: csv.field_size_limit(default_csv_field_size_limit) diff --git a/src/mavedb/lib/score_sets.py b/src/mavedb/lib/score_sets.py index 2acfa26f7..76cbab9e6 100644 --- a/src/mavedb/lib/score_sets.py +++ b/src/mavedb/lib/score_sets.py @@ -38,7 +38,7 @@ from mavedb.models.experiment_controlled_keyword import ExperimentControlledKeywordAssociation from mavedb.models.experiment_publication_identifier import ExperimentPublicationIdentifierAssociation from mavedb.models.experiment_set import ExperimentSet -from mavedb.models.clinical_control import ClinicalControl +from mavedb.models.clinical_control import ClinvarControl from mavedb.models.clinical_control_mapped_variant import mapped_variants_clinical_controls_association_table from mavedb.models.gnomad_variant import GnomADVariant from mavedb.models.mapped_variant import MappedVariant @@ -734,14 +734,14 @@ def get_score_set_variants_as_csv( idx = 2 if need_mappings else 1 gnomad_data.append(row[idx]) - # For each ClinVar namespace, fetch a mapping from mapped_variant_id to ClinicalControl. - clinvar_data_map: dict[str, dict[int, Optional[ClinicalControl]]] = {} + # For each ClinVar namespace, fetch a mapping from mapped_variant_id to ClinvarControl. + clinvar_data_map: dict[str, dict[int, Optional[ClinvarControl]]] = {} if clinvar_namespaces and mappings is not None: mv_ids = [m.id for m in mappings if m is not None] for ns, db_version in clinvar_namespaces.items(): - mv_to_cc: dict[int, Optional[ClinicalControl]] = {} + mv_to_cc: dict[int, Optional[ClinvarControl]] = {} if mv_ids: - aliased_cc = aliased(ClinicalControl) + aliased_cc = aliased(ClinvarControl) cc_query = ( select( mapped_variants_clinical_controls_association_table.c.mapped_variant_id, @@ -764,11 +764,11 @@ def get_score_set_variants_as_csv( clinvar_data_map[ns] = mv_to_cc # Build per-variant ClinVar lookup (list indexed in parallel with variants). - clinvar_per_variant: Optional[list[Optional[dict[str, Optional[ClinicalControl]]]]] = None + clinvar_per_variant: Optional[list[Optional[dict[str, Optional[ClinvarControl]]]]] = None if clinvar_namespaces and mappings is not None: clinvar_per_variant = [] for mapping in mappings: - row_clinvar: dict[str, Optional[ClinicalControl]] = {} + row_clinvar: dict[str, Optional[ClinvarControl]] = {} for ns, mv_to_cc in clinvar_data_map.items(): if mapping is not None and mapping.id is not None: row_clinvar[ns] = mv_to_cc.get(mapping.id) @@ -857,7 +857,7 @@ def variant_to_csv_row( columns: dict[str, list[str]], mapping: Optional[MappedVariant] = None, gnomad_data: Optional[GnomADVariant] = None, - clinvar_data_by_ns: Optional[dict[str, Optional[ClinicalControl]]] = None, + clinvar_data_by_ns: Optional[dict[str, Optional[ClinvarControl]]] = None, namespaced: Optional[bool] = None, na_rep="NA", ) -> dict[str, Any]: @@ -876,7 +876,7 @@ def variant_to_csv_row( Mapped variant corresponding to the variant. gnomad_data : variant.models.GnomADVariant, optional gnomAD variant data corresponding to the variant. - clinvar_data_by_ns : dict[str, Optional[ClinicalControl]], optional + clinvar_data_by_ns : dict[str, Optional[ClinvarControl]], optional Per-variant ClinVar data keyed by namespace (e.g. "clinvar.2024_01"). na_rep : str String to represent null values. @@ -1005,7 +1005,7 @@ def variants_to_csv_rows( columns: dict[str, list[str]], mappings: Optional[Sequence[Optional[MappedVariant]]] = None, gnomad_data: Optional[Sequence[Optional[GnomADVariant]]] = None, - clinvar_data_by_ns: Optional[Sequence[Optional[dict[str, Optional[ClinicalControl]]]]] = None, + clinvar_data_by_ns: Optional[Sequence[Optional[dict[str, Optional[ClinvarControl]]]]] = None, namespaced: Optional[bool] = None, na_rep="NA", ) -> Iterable[dict[str, Any]]: @@ -1024,7 +1024,7 @@ def variants_to_csv_rows( List of mapped variants corresponding to the variants. gnomad_data : list[Optional[variant.models.GnomADVariant]], optional List of gnomAD variant data corresponding to the variants. - clinvar_data_by_ns : list[Optional[dict[str, Optional[ClinicalControl]]]], optional + clinvar_data_by_ns : list[Optional[dict[str, Optional[ClinvarControl]]]], optional Per-variant ClinVar data keyed by namespace (e.g. "clinvar.2024_01"). na_rep : str String to represent null values. @@ -1036,7 +1036,7 @@ def variants_to_csv_rows( n = len(variants) _mappings: Sequence[Optional[MappedVariant]] = mappings if mappings is not None else [None] * n _gnomad: Sequence[Optional[GnomADVariant]] = gnomad_data if gnomad_data is not None else [None] * n - _clinvar: Sequence[Optional[dict[str, Optional[ClinicalControl]]]] = ( + _clinvar: Sequence[Optional[dict[str, Optional[ClinvarControl]]]] = ( clinvar_data_by_ns if clinvar_data_by_ns is not None else [None] * n ) return map( diff --git a/src/mavedb/models/__init__.py b/src/mavedb/models/__init__.py index b18e20ea7..c181e4099 100644 --- a/src/mavedb/models/__init__.py +++ b/src/mavedb/models/__init__.py @@ -4,6 +4,7 @@ "allele", "collection", "clinical_control", + "clinvar_allele_link", "controlled_keyword", "doi_identifier", "ensembl_identifier", diff --git a/src/mavedb/models/clinical_control.py b/src/mavedb/models/clinical_control.py index 0b989cb22..cf28b5b8b 100644 --- a/src/mavedb/models/clinical_control.py +++ b/src/mavedb/models/clinical_control.py @@ -8,18 +8,19 @@ from mavedb.models.clinical_control_mapped_variant import mapped_variants_clinical_controls_association_table if TYPE_CHECKING: + from mavedb.models.clinvar_allele_link import ClinvarAlleleLink from mavedb.models.mapped_variant import MappedVariant -class ClinicalControl(Base): - __tablename__ = "clinical_controls" +class ClinvarControl(Base): + __tablename__ = "clinvar_controls" __table_args__ = ( UniqueConstraint( - "db_name", "db_identifier", "db_version", name="uq_clinical_controls_db_name_identifier_version" + "db_name", "db_identifier", "db_version", name="uq_clinvar_controls_db_name_identifier_version" ), ) - id = Column(Integer, primary_key=True) + id: Mapped[int] = Column(Integer, primary_key=True) gene_symbol = Column(String, nullable=False, index=True) @@ -27,14 +28,25 @@ class ClinicalControl(Base): clinical_review_status = Column(String, nullable=False) db_name = Column(String, nullable=False, index=True) + # Currently the ClinVar Allele ID. db_identifier = Column(String, nullable=False, index=True) db_version = Column(String, nullable=False, index=True) + # ClinVar's canonical public identifier (ClinVar Variation ID). + clinvar_variation_id = Column(String, nullable=True) + creation_date = Column(Date, nullable=False, default=date.today) modification_date = Column(Date, nullable=False, default=date.today, onupdate=date.today) + # Frozen serving path: links to MappedVariant via the old association table (never written for new data). mapped_variants: Mapped[list["MappedVariant"]] = relationship( "MappedVariant", secondary=mapped_variants_clinical_controls_association_table, back_populates="clinical_controls", ) + + # New-model annotation links (one live link per allele per ClinVar release). + allele_links: Mapped[list["ClinvarAlleleLink"]] = relationship( + "ClinvarAlleleLink", + back_populates="clinvar_control", + ) diff --git a/src/mavedb/models/clinical_control_mapped_variant.py b/src/mavedb/models/clinical_control_mapped_variant.py index eabb7689a..b7fc5d84c 100644 --- a/src/mavedb/models/clinical_control_mapped_variant.py +++ b/src/mavedb/models/clinical_control_mapped_variant.py @@ -7,5 +7,5 @@ "mapped_variants_clinical_controls", Base.metadata, Column("mapped_variant_id", ForeignKey("mapped_variants.id"), primary_key=True), - Column("clinical_control_id", ForeignKey("clinical_controls.id"), primary_key=True), + Column("clinical_control_id", ForeignKey("clinvar_controls.id"), primary_key=True), ) diff --git a/src/mavedb/models/clinvar_allele_link.py b/src/mavedb/models/clinvar_allele_link.py new file mode 100644 index 000000000..941ee2f22 --- /dev/null +++ b/src/mavedb/models/clinvar_allele_link.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING + +from sqlalchemy import Column, ForeignKey, Index, Integer, text +from sqlalchemy.orm import Mapped, relationship + +from mavedb.db.base import Base +from mavedb.db.mixins import ValidTime + +if TYPE_CHECKING: + from .allele import Allele + from .clinical_control import ClinvarControl + + +class ClinvarAlleleLink(ValidTime, Base): + """Valid-time link between an :class:`Allele` and a :class:`ClinvarControl` release. + + Replaces the frozen ``mapped_variants_clinical_controls`` association table for new-model writes. + A link is live while ``valid_to`` is NULL. Unlike gnomAD/VEP (one live result per allele), the partial + unique index is ``(allele_id, clinvar_control_id) WHERE valid_to IS NULL`` — **multi-live**: an allele + accumulates one live link per ClinVar release, because each release is a distinct, versioned + ``ClinvarControl`` assertion that stacks rather than supersedes. A link retires only on two theoretical + paths (archival data does not change): ClinVar drops the variant from a release (a re-run finds no data + for it), or the allele re-resolves to a *different* control within the same release — the job supersedes + that newest-wins to preserve one live link per (allele, release), since this index only enforces one live + link per (allele, control). + """ + + __tablename__ = "clinvar_allele_links" + + id: Mapped[int] = Column(Integer, primary_key=True) + allele_id: Mapped[int] = Column( + Integer, + ForeignKey("alleles.id", ondelete="RESTRICT"), + nullable=False, + ) + clinvar_control_id: Mapped[int] = Column( + Integer, + ForeignKey("clinvar_controls.id", ondelete="RESTRICT"), + nullable=False, + ) + + # One-directional to Allele (no reverse collection there); back-ref on the ClinvarControl entity only. + allele: Mapped["Allele"] = relationship("Allele") + clinvar_control: Mapped["ClinvarControl"] = relationship("ClinvarControl", back_populates="allele_links") + + __table_args__ = ( + Index( + "ix_clinvar_allele_links_allele_id", + "allele_id", + ), + Index( + "ix_clinvar_allele_links_clinvar_control_id", + "clinvar_control_id", + ), + # Multi-live: one live link per (allele, release). Each ClinVar release is a distinct + # ClinvarControl row, so different releases stack as independent live links rather than + # superseding. Only live rows participate in this constraint. + Index( + "uq_clinvar_allele_links_live", + "allele_id", + "clinvar_control_id", + unique=True, + postgresql_where=text("valid_to IS NULL"), + ), + ) diff --git a/src/mavedb/models/mapped_variant.py b/src/mavedb/models/mapped_variant.py index b35f1bbb4..40325cf17 100644 --- a/src/mavedb/models/mapped_variant.py +++ b/src/mavedb/models/mapped_variant.py @@ -11,7 +11,7 @@ from mavedb.models.gnomad_variant_mapped_variant import gnomad_variants_mapped_variants_association_table if TYPE_CHECKING: - from .clinical_control import ClinicalControl + from .clinical_control import ClinvarControl from .gnomad_variant import GnomADVariant from .target_gene_mapping import TargetGeneMapping from .variant import Variant @@ -61,8 +61,8 @@ class MappedVariant(Base): hgvs_c = Column(String, nullable=True) hgvs_p = Column(String, nullable=True) - clinical_controls: Mapped[list["ClinicalControl"]] = relationship( - "ClinicalControl", + clinical_controls: Mapped[list["ClinvarControl"]] = relationship( + "ClinvarControl", secondary=mapped_variants_clinical_controls_association_table, back_populates="mapped_variants", ) diff --git a/src/mavedb/routers/score_sets.py b/src/mavedb/routers/score_sets.py index 371862d15..04f798a70 100644 --- a/src/mavedb/routers/score_sets.py +++ b/src/mavedb/routers/score_sets.py @@ -73,7 +73,7 @@ generate_score_set_urn, ) from mavedb.lib.workflow.pipeline_factory import PipelineFactory -from mavedb.models.clinical_control import ClinicalControl +from mavedb.models.clinical_control import ClinvarControl from mavedb.models.contributor import Contributor from mavedb.models.enums.processing_state import ProcessingState from mavedb.models.experiment import Experiment @@ -2411,7 +2411,7 @@ async def get_clinical_controls_for_score_set( user_data: UserData = Depends(get_current_user), db: Optional[str] = None, version: Optional[str] = None, -) -> Sequence[ClinicalControl]: +) -> Sequence[ClinvarControl]: """ Fetch relevant clinical controls for a given score set. """ @@ -2432,23 +2432,23 @@ async def get_clinical_controls_for_score_set( assert_permission(user_data, item, Action.READ) clinical_controls_query = ( - select(ClinicalControl) - .join(ClinicalControl.mapped_variants) + select(ClinvarControl) + .join(ClinvarControl.mapped_variants) .join(MappedVariant.variant) - .options(contains_eager(ClinicalControl.mapped_variants).contains_eager(MappedVariant.variant)) + .options(contains_eager(ClinvarControl.mapped_variants).contains_eager(MappedVariant.variant)) .filter(MappedVariant.current.is_(True)) .filter(Variant.score_set_id == item.id) ) if db_name is not None: save_to_logging_context({"db_name": db_name}) - clinical_controls_query = clinical_controls_query.filter(ClinicalControl.db_name == db_name) + clinical_controls_query = clinical_controls_query.filter(ClinvarControl.db_name == db_name) if db_version is not None: save_to_logging_context({"db_version": db_version}) - clinical_controls_query = clinical_controls_query.filter(ClinicalControl.db_version == db_version) + clinical_controls_query = clinical_controls_query.filter(ClinvarControl.db_version == db_version) - clinical_controls: Sequence[ClinicalControl] = _db.scalars(clinical_controls_query).unique().all() + clinical_controls: Sequence[ClinvarControl] = _db.scalars(clinical_controls_query).unique().all() if not clinical_controls: logger.info( @@ -2496,8 +2496,8 @@ async def get_clinical_controls_options_for_score_set( assert_permission(user_data, item, Action.READ) clinical_controls_query = ( - select(ClinicalControl.db_name, ClinicalControl.db_version) - .join(MappedVariant, ClinicalControl.mapped_variants) + select(ClinvarControl.db_name, ClinvarControl.db_version) + .join(MappedVariant, ClinvarControl.mapped_variants) .join(Variant) .where(MappedVariant.current.is_(True)) .where(Variant.score_set_id == item.id) diff --git a/src/mavedb/worker/jobs/external_services/clinvar.py b/src/mavedb/worker/jobs/external_services/clinvar.py index 68a34a04b..8041d14f7 100644 --- a/src/mavedb/worker/jobs/external_services/clinvar.py +++ b/src/mavedb/worker/jobs/external_services/clinvar.py @@ -1,35 +1,33 @@ -"""ClinVar integration jobs for variant annotation +"""ClinVar integration jobs for variant annotation. -This module contains job definitions and utility functions for integrating ClinVar -variant data into MaveDB. It includes functions to fetch and parse ClinVar variant -summary data, and update MaveDB records with the latest ClinVar annotations. +Links deduplicated alleles to ClinVar clinical-control data across every archival ClinVar +release. Each release is a distinct, versioned ``ClinvarControl`` entity, and an allele +accumulates one live ``ClinvarAlleleLink`` per release it appears in (multi-live, unlike +gnomAD/VEP which hold one live result per allele). Both ClinGen API calls and ClinVar TSV data fetches are automatically cached using aiocache with Redis backend: - ClinGen API calls: 24-hour TTL - ClinVar TSV files: 90-day TTL (archival data doesn't change) - -This significantly reduces redundant network requests when refreshing ClinVar -controls across multiple months/years. """ import logging from datetime import datetime import requests -from sqlalchemy import select +from sqlalchemy import func, select from sqlalchemy.dialects.postgresql import insert as pg_insert from mavedb.lib.annotation_status_manager import AnnotationStatusManager from mavedb.lib.clingen.allele_registry import get_associated_clinvar_allele_id +from mavedb.lib.clingen.alleles import get_alleles_for_score_set, group_alleles_for_annotation from mavedb.lib.clinvar.utils import fetch_clinvar_variant_data from mavedb.lib.types.workflow import JobExecutionOutcome -from mavedb.models.clinical_control import ClinicalControl +from mavedb.models.clinical_control import ClinvarControl +from mavedb.models.clinvar_allele_link import ClinvarAlleleLink from mavedb.models.enums.annotation_type import AnnotationType from mavedb.models.enums.job_pipeline import AnnotationFailureCategory, AnnotationStatus, FailureCategory -from mavedb.models.mapped_variant import MappedVariant from mavedb.models.score_set import ScoreSet -from mavedb.models.variant import Variant from mavedb.worker.jobs.utils.setup import validate_job_params from mavedb.worker.lib.decorators.pipeline_management import with_pipeline_management from mavedb.worker.lib.managers.job_manager import JobManager @@ -42,7 +40,7 @@ CLINVAR_START_MONTH = 2 -def generate_clinvar_versions() -> list[tuple[int, int]]: +def _generate_clinvar_versions() -> list[tuple[int, int]]: """Generate all ClinVar version (year, month) pairs from Feb 2015 to current Jan. Returns a list of (year, month) tuples representing each ClinVar archival @@ -55,15 +53,64 @@ def generate_clinvar_versions() -> list[tuple[int, int]]: return versions +def _annotate_clinvar( + annotation_manager: AnnotationStatusManager, + variant_ids: list[int], + version: str, + status: AnnotationStatus, + *, + failure_category: AnnotationFailureCategory | None = None, + error_message: str | None = None, + metadata: dict | None = None, +) -> None: + """Fan a CLINVAR_CONTROL annotation out to every variant served by an allele. + + AAS migration seam: the single choke point for ClinVar's per-variant VAS writes. At migration it + becomes an allele-keyed event writer; the per-variant fan-out goes away, and the variant + association narrows to provenance. See docs/design/allele-annotation-status.md. + + Unlike gnomAD/VEP, ClinVar writes a status row per ClinVar release, so retirement is + version-scoped (``replace_all_versions=False``) — each release keeps its own current row. + """ + annotation_data: dict = {"annotation_metadata": metadata or {}} + if error_message is not None: + annotation_data["error_message"] = error_message + + for variant_id in variant_ids: + annotation_manager.add_annotation( + variant_id=variant_id, + annotation_type=AnnotationType.CLINVAR_CONTROL, + version=version, + status=status, + failure_category=failure_category, + annotation_data=annotation_data, + current=True, + replace_all_versions=False, + ) + + @with_pipeline_management async def refresh_clinvar_controls(ctx: dict, job_id: int, job_manager: JobManager) -> JobExecutionOutcome: - """Refresh ClinVar clinical control data across all archival versions. - - Iterates over every ClinVar archival snapshot (Feb 2015, then Jan of each - subsequent year through the current year), fetching TSV data and updating - clinical control records for all mapped variants in the score set. Individual - version failures are logged and skipped — the job continues processing - remaining versions. + """Link deduplicated alleles to ClinVar clinical-control data across all archival versions. + + Iterates over every ClinVar archival snapshot (Feb 2015, then Jan of each subsequent year through + the current year). For each version it resolves each allele's ClinGen Allele ID (CAID) to a ClinVar + allele id, upserts the versioned :class:`ClinvarControl`, and establishes a live + :class:`ClinvarAlleleLink`. An allele accumulates one live link per release (multi-live). Individual + version fetch failures are logged and skipped — the job continues with the remaining versions. + + Job Parameters: + - score_set_id (int): The ID of the ScoreSet whose alleles to process. + - correlation_id (str): Correlation ID for tracing requests across services. + - force (bool, optional): Bypass the per-version skip and re-resolve every allele. The link + write still get-or-creates (no duplicate live link), so a forced re-run of unchanged data + writes no new links. + + Side Effects: + - Creates ClinvarControl and ClinvarAlleleLink rows. + + Returns: + JobExecutionOutcome: outcome with version and per-link counts. """ job = job_manager.get_job() @@ -72,8 +119,9 @@ async def refresh_clinvar_controls(ctx: dict, job_id: int, job_manager: JobManag score_set = job_manager.db.scalars(select(ScoreSet).where(ScoreSet.id == job.job_params["score_set_id"])).one() # type: ignore correlation_id = job.job_params["correlation_id"] # type: ignore + force = bool(job.job_params.get("force", False)) # type: ignore[union-attr] - versions = generate_clinvar_versions() + versions = _generate_clinvar_versions() job_manager.save_to_context( { @@ -83,24 +131,53 @@ async def refresh_clinvar_controls(ctx: dict, job_id: int, job_manager: JobManag "correlation_id": correlation_id, "versions": versions, "total_versions": len(versions), + "force": force, } ) job_manager.update_progress(0, 100, f"Starting ClinVar refresh across {len(versions)} versions.") logger.info(f"Starting ClinVar refresh across {len(versions)} versions", extra=job_manager.logging_context()) - variants_to_refresh = job_manager.db.scalars( - select(MappedVariant) - .join(Variant) - .where( - Variant.score_set_id == score_set.id, - MappedVariant.current.is_(True), + # One work-unit per allele (payload = CAID; alleles without one are dropped). Covers ALL the score + # set's alleles — authoritative and RT-derived — since the genomic allele ClinVar keys on is often + # the RT-derived one; VAS still fans only to authoritative_variant_ids (the bandaid seam). + allele_data = group_alleles_for_annotation( + get_alleles_for_score_set(job_manager.db, score_set.id), + payload=lambda row: row.clingen_allele_id, + ) + job_manager.save_to_context({"num_alleles_with_caids": len(allele_data)}) + + if not allele_data: + logger.warning( + msg="No current alleles with CAIDs were found for this score set. Skipping ClinVar refresh.", + extra=job_manager.logging_context(), + ) + job_manager.db.flush() + return JobExecutionOutcome.succeeded( + data={ + "versions_completed": 0, + "versions_total": len(versions), + "created_link_count": 0, + "preexisting_link_count": 0, + } + ) + + all_allele_ids = set(allele_data.keys()) + + def alleles_linked_at_version(clinvar_version: str) -> set[int]: + """Allele ids (within the work set) holding a live ClinvarAlleleLink to a control of this version.""" + return set( + job_manager.db.scalars( + select(ClinvarAlleleLink.allele_id) + .join(ClinvarControl, ClinvarControl.id == ClinvarAlleleLink.clinvar_control_id) + .where(ClinvarAlleleLink.allele_id.in_(all_allele_ids)) + .where(ClinvarAlleleLink.current) + .where(ClinvarControl.db_version == clinvar_version) + ).all() ) - ).all() - total_variants_to_refresh = len(variants_to_refresh) - job_manager.save_to_context({"total_variants_to_refresh": total_variants_to_refresh}) - total_refreshed = 0 total_failed = 0 + created_link_count = 0 + preexisting_link_count = 0 versions_completed = 0 for version_index, (year, month) in enumerate(versions): @@ -125,160 +202,175 @@ async def refresh_clinvar_controls(ctx: dict, job_id: int, job_manager: JobManag ) continue + # Cost: skip alleles already linked at this version (an archival release cannot change). force + # bypasses the skip but the link write still get-or-creates, so a forced no-op writes nothing. + already_linked = set() if force else alleles_linked_at_version(clinvar_version) + annotation_manager = AnnotationStatusManager(job_manager.db, job_run_id=job_manager.job_id) - for mapped_variant in variants_to_refresh: - clingen_id = mapped_variant.clingen_allele_id - - if clingen_id is None: - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.CLINVAR_CONTROL, - version=clinvar_version, - status=AnnotationStatus.SKIPPED, - failure_category=AnnotationFailureCategory.MISSING_IDENTIFIER, - annotation_data={ - "error_message": "Mapped variant does not have an associated ClinGen allele ID.", - }, - current=True, - replace_all_versions=False, + for allele_id, entry in allele_data.items(): + caid = entry.payload + + if allele_id in already_linked: + preexisting_link_count += 1 + _annotate_clinvar( + annotation_manager, + entry.authoritative_variant_ids, + clinvar_version, + AnnotationStatus.SUCCESS, + metadata={"clingen_allele_id": caid, "action": "preexisting"}, ) continue - if "," in clingen_id: - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.CLINVAR_CONTROL, - version=clinvar_version, - status=AnnotationStatus.SKIPPED, + if "," in caid: + _annotate_clinvar( + annotation_manager, + entry.authoritative_variant_ids, + clinvar_version, + AnnotationStatus.SKIPPED, failure_category=AnnotationFailureCategory.UNSUPPORTED_IDENTIFIER, - annotation_data={ - "error_message": "Multi-variant ClinGen allele IDs cannot be associated with ClinVar data.", - }, - current=True, - replace_all_versions=False, + error_message="Multi-variant ClinGen allele IDs cannot be associated with ClinVar data.", + metadata={"clingen_allele_id": caid}, ) continue try: - clinvar_allele_id = await get_associated_clinvar_allele_id(clingen_id) # type: ignore + clinvar_allele_id = await get_associated_clinvar_allele_id(caid) except requests.exceptions.RequestException as exc: - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.CLINVAR_CONTROL, - version=clinvar_version, - status=AnnotationStatus.FAILED, + total_failed += 1 + _annotate_clinvar( + annotation_manager, + entry.authoritative_variant_ids, + clinvar_version, + AnnotationStatus.FAILED, failure_category=AnnotationFailureCategory.EXTERNAL_API_ERROR, - annotation_data={ - "error_message": f"Failed to retrieve ClinVar allele ID from ClinGen API: {str(exc)}", - }, - current=True, - replace_all_versions=False, + error_message=f"Failed to retrieve ClinVar allele ID from ClinGen API: {str(exc)}", + metadata={"clingen_allele_id": caid}, ) logger.error( - f"Failed to retrieve ClinVar allele ID from ClinGen API for ClinGen allele ID {clingen_id}.", + f"Failed to retrieve ClinVar allele ID from ClinGen API for ClinGen allele ID {caid}.", extra=job_manager.logging_context(), exc_info=exc, ) - total_failed += 1 continue if not clinvar_allele_id: - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.CLINVAR_CONTROL, - version=clinvar_version, - status=AnnotationStatus.SKIPPED, + _annotate_clinvar( + annotation_manager, + entry.authoritative_variant_ids, + clinvar_version, + AnnotationStatus.SKIPPED, failure_category=AnnotationFailureCategory.NO_LINKED_ALLELE, - annotation_data={ - "error_message": "No ClinVar allele ID found for ClinGen allele ID.", - }, - current=True, - replace_all_versions=False, + error_message="No ClinVar allele ID found for ClinGen allele ID.", + metadata={"clingen_allele_id": caid}, ) continue if clinvar_allele_id not in tsv_data: - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.CLINVAR_CONTROL, - version=clinvar_version, - status=AnnotationStatus.SKIPPED, + _annotate_clinvar( + annotation_manager, + entry.authoritative_variant_ids, + clinvar_version, + AnnotationStatus.SKIPPED, failure_category=AnnotationFailureCategory.EXTERNAL_REFERENCE_NOT_FOUND, - annotation_data={ - "error_message": "No ClinVar data found for ClinVar allele ID.", - }, - current=True, - replace_all_versions=False, + error_message="No ClinVar data found for ClinVar allele ID.", + metadata={"clingen_allele_id": caid}, ) continue variant_data = tsv_data[clinvar_allele_id] - identifier = str(clinvar_allele_id) - # Atomic upsert — avoids a check-then-act race when two - # refresh_clinvar_controls jobs run concurrently for different - # score sets and encounter the same (db_name, db_identifier, - # db_version) tuple. ON CONFLICT DO UPDATE is guaranteed to - # return exactly one row regardless of concurrent inserts. + # Atomic upsert — avoids a check-then-act race when two refresh_clinvar_controls jobs run + # concurrently for different score sets and encounter the same + # (db_name, db_identifier, db_version) tuple. ON CONFLICT DO UPDATE returns exactly one row. upsert_stmt = ( - pg_insert(ClinicalControl) + pg_insert(ClinvarControl) .values( - db_identifier=identifier, + db_identifier=str(clinvar_allele_id), db_version=clinvar_version, db_name="ClinVar", gene_symbol=variant_data.get("GeneSymbol"), clinical_significance=variant_data.get("ClinicalSignificance"), clinical_review_status=variant_data.get("ReviewStatus"), + clinvar_variation_id=variant_data.get("VariationID"), ) .on_conflict_do_update( - constraint="uq_clinical_controls_db_name_identifier_version", + constraint="uq_clinvar_controls_db_name_identifier_version", set_={ "gene_symbol": variant_data.get("GeneSymbol"), "clinical_significance": variant_data.get("ClinicalSignificance"), "clinical_review_status": variant_data.get("ReviewStatus"), + "clinvar_variation_id": variant_data.get("VariationID"), }, ) - .returning(ClinicalControl) + .returning(ClinvarControl) ) - clinvar_variant = job_manager.db.scalars(upsert_stmt).one() - - job_manager.db.add(clinvar_variant) - job_manager.db.flush() - - if clinvar_variant not in mapped_variant.clinical_controls: - mapped_variant.clinical_controls.append(clinvar_variant) - job_manager.db.add(mapped_variant) - - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.CLINVAR_CONTROL, - version=clinvar_version, - status=AnnotationStatus.SUCCESS, - annotation_data={ - "annotation_metadata": { - "clinvar_allele_id": clinvar_allele_id, - }, - }, - current=True, - replace_all_versions=False, + clinvar_control = job_manager.db.scalars(upsert_stmt).one() + + # At most one live link per (allele, release). Normally a release is immutable, so the + # allele's live link for this version is either a reconfirm of this same control (no-op) or + # absent (insert) — multi-live accumulates only across *different* releases, never supersedes. + # Defensive guard: if the allele already holds a live link to a *different* control of this + # same version, the release re-resolved under us (re-ingestion / upstream correction — should + # never happen for archival data). Supersede newest-wins with a shared timestamp and log. + live_link = job_manager.db.scalar( + select(ClinvarAlleleLink) + .join(ClinvarControl, ClinvarControl.id == ClinvarAlleleLink.clinvar_control_id) + .where( + ClinvarAlleleLink.allele_id == allele_id, + ClinvarAlleleLink.current, + ClinvarControl.db_version == clinvar_version, + ) ) + if live_link is None: + job_manager.db.add(ClinvarAlleleLink(allele_id=allele_id, clinvar_control_id=clinvar_control.id)) + created_link_count += 1 + action = "created" + elif live_link.clinvar_control_id == clinvar_control.id: + preexisting_link_count += 1 + action = "preexisting" + else: + at = job_manager.db.scalar(select(func.now())) + assert at is not None # SELECT now() always returns a timestamp + live_link.retire(at=at) + job_manager.db.add( + ClinvarAlleleLink(allele_id=allele_id, clinvar_control_id=clinvar_control.id, valid_from=at) + ) + created_link_count += 1 + action = "superseded" + logger.warning( + msg=( + f"Allele {allele_id} held a live ClinVar link to control " + f"{live_link.clinvar_control_id} for version {clinvar_version}, but re-resolved to " + f"control {clinvar_control.id} (ClinVar allele {clinvar_allele_id}). Superseding " + "newest-wins; archival ClinVar data should be immutable — investigate the upstream " + "re-resolution." + ), + extra=job_manager.logging_context(), + ) - total_refreshed += 1 + _annotate_clinvar( + annotation_manager, + entry.authoritative_variant_ids, + clinvar_version, + AnnotationStatus.SUCCESS, + metadata={"clingen_allele_id": caid, "clinvar_allele_id": clinvar_allele_id, "action": action}, + ) annotation_manager.flush() + job_manager.db.flush() versions_completed += 1 logger.info( - f"Completed ClinVar version {clinvar_version} for {total_variants_to_refresh} variants.", + f"Completed ClinVar version {clinvar_version} for {len(allele_data)} alleles.", extra=job_manager.logging_context(), ) logger.info( f"ClinVar refresh complete: {versions_completed}/{len(versions)} versions, " - f"{total_refreshed} variant-version annotations.", + f"{created_link_count} new links, {preexisting_link_count} preexisting.", extra=job_manager.logging_context(), ) - if total_failed > 0 and total_refreshed == 0: + if total_failed > 0 and created_link_count == 0 and preexisting_link_count == 0: error_message = ( f"All {total_failed} ClinVar lookups failed for score set {score_set.urn}. Possible ClinGen API outage." ) @@ -289,7 +381,8 @@ async def refresh_clinvar_controls(ctx: dict, job_id: int, job_manager: JobManag data={ "versions_completed": versions_completed, "versions_total": len(versions), - "variant_annotations": 0, + "created_link_count": 0, + "preexisting_link_count": 0, }, failure_category=FailureCategory.DEPENDENCY_FAILURE, ) @@ -299,6 +392,7 @@ async def refresh_clinvar_controls(ctx: dict, job_id: int, job_manager: JobManag data={ "versions_completed": versions_completed, "versions_total": len(versions), - "variant_annotations": total_refreshed, + "created_link_count": created_link_count, + "preexisting_link_count": preexisting_link_count, } ) diff --git a/tests/helpers/util/score_set.py b/tests/helpers/util/score_set.py index b6d7801ae..83ebcbca8 100644 --- a/tests/helpers/util/score_set.py +++ b/tests/helpers/util/score_set.py @@ -8,7 +8,7 @@ from fastapi.testclient import TestClient from sqlalchemy import select -from mavedb.models.clinical_control import ClinicalControl as ClinicalControlDbModel +from mavedb.models.clinical_control import ClinvarControl as ClinicalControlDbModel from mavedb.models.gnomad_variant import GnomADVariant as GnomADVariantDbModel from mavedb.models.mapped_variant import MappedVariant as MappedVariantDbModel from mavedb.models.score_set import ScoreSet as ScoreSetDbModel diff --git a/tests/routers/conftest.py b/tests/routers/conftest.py index ba34c5489..0971bde47 100644 --- a/tests/routers/conftest.py +++ b/tests/routers/conftest.py @@ -3,7 +3,7 @@ import pytest -from mavedb.models.clinical_control import ClinicalControl +from mavedb.models.clinical_control import ClinvarControl from mavedb.models.contributor import Contributor from mavedb.models.controlled_keyword import ControlledKeyword from mavedb.models.enums.user_role import UserRole @@ -52,8 +52,8 @@ def setup_router_db(session): db.add(License(**TEST_INACTIVE_LICENSE)) db.add(License(**EXTRA_LICENSE)) db.add(Contributor(**EXTRA_CONTRIBUTOR)) - db.add(ClinicalControl(**TEST_CLINVAR_CONTROL)) - db.add(ClinicalControl(**TEST_GENERIC_CLINICAL_CONTROL)) + db.add(ClinvarControl(**TEST_CLINVAR_CONTROL)) + db.add(ClinvarControl(**TEST_GENERIC_CLINICAL_CONTROL)) db.add(GnomADVariant(**TEST_GNOMAD_VARIANT)) db.bulk_save_objects([ControlledKeyword(**keyword_obj) for keyword_obj in TEST_DB_KEYWORDS]) db.commit() diff --git a/tests/worker/jobs/external_services/network/test_clinvar.py b/tests/worker/jobs/external_services/network/test_clinvar.py index f29842158..4569d08f7 100644 --- a/tests/worker/jobs/external_services/network/test_clinvar.py +++ b/tests/worker/jobs/external_services/network/test_clinvar.py @@ -10,11 +10,11 @@ from mavedb.lib.clinvar.constants import CLINVAR_FIELDS_TO_KEEP from mavedb.lib.clinvar.utils import fetch_clinvar_variant_data -from mavedb.models.clinical_control import ClinicalControl +from mavedb.models.clinical_control import ClinvarControl from mavedb.models.enums.annotation_type import AnnotationType from mavedb.models.enums.job_pipeline import AnnotationStatus, JobStatus from mavedb.models.variant_annotation_status import VariantAnnotationStatus -from mavedb.worker.jobs.external_services.clinvar import generate_clinvar_versions +from mavedb.worker.jobs.external_services.clinvar import _generate_clinvar_versions pytestmark = pytest.mark.usefixtures("patch_db_session_ctxmgr") @@ -43,7 +43,7 @@ async def test_refresh_clinvar_controls_e2e( arq_redis, arq_worker, standalone_worker_context, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, with_refresh_clinvar_controls_job, sample_refresh_clinvar_controls_job_run, ): @@ -65,8 +65,9 @@ async def test_refresh_clinvar_controls_e2e( await arq_worker.async_run() await arq_worker.run_check() - # The variant should be present in the 2025-01 archive. - clinical_controls = session.scalars(select(ClinicalControl)).all() + # Verify that clinical controls were added successfully — one row per ClinVar version + # that contains the variant, so there may be more than one. + clinical_controls = session.scalars(select(ClinvarControl)).all() assert len(clinical_controls) >= 1 assert all(cc.db_identifier == "3045425" for cc in clinical_controls) @@ -101,7 +102,7 @@ async def test_latest_clinvar_archive_has_expected_schema(self): CLINVAR_FIELDS_TO_KEEP is present. Fails fast if ClinVar renames or removes a column before a deploy reaches production. """ - year, month = generate_clinvar_versions()[-1] + year, month = _generate_clinvar_versions()[-1] data = await fetch_clinvar_variant_data(month, year) assert len(data) > 0, "ClinVar archive returned no records" diff --git a/tests/worker/jobs/external_services/test_clinvar.py b/tests/worker/jobs/external_services/test_clinvar.py index 1ac616ffc..6b5011236 100644 --- a/tests/worker/jobs/external_services/test_clinvar.py +++ b/tests/worker/jobs/external_services/test_clinvar.py @@ -3,19 +3,18 @@ import pytest import requests -from mavedb.models.clinical_control import ClinicalControl -from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationStatus, FailureCategory, JobStatus, PipelineStatus -from mavedb.models.variant_annotation_status import VariantAnnotationStatus - pytest.importorskip("arq") from unittest.mock import patch +from sqlalchemy import select + from mavedb.lib.types.workflow import JobExecutionOutcome -from mavedb.models.mapped_variant import MappedVariant -from mavedb.models.score_set import ScoreSet -from mavedb.models.variant import Variant +from mavedb.models.clinical_control import ClinvarControl +from mavedb.models.clinvar_allele_link import ClinvarAlleleLink +from mavedb.models.enums.annotation_type import AnnotationType +from mavedb.models.enums.job_pipeline import AnnotationFailureCategory, AnnotationStatus, FailureCategory, JobStatus +from mavedb.models.variant_annotation_status import VariantAnnotationStatus from mavedb.worker.jobs.external_services.clinvar import refresh_clinvar_controls from mavedb.worker.lib.managers.job_manager import JobManager @@ -26,6 +25,7 @@ "GeneSymbol": "TEST", "ClinicalSignificance": "benign", "ReviewStatus": "reviewed by expert panel", + "VariationID": "987654", }, } @@ -33,32 +33,29 @@ @pytest.mark.unit @pytest.mark.asyncio class TestRefreshClinvarControlsUnit: - """Tests for the refresh_clinvar_controls job function.""" + """Unit tests for the allele-model refresh_clinvar_controls job.""" @pytest.fixture(autouse=True) def _mock_clinvar_versions(self): - """Mock generate_clinvar_versions to return a single version for testing.""" + """Pin _generate_clinvar_versions to a single version so clinvar_version == '01_2026'.""" with patch( - "mavedb.worker.jobs.external_services.clinvar.generate_clinvar_versions", + "mavedb.worker.jobs.external_services.clinvar._generate_clinvar_versions", return_value=[(2026, 1)], ): yield - async def test_refresh_clinvar_controls_skips_version_on_fetch_failure( + async def test_no_alleles_with_caids( self, mock_worker_ctx, session, + with_populated_domain_data, with_refresh_clinvar_controls_job, sample_refresh_clinvar_controls_job_run, ): - """Test that a fetch failure for a version is logged and skipped, not propagated.""" - - async def awaitable_exception(*args, **kwargs): - raise Exception("Network error") - + """No current alleles carry a CAID -> the job succeeds with nothing to do.""" with patch( "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - side_effect=awaitable_exception, + return_value=MOCK_CLINVAR_DATA, ): result = await refresh_clinvar_controls( mock_worker_ctx, @@ -68,61 +65,26 @@ async def awaitable_exception(*args, **kwargs): assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - assert result.data["versions_completed"] == 0 + assert session.scalars(select(ClinvarControl)).all() == [] + assert session.scalars(select(ClinvarAlleleLink)).all() == [] + assert session.query(VariantAnnotationStatus).all() == [] - async def test_refresh_clinvar_controls_no_mapped_variants( + async def test_fetch_failure_skips_version( self, mock_worker_ctx, session, with_refresh_clinvar_controls_job, sample_refresh_clinvar_controls_job_run, + setup_sample_alleles_with_caid, ): - """Test that the job completes successfully when there are no mapped variants.""" - - with patch( - "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value={}, - ): - result = await refresh_clinvar_controls( - mock_worker_ctx, - sample_refresh_clinvar_controls_job_run.id, - JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), - ) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED + """A version whose TSV fetch fails is logged and skipped, not propagated.""" - async def test_refresh_clinvar_controls_no_variants_have_caids( - self, - mock_worker_ctx, - session, - with_refresh_clinvar_controls_job, - sample_refresh_clinvar_controls_job_run, - ): - """Test that the job completes successfully when no variants have CAIDs.""" - # Add a variant without a CAID - score_set = session.get(ScoreSet, sample_refresh_clinvar_controls_job_run.job_params["score_set_id"]) - variant = Variant( - urn="urn:variant:test-variant-no-caid", - score_set_id=score_set.id, - hgvs_nt="NM_000000.1:c.2G>A", - hgvs_pro="NP_000000.1:p.Val2Ile", - data={"hgvs_c": "NM_000000.1:c.2G>A", "hgvs_p": "NP_000000.1:p.Val2Ile"}, - ) - session.add(variant) - session.commit() - mapped_variant = MappedVariant( - variant_id=variant.id, - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant) - session.commit() + async def boom(*args, **kwargs): + raise Exception("Network error") with patch( "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, + side_effect=boom, ): result = await refresh_clinvar_controls( mock_worker_ctx, @@ -130,29 +92,21 @@ async def test_refresh_clinvar_controls_no_variants_have_caids( JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), ) - assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED + assert result.data["versions_completed"] == 0 + assert session.scalars(select(ClinvarAlleleLink)).all() == [] - # Verify an annotation status was created for the variant without a CAID - variant_no_caid = ( - session.query(VariantAnnotationStatus).filter(VariantAnnotationStatus.variant_id == variant.id).one() - ) - assert variant_no_caid.status == AnnotationStatus.SKIPPED - assert variant_no_caid.annotation_type == AnnotationType.CLINVAR_CONTROL - assert variant_no_caid.error_message == "Mapped variant does not have an associated ClinGen allele ID." - - async def test_refresh_clinvar_controls_variants_are_multivariants( + async def test_multi_variant_caid_skipped( self, mock_worker_ctx, session, with_refresh_clinvar_controls_job, sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, ): - """Test that the job completes successfully when all variants are multi-variant CAIDs.""" - # Update the mapped variant to have a multi-variant CAID - mapped_variant = session.query(MappedVariant).first() - mapped_variant.clingen_allele_id = "CA-MULTI-001,CA-MULTI-002" + """An allele whose CAID is a multi-variant identifier is skipped (ClinVar can't key on it).""" + variant, allele = setup_sample_alleles_with_caid + allele.clingen_allele_id = "CA-MULTI-001,CA-MULTI-002" session.commit() with patch( @@ -165,37 +119,29 @@ async def test_refresh_clinvar_controls_variants_are_multivariants( JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), ) - assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED + assert session.scalars(select(ClinvarAlleleLink)).all() == [] - # Verify an annotation status was created for the multi-variant CAID - variant_with_multicid = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant.variant_id) - .one() - ) - assert variant_with_multicid.status == AnnotationStatus.SKIPPED - assert variant_with_multicid.annotation_type == AnnotationType.CLINVAR_CONTROL - assert ( - variant_with_multicid.error_message - == "Multi-variant ClinGen allele IDs cannot be associated with ClinVar data." - ) + vas = session.query(VariantAnnotationStatus).filter_by(variant_id=variant.id).one() + assert vas.status == AnnotationStatus.SKIPPED + assert vas.annotation_type == AnnotationType.CLINVAR_CONTROL + assert vas.failure_category == AnnotationFailureCategory.UNSUPPORTED_IDENTIFIER - async def test_refresh_clinvar_controls_clingen_api_failure( + async def test_no_associated_clinvar_allele_id_skipped( self, mock_worker_ctx, session, with_refresh_clinvar_controls_job, sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, ): - """Test that the job handles ClinGen API failures gracefully.""" + """ClinGen returns no ClinVar allele id for the CAID -> SKIPPED/NO_LINKED_ALLELE, no link.""" + variant, _ = setup_sample_alleles_with_caid - # Mock the get_associated_clinvar_allele_id function to raise an exception with ( patch( "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - side_effect=requests.exceptions.RequestException("ClinGen API error"), + return_value=None, ), patch( "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", @@ -208,40 +154,31 @@ async def test_refresh_clinvar_controls_clingen_api_failure( JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), ) - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.FAILED - assert result.failure_category == FailureCategory.DEPENDENCY_FAILURE - - # Verify an annotation status was created for the variant due to ClinGen API failure - mapped_variant = session.query(MappedVariant).first() - variant_with_api_failure = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant.variant_id) - .one() - ) - assert variant_with_api_failure.status == AnnotationStatus.FAILED - assert variant_with_api_failure.annotation_type == AnnotationType.CLINVAR_CONTROL - assert "Failed to retrieve ClinVar allele ID from ClinGen API" in variant_with_api_failure.error_message + assert result.status == JobStatus.SUCCEEDED + assert session.scalars(select(ClinvarAlleleLink)).all() == [] + vas = session.query(VariantAnnotationStatus).filter_by(variant_id=variant.id).one() + assert vas.status == AnnotationStatus.SKIPPED + assert vas.failure_category == AnnotationFailureCategory.NO_LINKED_ALLELE - async def test_refresh_clinvar_controls_no_associated_clinvar_allele_id( + async def test_clinvar_data_not_found_skipped( self, mock_worker_ctx, session, with_refresh_clinvar_controls_job, sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, ): - """Test that the job handles no associated ClinVar Allele ID gracefully.""" + """The resolved ClinVar allele id is absent from the version's TSV -> SKIPPED, no link.""" + variant, _ = setup_sample_alleles_with_caid - # Mock the get_associated_clinvar_allele_id function to return None with ( patch( "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - return_value=None, + return_value="VCV000000123", ), patch( "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, + return_value={"VCV000000999": {"GeneSymbol": "X", "ClinicalSignificance": "y", "ReviewStatus": "z"}}, ), ): result = await refresh_clinvar_controls( @@ -250,48 +187,31 @@ async def test_refresh_clinvar_controls_no_associated_clinvar_allele_id( JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), ) - assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED + assert session.scalars(select(ClinvarAlleleLink)).all() == [] + vas = session.query(VariantAnnotationStatus).filter_by(variant_id=variant.id).one() + assert vas.status == AnnotationStatus.SKIPPED + assert vas.failure_category == AnnotationFailureCategory.EXTERNAL_REFERENCE_NOT_FOUND - # Verify an annotation status was created for the variant due to no associated ClinVar Allele ID - mapped_variant = session.query(MappedVariant).first() - variant_no_clinvar_allele = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant.variant_id) - .one() - ) - assert variant_no_clinvar_allele.status == AnnotationStatus.SKIPPED - assert variant_no_clinvar_allele.annotation_type == AnnotationType.CLINVAR_CONTROL - assert "No ClinVar allele ID found for ClinGen allele ID" in variant_no_clinvar_allele.error_message - - async def test_refresh_clinvar_controls_no_clinvar_data_found( + async def test_clingen_api_failure_fails_when_total( self, mock_worker_ctx, session, with_refresh_clinvar_controls_job, sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, ): - """Test that the job handles no ClinVar data found for the associated ClinVar Allele ID.""" - - # TSV data with a different allele ID than the one being looked up - non_matching_clinvar_data = { - "VCV000000001": { - "GeneSymbol": "TEST", - "ClinicalSignificance": "benign", - "ReviewStatus": "reviewed by expert panel", - }, - } + """A ClinGen API error with no successful links returns FAILED/DEPENDENCY_FAILURE.""" + variant, _ = setup_sample_alleles_with_caid - # Mock the get_associated_clinvar_allele_id function to return a ClinVar Allele ID with ( patch( "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - return_value="VCV000000123", + side_effect=requests.exceptions.RequestException("ClinGen API error"), ), patch( "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=non_matching_clinvar_data, + return_value=MOCK_CLINVAR_DATA, ), ): result = await refresh_clinvar_controls( @@ -300,31 +220,23 @@ async def test_refresh_clinvar_controls_no_clinvar_data_found( JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), ) - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - # Verify an annotation status was created for the variant due to no ClinVar data found - mapped_variant = session.query(MappedVariant).first() - variant_no_clinvar_data = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant.variant_id) - .one() - ) - assert variant_no_clinvar_data.status == AnnotationStatus.SKIPPED - assert variant_no_clinvar_data.annotation_type == AnnotationType.CLINVAR_CONTROL - assert "No ClinVar data found for ClinVar allele ID" in variant_no_clinvar_data.error_message + assert result.status == JobStatus.FAILED + assert result.failure_category == FailureCategory.DEPENDENCY_FAILURE + vas = session.query(VariantAnnotationStatus).filter_by(variant_id=variant.id).one() + assert vas.status == AnnotationStatus.FAILED + assert vas.failure_category == AnnotationFailureCategory.EXTERNAL_API_ERROR - async def test_refresh_clinvar_controls_successful_annotation_existing_control( + async def test_successful_link_new_control( self, mock_worker_ctx, session, with_refresh_clinvar_controls_job, sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, ): - """Test that the job successfully annotates a variant with ClinVar control data.""" + """A resolved + present CAID creates a ClinvarControl, a live link, and a SUCCESS VAS.""" + variant, allele = setup_sample_alleles_with_caid - # Mock the get_associated_clinvar_allele_id function to return a ClinVar Allele ID with ( patch( "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", @@ -341,54 +253,48 @@ async def test_refresh_clinvar_controls_successful_annotation_existing_control( JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), ) - assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - - # Verify an annotation status was created for the variant with successful annotation - mapped_variant = session.query(MappedVariant).first() - annotated_variant = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant.variant_id) - .one() - ) - assert annotated_variant.status == AnnotationStatus.SUCCESS - assert annotated_variant.annotation_type == AnnotationType.CLINVAR_CONTROL - assert annotated_variant.error_message is None - - async def test_refresh_clinvar_controls_successful_annotation_new_control( + assert result.data["created_link_count"] == 1 + + control = session.scalars(select(ClinvarControl)).one() + assert control.db_identifier == "VCV000000123" + assert control.db_version == "01_2026" + assert control.clinical_significance == "benign" + # AlleleID stays in db_identifier; the canonical VariationID is captured alongside it. + assert control.clinvar_variation_id == "987654" + + links = session.scalars( + select(ClinvarAlleleLink).where(ClinvarAlleleLink.allele_id == allele.id, ClinvarAlleleLink.current) + ).all() + assert len(links) == 1 + assert links[0].clinvar_control_id == control.id + + vas = session.query(VariantAnnotationStatus).filter_by(variant_id=variant.id).one() + assert vas.status == AnnotationStatus.SUCCESS + assert vas.annotation_metadata["action"] == "created" + + async def test_links_rt_derived_allele_but_annotates_only_authoritative( self, mock_worker_ctx, session, with_refresh_clinvar_controls_job, sample_refresh_clinvar_controls_job_run, + setup_rt_derived_allele_with_caid, ): - """Test that the job successfully annotates a variant with ClinVar control data when no prior status exists.""" - # Add a variant and mapped variant to the database with a CAID - score_set = session.get(ScoreSet, sample_refresh_clinvar_controls_job_run.job_params["score_set_id"]) - variant = Variant( - urn="urn:variant:test-variant-with-caid-2", - score_set_id=score_set.id, - hgvs_nt="NM_000000.1:c.3C>T", - hgvs_pro="NP_000000.1:p.Ala3Val", - data={"hgvs_c": "NM_000000.1:c.3C>T", "hgvs_p": "NP_000000.1:p.Ala3Val"}, - ) - session.add(variant) - session.commit() - mapped_variant = MappedVariant( - variant_id=variant.id, - clingen_allele_id="CA124", - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant) - session.commit() + """ClinVar linkage covers the full allele set: the RT-derived allele carries the matching CAID + and is linked, while per-variant VAS is written only for the authoritative link (the bandaid). + """ + variant, authoritative_allele, rt_allele = setup_rt_derived_allele_with_caid + rt_caid = rt_allele.clingen_allele_id + + async def resolve(caid): + # Only the RT-derived allele's CAID resolves to a ClinVar id present in the TSV. + return "VCV000000123" if caid == rt_caid else None - # Mock the get_associated_clinvar_allele_id function to return a ClinVar Allele ID with ( patch( "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - return_value="VCV000000123", + side_effect=resolve, ), patch( "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", @@ -401,1105 +307,276 @@ async def test_refresh_clinvar_controls_successful_annotation_new_control( JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), ) - assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - # Verify an annotation status was created for the variant with successful annotation - annotated_variant = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant.variant_id) - .one() + # The RT-derived allele IS linked — the core fix. + rt_links = session.scalars( + select(ClinvarAlleleLink).where(ClinvarAlleleLink.allele_id == rt_allele.id, ClinvarAlleleLink.current) + ).all() + assert len(rt_links) == 1 + # The authoritative allele's CAID had no ClinVar match, so it is not linked. + assert ( + session.scalars( + select(ClinvarAlleleLink).where(ClinvarAlleleLink.allele_id == authoritative_allele.id) + ).all() + == [] ) - assert annotated_variant.status == AnnotationStatus.SUCCESS - assert annotated_variant.annotation_type == AnnotationType.CLINVAR_CONTROL - assert annotated_variant.error_message is None - async def test_refresh_clinvar_controls_idempotent_run( + # VAS is written only for the authoritative link: exactly one row, keyed to the variant. + statuses = session.query(VariantAnnotationStatus).all() + assert len(statuses) == 1 + assert statuses[0].variant_id == variant.id + assert statuses[0].annotation_type == AnnotationType.CLINVAR_CONTROL + + async def test_idempotent_rerun_skips_and_does_not_duplicate( self, mock_worker_ctx, session, with_refresh_clinvar_controls_job, sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, ): - """Test that running the job multiple times does not create duplicate annotation statuses.""" + """A second run finds the allele already linked at the version, skips the resolution, reports + preexisting, and creates neither a duplicate control nor a duplicate link.""" + variant, allele = setup_sample_alleles_with_caid - # Mock the get_associated_clinvar_allele_id function to return a ClinVar Allele ID with ( patch( "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", return_value="VCV000000123", - ), + ) as resolve_spy, patch( "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - side_effect=[MOCK_CLINVAR_DATA, MOCK_CLINVAR_DATA], + return_value=MOCK_CLINVAR_DATA, ), ): - # First run - result1 = await refresh_clinvar_controls( + first = await refresh_clinvar_controls( mock_worker_ctx, sample_refresh_clinvar_controls_job_run.id, JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), ) - session.commit() + assert resolve_spy.await_count == 1 - # Second run - result2 = await refresh_clinvar_controls( + second = await refresh_clinvar_controls( mock_worker_ctx, sample_refresh_clinvar_controls_job_run.id, JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), ) + # Second run skipped the resolution entirely (version-keyed skip). + assert resolve_spy.await_count == 1 - assert isinstance(result1, JobExecutionOutcome) - assert result1.status == JobStatus.SUCCEEDED - assert isinstance(result2, JobExecutionOutcome) - assert result2.status == JobStatus.SUCCEEDED - - # Verify only one clinical control annotation exists for the variant - clinical_controls = session.query(ClinicalControl).all() - assert len(clinical_controls) == 1 - - # Verify two annotated variants exist but both reflect the same successful annotation, and only - # one is current - annotated_variants = session.query(VariantAnnotationStatus).all() - assert len(annotated_variants) == 2 - statuses = [av.status for av in annotated_variants] - assert statuses.count(AnnotationStatus.SUCCESS) == 2 - current_statuses = [av for av in annotated_variants if av.current] - assert len(current_statuses) == 1 - - async def test_refresh_clinvar_controls_partial_failure( + assert first.data["created_link_count"] == 1 + assert second.data["preexisting_link_count"] == 1 + assert second.data["created_link_count"] == 0 + + # One control, one live link — no churn. + assert len(session.scalars(select(ClinvarControl)).all()) == 1 + links = session.scalars(select(ClinvarAlleleLink).where(ClinvarAlleleLink.allele_id == allele.id)).all() + assert len(links) == 1 + assert links[0].valid_to is None + + # Per-version VAS: two rows for the variant, only one current. + statuses = session.query(VariantAnnotationStatus).filter_by(variant_id=variant.id).all() + assert len(statuses) == 2 + assert len([s for s in statuses if s.current]) == 1 + + async def test_release_reresolution_supersedes_newest_wins( self, mock_worker_ctx, session, with_refresh_clinvar_controls_job, sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, ): - """Test that the job handles partial failures gracefully.""" - - variant1, mapped_variant1 = setup_sample_variants_with_caid - - # Add an additional mapped variant to the database with a CAID - score_set = session.get(ScoreSet, sample_refresh_clinvar_controls_job_run.job_params["score_set_id"]) - variant2 = Variant( - urn="urn:variant:test-variant-with-caid-2", - score_set_id=score_set.id, - hgvs_nt="NM_000000.1:c.4G>C", - hgvs_pro="NP_000000.1:p.Gly4Ala", - data={"hgvs_c": "NM_000000.1:c.4G>C", "hgvs_p": "NP_000000.1:p.Gly4Ala"}, - ) - session.add(variant2) - session.commit() - mapped_variant2 = MappedVariant( - variant_id=variant2.id, - clingen_allele_id="CA125", - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant2) + """Defensive guard: if an allele re-resolves to a *different* control within the same release + (should never happen — archival data is immutable), the old live link is superseded + newest-wins, leaving exactly one live link per (allele, version).""" + variant, allele = setup_sample_alleles_with_caid + sample_refresh_clinvar_controls_job_run.job_params = { + **sample_refresh_clinvar_controls_job_run.job_params, + "force": True, + } session.commit() - # Mock the get_associated_clinvar_allele_id function to raise an exception for the first call - def side_effect_get_associated_clinvar_allele_id(clingen_allele_id): - if clingen_allele_id == "CA125": - raise requests.exceptions.RequestException("ClinGen API error") - return "VCV000000123" + tsv = { + "VCV000000123": { + "GeneSymbol": "A", + "ClinicalSignificance": "benign", + "ReviewStatus": "ok", + "VariationID": "111", + }, + "VCV000000999": { + "GeneSymbol": "B", + "ClinicalSignificance": "pathogenic", + "ReviewStatus": "ok", + "VariationID": "222", + }, + } with ( patch( "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - side_effect=side_effect_get_associated_clinvar_allele_id, + side_effect=["VCV000000123", "VCV000000999"], ), patch( "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, + return_value=tsv, ), ): - result = await refresh_clinvar_controls( + await refresh_clinvar_controls( mock_worker_ctx, sample_refresh_clinvar_controls_job_run.id, JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), ) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - # Verify annotation statuses for both variants - variant_with_api_failure = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant2.variant_id) - .one() - ) - assert variant_with_api_failure.status == AnnotationStatus.FAILED - assert variant_with_api_failure.annotation_type == AnnotationType.CLINVAR_CONTROL - assert "Failed to retrieve ClinVar allele ID from ClinGen API" in variant_with_api_failure.error_message - - annotated_variant2 = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant1.variant_id) - .one() - ) - assert annotated_variant2.status == AnnotationStatus.SUCCESS - assert annotated_variant2.annotation_type == AnnotationType.CLINVAR_CONTROL - assert annotated_variant2.error_message is None - - async def test_total_api_failure_returns_failed( - self, - mock_worker_ctx, - session, - with_refresh_clinvar_controls_job, - sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, - ): - """Test that the job returns FAILED when all ClinVar lookups fail.""" - with ( - patch( - "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - side_effect=requests.exceptions.RequestException("ClinGen API error"), - ), - patch( - "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, - ), - ): - result = await refresh_clinvar_controls( + session.commit() + await refresh_clinvar_controls( mock_worker_ctx, sample_refresh_clinvar_controls_job_run.id, JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), ) - assert result.status == JobStatus.FAILED - assert result.failure_category == FailureCategory.DEPENDENCY_FAILURE + control_a = session.scalars(select(ClinvarControl).where(ClinvarControl.db_identifier == "VCV000000123")).one() + control_b = session.scalars(select(ClinvarControl).where(ClinvarControl.db_identifier == "VCV000000999")).one() - async def test_upsert_does_not_create_duplicate_control_when_row_already_exists( + all_links = session.scalars(select(ClinvarAlleleLink).where(ClinvarAlleleLink.allele_id == allele.id)).all() + live_links = [link for link in all_links if link.valid_to is None] + + # Exactly one live link per (allele, version) — the new control — and the old one is retired. + assert len(live_links) == 1 + assert live_links[0].clinvar_control_id == control_b.id + retired = [link for link in all_links if link.clinvar_control_id == control_a.id] + assert len(retired) == 1 + assert retired[0].valid_to is not None + # Gap-free handoff: the retired link's valid_to equals the successor's valid_from. + assert retired[0].valid_to == live_links[0].valid_from + + async def test_force_reresolves_without_duplicating_link( self, mock_worker_ctx, session, with_refresh_clinvar_controls_job, sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, ): - """Test that the upsert handles a pre-existing ClinicalControl row without creating a duplicate. - - This covers the concurrent-job race condition where two refresh_clinvar_controls - jobs run simultaneously for different score sets and both try to insert a - ClinicalControl for the same (db_name, db_identifier, db_version). The second - job's upsert must hit ON CONFLICT DO UPDATE rather than inserting a second row. - """ - # Simulate the state left by a concurrent job: the ClinicalControl row - # for this identifier/version already exists in the DB with stale data. - pre_existing_control = ClinicalControl( - db_name="ClinVar", - db_identifier="VCV000000123", - db_version="01_2026", - gene_symbol="OLD_SYMBOL", - clinical_significance="likely pathogenic", - clinical_review_status="criteria provided, single submitter", - ) - session.add(pre_existing_control) + """force bypasses the version-keyed skip and re-resolves, but the get-or-create link write + does not duplicate an existing live link.""" + variant, allele = setup_sample_alleles_with_caid + sample_refresh_clinvar_controls_job_run.job_params = { + **sample_refresh_clinvar_controls_job_run.job_params, + "force": True, + } session.commit() with ( patch( "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", return_value="VCV000000123", - ), + ) as resolve_spy, patch( "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", return_value=MOCK_CLINVAR_DATA, ), ): - result = await refresh_clinvar_controls( + await refresh_clinvar_controls( + mock_worker_ctx, + sample_refresh_clinvar_controls_job_run.id, + JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), + ) + session.commit() + second = await refresh_clinvar_controls( mock_worker_ctx, sample_refresh_clinvar_controls_job_run.id, JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), ) - assert result.status == JobStatus.SUCCEEDED - - # Only one row should exist — the upsert must not have inserted a second one. - controls = session.query(ClinicalControl).filter_by(db_identifier="VCV000000123", db_version="01_2026").all() - assert len(controls) == 1 - - # The upsert should have updated the stale data from the concurrent job. - session.refresh(controls[0]) - assert controls[0].gene_symbol == "TEST" - assert controls[0].clinical_significance == "benign" - assert controls[0].clinical_review_status == "reviewed by expert panel" + # force re-resolved on the second run despite an existing link. + assert resolve_spy.await_count == 2 + assert second.data["preexisting_link_count"] == 1 + assert second.data["created_link_count"] == 0 + links = session.scalars(select(ClinvarAlleleLink).where(ClinvarAlleleLink.allele_id == allele.id)).all() + assert len(links) == 1 + assert links[0].valid_to is None @pytest.mark.integration @pytest.mark.asyncio -class TestRefreshClinvarControlsIntegration: - """Integration tests for the refresh_clinvar_controls job function.""" +class TestRefreshClinvarControlsArqContext: + """End-to-end tests for refresh_clinvar_controls within an ARQ worker context.""" @pytest.fixture(autouse=True) def _mock_clinvar_versions(self): - """Mock generate_clinvar_versions to return a single version for testing.""" with patch( - "mavedb.worker.jobs.external_services.clinvar.generate_clinvar_versions", + "mavedb.worker.jobs.external_services.clinvar._generate_clinvar_versions", return_value=[(2026, 1)], ): yield - async def test_refresh_clinvar_controls_no_mapped_variants( - self, - session, - with_populated_domain_data, - with_refresh_clinvar_controls_job, - mock_worker_ctx, - sample_refresh_clinvar_controls_job_run, - ): - """Integration test: job completes successfully when there are no mapped variants.""" - - with ( - patch( - "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, - ), - ): - result = await refresh_clinvar_controls(mock_worker_ctx, sample_refresh_clinvar_controls_job_run.id) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - # Verify no controls were added - clinical_controls = session.query(ClinicalControl).all() - assert len(clinical_controls) == 0 - - # Verify no annotation statuses were created - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 0 - - # Verify job run status is marked as completed - session.refresh(sample_refresh_clinvar_controls_job_run) - assert sample_refresh_clinvar_controls_job_run.status == JobStatus.SUCCEEDED - - async def test_refresh_clinvar_controls_no_variants_with_caid( + async def test_arq_context_successful_link( self, + arq_redis, + arq_worker, session, with_populated_domain_data, with_refresh_clinvar_controls_job, - mock_worker_ctx, sample_refresh_clinvar_controls_job_run, + setup_sample_alleles_with_caid, ): - """Integration test: job completes successfully when no variants have CAIDs.""" - # Add a variant without a CAID - score_set = session.get(ScoreSet, sample_refresh_clinvar_controls_job_run.job_params["score_set_id"]) - variant = Variant( - urn="urn:variant:integration-test-variant-no-caid", - score_set_id=score_set.id, - hgvs_nt="NM_000000.1:c.5T>A", - hgvs_pro="NP_000000.1:p.Leu5Gln", - data={"hgvs_c": "NM_000000.1:c.5T>A", "hgvs_p": "NP_000000.1:p.Leu5Gln"}, - ) - session.add(variant) - session.commit() - mapped_variant = MappedVariant( - variant_id=variant.id, - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant) - session.commit() - + """The job links an allele and records a SUCCESS annotation under an ARQ worker.""" with ( patch( - "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, + "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", + return_value="VCV000000123", ), - ): - result = await refresh_clinvar_controls(mock_worker_ctx, sample_refresh_clinvar_controls_job_run.id) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - # Verify an annotation status was created for the variant without a CAID - variant_no_caid = ( - session.query(VariantAnnotationStatus).filter(VariantAnnotationStatus.variant_id == variant.id).one() - ) - assert variant_no_caid.status == AnnotationStatus.SKIPPED - assert variant_no_caid.annotation_type == AnnotationType.CLINVAR_CONTROL - assert variant_no_caid.error_message == "Mapped variant does not have an associated ClinGen allele ID." - - # Verify no clinical controls were added - clinical_controls = session.query(ClinicalControl).all() - assert len(clinical_controls) == 0 - - # Verify job run status is marked as completed - session.refresh(sample_refresh_clinvar_controls_job_run) - assert sample_refresh_clinvar_controls_job_run.status == JobStatus.SUCCEEDED - - async def test_refresh_clinvar_controlsvariants_are_multivariants( - self, - session, - with_populated_domain_data, - with_refresh_clinvar_controls_job, - mock_worker_ctx, - sample_refresh_clinvar_controls_job_run, - ): - """Integration test: job completes successfully when all variants are multi-variant CAIDs.""" - # Add a variant with a multi-variant CAID - score_set = session.get(ScoreSet, sample_refresh_clinvar_controls_job_run.job_params["score_set_id"]) - variant = Variant( - urn="urn:variant:integration-test-variant-multicid", - score_set_id=score_set.id, - hgvs_nt="NM_000000.1:c.6A>G", - hgvs_pro="NP_000000.1:p.Thr6Ala", - data={"hgvs_c": "NM_000000.1:c.6A>G", "hgvs_p": "NP_000000.1:p.Thr6Ala"}, - ) - session.add(variant) - session.commit() - mapped_variant = MappedVariant( - variant_id=variant.id, - clingen_allele_id="CA-MULTI-003,CA-MULTI-004", - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant) - session.commit() - - with ( patch( "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", return_value=MOCK_CLINVAR_DATA, ), ): - result = await refresh_clinvar_controls(mock_worker_ctx, sample_refresh_clinvar_controls_job_run.id) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED + await arq_redis.enqueue_job("refresh_clinvar_controls", sample_refresh_clinvar_controls_job_run.id) + await arq_worker.async_run() + await arq_worker.run_check() - # Verify an annotation status was created for the multi-variant CAID - variant_with_multicid = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant.variant_id) - .one() - ) - assert variant_with_multicid.status == AnnotationStatus.SKIPPED - assert variant_with_multicid.annotation_type == AnnotationType.CLINVAR_CONTROL - assert ( - variant_with_multicid.error_message - == "Multi-variant ClinGen allele IDs cannot be associated with ClinVar data." - ) + assert len(session.scalars(select(ClinvarControl)).all()) >= 1 + assert len(session.scalars(select(ClinvarAlleleLink).where(ClinvarAlleleLink.current)).all()) == 1 - # Verify no clinical controls were added - clinical_controls = session.query(ClinicalControl).all() - assert len(clinical_controls) == 0 + statuses = session.query(VariantAnnotationStatus).all() + assert len(statuses) == 1 + assert statuses[0].status == AnnotationStatus.SUCCESS + assert statuses[0].annotation_type == AnnotationType.CLINVAR_CONTROL - # Verify job run status is marked as completed session.refresh(sample_refresh_clinvar_controls_job_run) assert sample_refresh_clinvar_controls_job_run.status == JobStatus.SUCCEEDED - async def test_refresh_clinvar_controls_no_associated_clinvar_allele_id( + async def test_arq_context_exception_handling( self, + arq_redis, + arq_worker, session, with_populated_domain_data, with_refresh_clinvar_controls_job, - mock_worker_ctx, sample_refresh_clinvar_controls_job_run, + setup_sample_alleles_with_caid, ): - """Integration test: job handles no associated ClinVar Allele ID gracefully.""" - # Add a variant with a CAID - score_set = session.get(ScoreSet, sample_refresh_clinvar_controls_job_run.job_params["score_set_id"]) - variant = Variant( - urn="urn:variant:integration-test-variant-with-caid", - score_set_id=score_set.id, - hgvs_nt="NM_000000.1:c.7C>A", - hgvs_pro="NP_000000.1:p.Ser7Tyr", - data={"hgvs_c": "NM_000000.1:c.7C>A", "hgvs_p": "NP_000000.1:p.Ser7Tyr"}, - ) - session.add(variant) - session.commit() - mapped_variant = MappedVariant( - variant_id=variant.id, - clingen_allele_id="CA126", - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant) - session.commit() - - # Mock the get_associated_clinvar_allele_id function to return None + """An unexpected error during resolution is caught by the decorators and the job errors.""" with ( patch( "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - return_value=None, + side_effect=ValueError("Unexpected error"), ), patch( "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", return_value=MOCK_CLINVAR_DATA, ), + patch("mavedb.worker.lib.decorators.job_management.send_slack_job_error") as mock_slack, ): - result = await refresh_clinvar_controls(mock_worker_ctx, sample_refresh_clinvar_controls_job_run.id) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - # Verify an annotation status was created for the variant due to no associated ClinVar Allele ID - variant_no_clinvar_allele = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant.variant_id) - .one() - ) - assert variant_no_clinvar_allele.status == AnnotationStatus.SKIPPED - assert variant_no_clinvar_allele.annotation_type == AnnotationType.CLINVAR_CONTROL - assert "No ClinVar allele ID found for ClinGen allele ID" in variant_no_clinvar_allele.error_message + await arq_redis.enqueue_job("refresh_clinvar_controls", sample_refresh_clinvar_controls_job_run.id) + await arq_worker.async_run() + await arq_worker.run_check() - # Verify no clinical controls were added - clinical_controls = session.query(ClinicalControl).all() - assert len(clinical_controls) == 0 + mock_slack.assert_called_once() + assert session.scalars(select(ClinvarAlleleLink)).all() == [] + assert session.query(VariantAnnotationStatus).all() == [] - # Verify job run status is marked as completed - session.refresh(sample_refresh_clinvar_controls_job_run) - assert sample_refresh_clinvar_controls_job_run.status == JobStatus.SUCCEEDED - - async def test_refresh_clinvar_controls_no_clinvar_data( - self, - session, - with_populated_domain_data, - with_refresh_clinvar_controls_job, - mock_worker_ctx, - sample_refresh_clinvar_controls_job_run, - ): - """Integration test: job handles no ClinVar data found for the associated ClinVar Allele ID.""" - # Add a variant with a CAID - score_set = session.get(ScoreSet, sample_refresh_clinvar_controls_job_run.job_params["score_set_id"]) - variant = Variant( - urn="urn:variant:integration-test-variant-with-caid", - score_set_id=score_set.id, - hgvs_nt="NM_000000.1:c.8G>T", - hgvs_pro="NP_000000.1:p.Val8Phe", - data={"hgvs_c": "NM_000000.1:c.8G>T", "hgvs_p": "NP_000000.1:p.Val8Phe"}, - ) - session.add(variant) - session.commit() - mapped_variant = MappedVariant( - variant_id=variant.id, - clingen_allele_id="CA127", - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant) - session.commit() - - # Mock the get_associated_clinvar_allele_id function to return a ClinVar Allele ID - with ( - patch( - "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - return_value="VCV000000001", - ), - patch( - "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, - ), - ): - result = await refresh_clinvar_controls(mock_worker_ctx, sample_refresh_clinvar_controls_job_run.id) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - # Verify an annotation status was created for the variant due to no ClinVar data found - variant_no_clinvar_data = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant.variant_id) - .one() - ) - assert variant_no_clinvar_data.status == AnnotationStatus.SKIPPED - assert variant_no_clinvar_data.annotation_type == AnnotationType.CLINVAR_CONTROL - assert "No ClinVar data found for ClinVar allele ID" in variant_no_clinvar_data.error_message - - # Verify no clinical controls were added - clinical_controls = session.query(ClinicalControl).all() - assert len(clinical_controls) == 0 - - # Verify job run status is marked as completed - session.refresh(sample_refresh_clinvar_controls_job_run) - assert sample_refresh_clinvar_controls_job_run.status == JobStatus.SUCCEEDED - - async def test_refresh_clinvar_controls_successful_annotation_existing_control( - self, - session, - with_populated_domain_data, - with_refresh_clinvar_controls_job, - mock_worker_ctx, - sample_refresh_clinvar_controls_job_run, - ): - """Integration test: job successfully annotates a variant with ClinVar control data.""" - # Add a variant with a CAID - score_set = session.get(ScoreSet, sample_refresh_clinvar_controls_job_run.job_params["score_set_id"]) - variant = Variant( - urn="urn:variant:integration-test-variant-with-caid", - score_set_id=score_set.id, - hgvs_nt="NM_000000.1:c.9A>C", - hgvs_pro="NP_000000.1:p.Lys9Thr", - data={"hgvs_c": "NM_000000.1:c.9A>C", "hgvs_p": "NP_000000.1:p.Lys9Thr"}, - ) - session.add(variant) - session.commit() - mapped_variant = MappedVariant( - variant_id=variant.id, - clingen_allele_id="CA128", - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant) - session.commit() - clinical_control = ClinicalControl( - db_name="ClinVar", - db_identifier="VCV000000123", - clinical_significance="likely pathogenic", - gene_symbol="TEST", - clinical_review_status="criteria provided, single submitter", - db_version="01_2026", - ) - session.add(clinical_control) - session.commit() - - mapped_variant.clinical_controls.append(clinical_control) - session.commit() - - # Mock the get_associated_clinvar_allele_id function to return a ClinVar Allele ID - with ( - patch( - "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - return_value="VCV000000123", - ), - patch( - "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, - ), - ): - result = await refresh_clinvar_controls(mock_worker_ctx, sample_refresh_clinvar_controls_job_run.id) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - # Verify an annotation status was created for the variant with successful annotation - annotated_variant = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant.variant_id) - .one() - ) - assert annotated_variant.status == AnnotationStatus.SUCCESS - assert annotated_variant.annotation_type == AnnotationType.CLINVAR_CONTROL - assert annotated_variant.error_message is None - - # Verify the clinical control was updated - session.refresh(clinical_control) - assert clinical_control.clinical_significance == "benign" - assert clinical_control.clinical_review_status == "reviewed by expert panel" - assert mapped_variant in clinical_control.mapped_variants - - # Verify job run status is marked as completed - session.refresh(sample_refresh_clinvar_controls_job_run) - assert sample_refresh_clinvar_controls_job_run.status == JobStatus.SUCCEEDED - - async def test_refresh_clinvar_controls_successful_annotation_new_control( - self, - session, - with_populated_domain_data, - with_refresh_clinvar_controls_job, - mock_worker_ctx, - sample_refresh_clinvar_controls_job_run, - ): - """Integration test: job successfully annotates a variant with ClinVar control data when no prior status exists.""" - # Add a variant with a CAID - score_set = session.get(ScoreSet, sample_refresh_clinvar_controls_job_run.job_params["score_set_id"]) - variant = Variant( - urn="urn:variant:integration-test-variant-with-caid", - score_set_id=score_set.id, - hgvs_nt="NM_000000.1:c.10C>G", - hgvs_pro="NP_000000.1:p.Pro10Arg", - data={"hgvs_c": "NM_000000.1:c.10C>G", "hgvs_p": "NP_000000.1:p.Pro10Arg"}, - ) - session.add(variant) - session.commit() - mapped_variant = MappedVariant( - variant_id=variant.id, - clingen_allele_id="CA129", - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant) - session.commit() - - # Mock the get_associated_clinvar_allele_id function to return a ClinVar Allele ID - with ( - patch( - "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - return_value="VCV000000123", - ), - patch( - "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, - ), - ): - result = await refresh_clinvar_controls(mock_worker_ctx, sample_refresh_clinvar_controls_job_run.id) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - # Verify an annotation status was created for the variant with successful annotation - annotated_variant = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant.variant_id) - .one() - ) - assert annotated_variant.status == AnnotationStatus.SUCCESS - assert annotated_variant.annotation_type == AnnotationType.CLINVAR_CONTROL - assert annotated_variant.error_message is None - - # Verify the clinical control was added - clinical_control = ( - session.query(ClinicalControl).filter(ClinicalControl.mapped_variants.contains(mapped_variant)).one() - ) - assert clinical_control.db_identifier == "VCV000000123" - - # Verify job run status is marked as completed - session.refresh(sample_refresh_clinvar_controls_job_run) - assert sample_refresh_clinvar_controls_job_run.status == JobStatus.SUCCEEDED - - async def test_refresh_clinvar_controls_successful_annotation_pipeline_context( - self, - session, - with_populated_domain_data, - with_refresh_clinvar_controls_job, - mock_worker_ctx, - sample_refresh_clinvar_controls_pipeline, - sample_refresh_clinvar_controls_job_in_pipeline, - ): - """Integration test: job successfully annotates a variant with ClinVar control data in a pipeline context.""" - # Add a variant with a CAID - score_set = session.get(ScoreSet, sample_refresh_clinvar_controls_job_in_pipeline.job_params["score_set_id"]) - variant = Variant( - urn="urn:variant:integration-test-variant-with-caid", - score_set_id=score_set.id, - hgvs_nt="NM_000000.1:c.12G>A", - hgvs_pro="NP_000000.1:p.Met12Ile", - data={"hgvs_c": "NM_000000.1:c.12G>A", "hgvs_p": "NP_000000.1:p.Met12Ile"}, - ) - session.add(variant) - session.commit() - mapped_variant = MappedVariant( - variant_id=variant.id, - clingen_allele_id="CA130", - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant) - session.commit() - - # Mock the get_associated_clinvar_allele_id function to return a ClinVar Allele ID - with ( - patch( - "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - return_value="VCV000000123", - ), - patch( - "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, - ), - ): - result = await refresh_clinvar_controls(mock_worker_ctx, sample_refresh_clinvar_controls_job_in_pipeline.id) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - # Verify an annotation status was created for the variant with successful annotation - annotated_variant = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant.variant_id) - .one() - ) - assert annotated_variant.status == AnnotationStatus.SUCCESS - assert annotated_variant.annotation_type == AnnotationType.CLINVAR_CONTROL - assert annotated_variant.error_message is None - - # Verify the clinical control was added - clinical_control = ( - session.query(ClinicalControl).filter(ClinicalControl.mapped_variants.contains(mapped_variant)).one() - ) - assert clinical_control.db_identifier == "VCV000000123" - - # Verify job run status is marked as completed - session.refresh(sample_refresh_clinvar_controls_job_in_pipeline) - assert sample_refresh_clinvar_controls_job_in_pipeline.status == JobStatus.SUCCEEDED - - # Verify the pipeline is marked as completed - session.refresh(sample_refresh_clinvar_controls_pipeline) - assert sample_refresh_clinvar_controls_pipeline.status == PipelineStatus.SUCCEEDED - - async def test_refresh_clinvar_controls_idempotent_run( - self, - session, - with_populated_domain_data, - with_refresh_clinvar_controls_job, - mock_worker_ctx, - sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, - ): - """Integration test: running the job multiple times does not create duplicate annotation statuses.""" - - # Mock the get_associated_clinvar_allele_id function to return a ClinVar Allele ID - with ( - patch( - "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - return_value="VCV000000123", - ), - patch( - "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - side_effect=[MOCK_CLINVAR_DATA, MOCK_CLINVAR_DATA], - ), - ): - # First run - result1 = await refresh_clinvar_controls(mock_worker_ctx, sample_refresh_clinvar_controls_job_run.id) - - session.commit() - # reset the job run status to pending for the second run - sample_refresh_clinvar_controls_job_run.status = JobStatus.PENDING - session.commit() - - # Second run - result2 = await refresh_clinvar_controls(mock_worker_ctx, sample_refresh_clinvar_controls_job_run.id) - - assert isinstance(result1, JobExecutionOutcome) - assert result1.status == JobStatus.SUCCEEDED - assert isinstance(result2, JobExecutionOutcome) - assert result2.status == JobStatus.SUCCEEDED - - # Verify only one clinical control annotation exists for the variant - clinical_controls = session.query(ClinicalControl).all() - assert len(clinical_controls) == 1 - - # Verify two annotated variants exist but both reflect the same successful annotation, and only - # one is current - annotated_variants = session.query(VariantAnnotationStatus).all() - assert len(annotated_variants) == 2 - statuses = [av.status for av in annotated_variants] - assert statuses.count(AnnotationStatus.SUCCESS) == 2 - current_statuses = [av for av in annotated_variants if av.current] - assert len(current_statuses) == 1 - - # Verify job run status is marked as completed - session.refresh(sample_refresh_clinvar_controls_job_run) - assert sample_refresh_clinvar_controls_job_run.status == JobStatus.SUCCEEDED - - async def test_refresh_clinvar_controls_partial_failure( - self, - session, - with_populated_domain_data, - with_refresh_clinvar_controls_job, - mock_worker_ctx, - sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, - ): - """Integration test: job handles partial failures gracefully.""" - - variant1, mapped_variant1 = setup_sample_variants_with_caid - # Add an additional mapped variant to the database with a CAID - score_set = session.get(ScoreSet, sample_refresh_clinvar_controls_job_run.job_params["score_set_id"]) - variant2 = Variant( - urn="urn:variant:integration-test-variant-with-caid-2", - score_set_id=score_set.id, - hgvs_nt="NM_000000.1:c.11G>C", - hgvs_pro="NP_000000.1:p.Gly11Ala", - data={"hgvs_c": "NM_000000.1:c.11G>C", "hgvs_p": "NP_000000.1:p.Gly11Ala"}, - ) - session.add(variant2) - session.commit() - mapped_variant2 = MappedVariant( - variant_id=variant2.id, - clingen_allele_id="CA130", - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant2) - session.commit() - - # Mock the get_associated_clinvar_allele_id function to raise an exception for the first call - def side_effect_get_associated_clinvar_allele_id(clingen_allele_id): - if clingen_allele_id == "CA130": - raise requests.exceptions.RequestException("ClinGen API error") - return "VCV000000123" - - with ( - patch( - "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - side_effect=side_effect_get_associated_clinvar_allele_id, - ), - patch( - "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, - ), - ): - result = await refresh_clinvar_controls(mock_worker_ctx, sample_refresh_clinvar_controls_job_run.id) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - # Verify annotation statuses for both variants - variant_with_api_failure = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant2.variant_id) - .one() - ) - assert variant_with_api_failure.status == AnnotationStatus.FAILED - assert variant_with_api_failure.annotation_type == AnnotationType.CLINVAR_CONTROL - assert "Failed to retrieve ClinVar allele ID from ClinGen API" in variant_with_api_failure.error_message - - annotated_variant2 = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant1.variant_id) - .one() - ) - assert annotated_variant2.status == AnnotationStatus.SUCCESS - assert annotated_variant2.annotation_type == AnnotationType.CLINVAR_CONTROL - assert annotated_variant2.error_message is None - - # Verify a clinical control was added for the successfully annotated variant and not the unsuccessful one - clinical_control1 = ( - session.query(ClinicalControl).filter(ClinicalControl.mapped_variants.contains(mapped_variant1)).one() - ) - assert clinical_control1.db_identifier == "VCV000000123" - - clinical_control2 = ( - session.query(ClinicalControl).filter(ClinicalControl.mapped_variants.contains(mapped_variant2)).all() - ) - assert len(clinical_control2) == 0 - - # Verify job run status is marked as completed - session.refresh(sample_refresh_clinvar_controls_job_run) - assert sample_refresh_clinvar_controls_job_run.status == JobStatus.SUCCEEDED - - async def test_refresh_clinvar_controls_propagates_exceptions_to_decorator( - self, - mock_worker_ctx, - session, - with_refresh_clinvar_controls_job, - sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, - ): - """Test that unexpected exceptions are propagated.""" - - # Mock the get_associated_clinvar_allele_id function to raise an unexpected exception - with ( - patch( - "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - side_effect=ValueError("Unexpected error"), - ), - patch( - "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, - ), - ): - result = await refresh_clinvar_controls( - mock_worker_ctx, - sample_refresh_clinvar_controls_job_run.id, - JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), - ) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.ERRORED - - # Verify no annotation statuses were created - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 0 - - # Verify no clinical controls were added - clinical_controls = session.query(ClinicalControl).all() - assert len(clinical_controls) == 0 - - # Verify job run status is marked as errored (unhandled exception caught by decorator) - session.refresh(sample_refresh_clinvar_controls_job_run) - assert sample_refresh_clinvar_controls_job_run.status == JobStatus.ERRORED - - -@pytest.mark.asyncio -@pytest.mark.integration -class TestRefreshClinvarControlsArqContext: - """Tests for running the refresh_clinvar_controls job function within an ARQ worker context.""" - - @pytest.fixture(autouse=True) - def _mock_clinvar_versions(self): - """Mock generate_clinvar_versions to return a single version for testing.""" - with patch( - "mavedb.worker.jobs.external_services.clinvar.generate_clinvar_versions", - return_value=[(2026, 1)], - ): - yield - - async def test_refresh_clinvar_controls_with_arq_context_independent( - self, - arq_redis, - arq_worker, - session, - with_populated_domain_data, - with_refresh_clinvar_controls_job, - sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, - ): - """Integration test: job completes successfully within an ARQ worker context.""" - - # Patch external service calls - with ( - patch( - "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - return_value="VCV000000123", - ), - patch( - "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, - ), - ): - await arq_redis.enqueue_job("refresh_clinvar_controls", sample_refresh_clinvar_controls_job_run.id) - await arq_worker.async_run() - await arq_worker.run_check() - - # Verify that clinical controls were added - clinical_controls = session.query(ClinicalControl).all() - assert len(clinical_controls) > 0 - - # Verify annotation status was created - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == AnnotationStatus.SUCCESS - assert annotation_statuses[0].annotation_type == AnnotationType.CLINVAR_CONTROL - - # Verify that the job completed successfully - session.refresh(sample_refresh_clinvar_controls_job_run) - assert sample_refresh_clinvar_controls_job_run.status == JobStatus.SUCCEEDED - - async def test_refresh_clinvar_controls_with_arq_context_pipeline( - self, - arq_redis, - arq_worker, - session, - with_populated_domain_data, - with_refresh_clinvar_controls_job, - sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, - ): - """Integration test: job completes successfully within an ARQ worker context in a pipeline context.""" - - # Patch external service calls - with ( - patch( - "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - return_value="VCV000000123", - ), - patch( - "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, - ), - ): - await arq_redis.enqueue_job("refresh_clinvar_controls", sample_refresh_clinvar_controls_job_run.id) - await arq_worker.async_run() - await arq_worker.run_check() - - # Verify that clinical controls were added - clinical_controls = session.query(ClinicalControl).all() - assert len(clinical_controls) > 0 - - # Verify annotation status was created - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == AnnotationStatus.SUCCESS - assert annotation_statuses[0].annotation_type == AnnotationType.CLINVAR_CONTROL - - # Verify that the job completed successfully - session.refresh(sample_refresh_clinvar_controls_job_run) - assert sample_refresh_clinvar_controls_job_run.status == JobStatus.SUCCEEDED - - # Verify the pipeline is marked as completed - pass - - async def test_refresh_clinvar_controls_with_arq_context_exception_handling_independent( - self, - arq_redis, - arq_worker, - session, - with_populated_domain_data, - with_refresh_clinvar_controls_job, - sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, - ): - """Integration test: job handles exceptions properly within an ARQ worker context.""" - # Patch external service calls to raise an exception - with ( - patch( - "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - side_effect=ValueError("Unexpected error"), - ), - patch( - "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, - ), - ): - await arq_redis.enqueue_job("refresh_clinvar_controls", sample_refresh_clinvar_controls_job_run.id) - await arq_worker.async_run() - await arq_worker.run_check() - - # Verify no annotation statuses were created - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 0 - - # Verify no clinical controls were added - clinical_controls = session.query(ClinicalControl).all() - assert len(clinical_controls) == 0 - - # Verify job run status is marked as errored (unhandled exception caught by decorator) - session.refresh(sample_refresh_clinvar_controls_job_run) - assert sample_refresh_clinvar_controls_job_run.status == JobStatus.ERRORED - - async def test_refresh_clinvar_controls_with_arq_context_exception_handling_pipeline( - self, - arq_redis, - arq_worker, - session, - with_populated_domain_data, - with_refresh_clinvar_controls_job, - sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, - ): - """Integration test: job handles exceptions properly within an ARQ worker context in a pipeline context.""" - # Patch external service calls to raise an exception - with ( - patch( - "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - side_effect=ValueError("Unexpected error"), - ), - patch( - "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, - ), - ): - await arq_redis.enqueue_job("refresh_clinvar_controls", sample_refresh_clinvar_controls_job_run.id) - await arq_worker.async_run() - await arq_worker.run_check() - - # Verify no annotation statuses were created - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 0 - - # Verify no clinical controls were added - clinical_controls = session.query(ClinicalControl).all() - assert len(clinical_controls) == 0 - - # Verify job run status is marked as errored (unhandled exception caught by decorator) session.refresh(sample_refresh_clinvar_controls_job_run) assert sample_refresh_clinvar_controls_job_run.status == JobStatus.ERRORED - - # Verify the pipeline is marked as failed - pass From 3d9b5fa836d39ad7c48dc31f53968ec92e8cab40 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 23 Jun 2026 09:26:05 -0700 Subject: [PATCH 032/252] refactor(annotation): retire the HGVS and variant-translation jobs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Steps 4 & 5 of the #742 migration. Both jobs are redundant under the allele model: Allele.hgvs_g/c/p are populated by the mapping job, and the reverse-translation equivalence space (genomic/coding/protein alleles linked per MappingRecord) replaces the ClinGen PA<->CA translation table. - Remove populate_hgvs_for_score_set and populate_variant_translations_for_score_set from the pipeline DAG (both were leaf nodes — no dependency edges to repair), the worker registry (BACKGROUND_FUNCTIONS + STANDALONE_JOB_DEFINITIONS), and the external_services package exports. - Delete the two job modules and the now-orphaned ClinGen HGVS helpers (extract_hgvs_from_ca_allele_data / extract_hgvs_from_pa_allele_data), used only by the HGVS job. - Keep lib/variant_translations.py and the variant_translations table/model, marked FROZEN (serving-only) — they back old-model serving and are dropped at read-cutover. - Delete the obsolete job tests and their conftest fixtures. --- src/mavedb/lib/clingen/allele_registry.py | 74 -- src/mavedb/lib/variant_translations.py | 6 + src/mavedb/lib/workflow/definitions.py | 20 - src/mavedb/models/variant_translation.py | 4 + .../worker/jobs/external_services/__init__.py | 6 - .../worker/jobs/external_services/hgvs.py | 298 ------- .../external_services/variant_translation.py | 371 --------- src/mavedb/worker/jobs/registry.py | 18 - tests/worker/jobs/conftest.py | 192 ----- .../external_services/network/test_hgvs.py | 54 -- .../network/test_variant_translations.py | 56 -- .../jobs/external_services/test_hgvs.py | 544 ------------- .../test_variant_translation.py | 770 ------------------ 13 files changed, 10 insertions(+), 2403 deletions(-) delete mode 100644 src/mavedb/worker/jobs/external_services/hgvs.py delete mode 100644 src/mavedb/worker/jobs/external_services/variant_translation.py delete mode 100644 tests/worker/jobs/external_services/network/test_hgvs.py delete mode 100644 tests/worker/jobs/external_services/network/test_variant_translations.py delete mode 100644 tests/worker/jobs/external_services/test_hgvs.py delete mode 100644 tests/worker/jobs/external_services/test_variant_translation.py diff --git a/src/mavedb/lib/clingen/allele_registry.py b/src/mavedb/lib/clingen/allele_registry.py index b773e6891..b28a184d1 100644 --- a/src/mavedb/lib/clingen/allele_registry.py +++ b/src/mavedb/lib/clingen/allele_registry.py @@ -136,80 +136,6 @@ async def get_associated_clinvar_allele_id(clingen_allele_id: str) -> str: return "" -def extract_hgvs_from_ca_allele_data( - data: dict, - target_is_coding: bool, - transcript_accession: Optional[str], -) -> tuple[Optional[str], Optional[str], Optional[str]]: - """Extract HGVS strings from ClinGen allele data for a CA (canonical allele) ID. - - Parses the ClinGen API response to find GRCh38 genomic HGVS, coding HGVS - matching the target transcript (or MANE fallback), and protein HGVS. - - Args: - data: Parsed JSON response from the ClinGen Allele Registry API. - target_is_coding: Whether the score set target is protein-coding. - transcript_accession: Specific transcript accession to match, or None to use MANE. - - Returns: - Tuple of (hgvs_g, hgvs_c, hgvs_p), any of which may be None. - """ - hgvs_g: Optional[str] = None - hgvs_c: Optional[str] = None - hgvs_p: Optional[str] = None - - if data.get("genomicAlleles"): - for allele in data["genomicAlleles"]: - if allele.get("referenceGenome") == "GRCh38" and allele.get("hgvs"): - hgvs_g = allele["hgvs"][0] - break - - if target_is_coding and data.get("transcriptAlleles"): - if transcript_accession: - for allele in data["transcriptAlleles"]: - if allele.get("hgvs"): - for hgvs_string in allele["hgvs"]: - hgvs_reference_sequence = hgvs_string.split(":")[0] - if transcript_accession == hgvs_reference_sequence: - hgvs_c = hgvs_string - break - if hgvs_c: - if allele.get("proteinEffect"): - hgvs_p = allele["proteinEffect"].get("hgvs") - break - else: - # No transcript specified; use MANE if available - for allele in data["transcriptAlleles"]: - if allele.get("MANE"): - hgvs_c = allele["MANE"].get("nucleotide", {}).get("RefSeq", {}).get("hgvs") - hgvs_p = allele["MANE"].get("protein", {}).get("RefSeq", {}).get("hgvs") - break - - return hgvs_g, hgvs_c, hgvs_p - - -def extract_hgvs_from_pa_allele_data(data: dict) -> tuple[Optional[str], Optional[str], Optional[str]]: - """Extract HGVS strings from ClinGen allele data for a PA (protein allele) ID. - - For PA alleles, only hgvs_p is extracted from aminoAcidAlleles. - - Args: - data: Parsed JSON response from the ClinGen Allele Registry API. - - Returns: - Tuple of (None, None, hgvs_p), where hgvs_p may be None. - """ - hgvs_p: Optional[str] = None - - if data.get("aminoAcidAlleles"): - for allele in data["aminoAcidAlleles"]: - if allele.get("hgvs"): - hgvs_p = allele["hgvs"][0] - break - - return None, None, hgvs_p - - def expand_allele_ids(clingen_allele_ids: list[Optional[str]]) -> set[str]: """Expand comma-separated multi-variant ClinGen allele IDs into individual IDs. diff --git a/src/mavedb/lib/variant_translations.py b/src/mavedb/lib/variant_translations.py index 701cc17d1..31d89e6df 100644 --- a/src/mavedb/lib/variant_translations.py +++ b/src/mavedb/lib/variant_translations.py @@ -3,6 +3,12 @@ This module provides database operations for the variant_translations table, which stores relationships between protein allele (PA) and nucleotide allele (CA) ClinGen IDs. + +FROZEN (serving-only). The populate_variant_translations_for_score_set job that wrote this table was +retired in the #742 migration: the reverse-translation allele equivalence space (genomic/coding/protein +VRS alleles per variant, linked via MappingRecordAllele with HGVS on Allele) now covers PA<->CA +relationships without querying ClinGen. These helpers and the variant_translations table remain only to +serve existing old-model data; they are never written for new score sets and are dropped at read-cutover. """ from typing import cast diff --git a/src/mavedb/lib/workflow/definitions.py b/src/mavedb/lib/workflow/definitions.py index 3d3b70511..93b4b6de1 100644 --- a/src/mavedb/lib/workflow/definitions.py +++ b/src/mavedb/lib/workflow/definitions.py @@ -96,16 +96,6 @@ def annotation_pipeline_job_definitions( }, "dependencies": [("warm_clingen_cache", DependencyType.SUCCESS_REQUIRED)], }, - { - "key": "populate_hgvs_for_score_set", - "function": "populate_hgvs_for_score_set", - "type": JobType.MAPPED_VARIANT_ANNOTATION, - "params": { - "correlation_id": None, # Required param to be filled in at runtime - "score_set_id": None, # Required param to be filled in at runtime - }, - "dependencies": [("warm_clingen_cache", DependencyType.SUCCESS_REQUIRED)], - }, { "key": "populate_vep_for_score_set", "function": "populate_vep_for_score_set", @@ -116,16 +106,6 @@ def annotation_pipeline_job_definitions( }, "dependencies": [("submit_score_set_mappings_to_car", DependencyType.SUCCESS_REQUIRED)], }, - { - "key": "populate_variant_translations_for_score_set", - "function": "populate_variant_translations_for_score_set", - "type": JobType.MAPPED_VARIANT_ANNOTATION, - "params": { - "correlation_id": None, # Required param to be filled in at runtime - "score_set_id": None, # Required param to be filled in at runtime - }, - "dependencies": [("warm_clingen_cache", DependencyType.SUCCESS_REQUIRED)], - }, ] diff --git a/src/mavedb/models/variant_translation.py b/src/mavedb/models/variant_translation.py index a50d36ce2..36aa0dfdf 100644 --- a/src/mavedb/models/variant_translation.py +++ b/src/mavedb/models/variant_translation.py @@ -6,6 +6,10 @@ class VariantTranslation(Base): + """FROZEN (serving-only). Written by the retired populate_variant_translations_for_score_set job; + superseded by the reverse-translation allele equivalence space. Read for existing old-model data, + never written for new score sets, dropped at read-cutover. See lib/variant_translations.py.""" + __tablename__ = "variant_translations" aa_clingen_id = Column(String, nullable=False, primary_key=True) diff --git a/src/mavedb/worker/jobs/external_services/__init__.py b/src/mavedb/worker/jobs/external_services/__init__.py index 4537c0edd..78af92516 100644 --- a/src/mavedb/worker/jobs/external_services/__init__.py +++ b/src/mavedb/worker/jobs/external_services/__init__.py @@ -5,8 +5,6 @@ - ClinGen cache pre-warming to prevent stampede on downstream annotation jobs - UniProt for protein sequence annotation and ID mapping - gnomAD for population frequency and genomic context data -- HGVS for standardized variant nomenclature population -- Variant Translation for PA<->CA allele relationship mapping - VEP for functional consequence annotation """ @@ -18,12 +16,10 @@ from .clingen_cache import warm_clingen_cache from .clinvar import refresh_clinvar_controls from .gnomad import link_gnomad_variants -from .hgvs import populate_hgvs_for_score_set from .uniprot import ( poll_uniprot_mapping_jobs_for_score_set, submit_uniprot_mapping_jobs_for_score_set, ) -from .variant_translation import populate_variant_translations_for_score_set from .vep import populate_vep_for_score_set __all__ = [ @@ -32,8 +28,6 @@ "warm_clingen_cache", "refresh_clinvar_controls", "link_gnomad_variants", - "populate_hgvs_for_score_set", - "populate_variant_translations_for_score_set", "poll_uniprot_mapping_jobs_for_score_set", "submit_uniprot_mapping_jobs_for_score_set", "populate_vep_for_score_set", diff --git a/src/mavedb/worker/jobs/external_services/hgvs.py b/src/mavedb/worker/jobs/external_services/hgvs.py deleted file mode 100644 index 0b4687398..000000000 --- a/src/mavedb/worker/jobs/external_services/hgvs.py +++ /dev/null @@ -1,298 +0,0 @@ -"""ClinGen allele HGVS population jobs for mapped variant annotation. - -This module populates mapped variants with HGVS representations (genomic, coding, -protein) by querying the ClinGen Allele Registry. It uses ClinGen allele IDs -(CAIDs) already associated with mapped variants to look up standardized HGVS -nomenclature at different levels (hgvs_g, hgvs_c, hgvs_p), plus the assay-level -HGVS derived from post-mapped VRS data. -""" - -import logging -from typing import Optional - -import requests -from sqlalchemy import select - -from mavedb.lib.annotation_status_manager import AnnotationStatusManager -from mavedb.lib.clingen.allele_registry import ( - extract_hgvs_from_ca_allele_data, - extract_hgvs_from_pa_allele_data, - get_clingen_allele_data, -) -from mavedb.lib.slack import log_and_send_slack_message -from mavedb.lib.target_genes import get_target_coding_info -from mavedb.lib.types.workflow import JobExecutionOutcome -from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationFailureCategory, AnnotationStatus -from mavedb.models.mapped_variant import MappedVariant -from mavedb.models.score_set import ScoreSet -from mavedb.models.variant import Variant -from mavedb.worker.jobs.utils.setup import validate_job_params -from mavedb.worker.lib.decorators.pipeline_management import with_pipeline_management -from mavedb.worker.lib.managers.job_manager import JobManager - -logger = logging.getLogger(__name__) - - -@with_pipeline_management -async def populate_hgvs_for_score_set(ctx: dict, job_id: int, job_manager: JobManager) -> JobExecutionOutcome: - """Populate mapped variants with HGVS representations for a score set. - - Queries the ClinGen Allele Registry using existing ClinGen allele IDs to populate - standardized HGVS nomenclature (genomic, coding, protein) on mapped variants. - Also extracts the assay-level HGVS from post-mapped VRS data. - - Required job_params in the JobRun: - - score_set_id (int): ID of the ScoreSet to process - - correlation_id (str): Correlation ID for tracking - - Args: - ctx: Worker context containing DB and Redis connections. - job_id: The ID of the job run. - job_manager: Manager for job lifecycle and DB operations. - - Side Effects: - - Updates MappedVariant records with hgvs_assay_level, hgvs_g, hgvs_c, hgvs_p. - - Creates AnnotationStatus records for each processed variant. - - Returns: - JobExecutionOutcome indicating success, failure, or skip. - """ - job = job_manager.get_job() - - _job_required_params = ["score_set_id", "correlation_id"] - validate_job_params(_job_required_params, job) - - # Fetch required resources based on param inputs. Safely ignore mypy warnings here, as they were checked above. - score_set = job_manager.db.scalars(select(ScoreSet).where(ScoreSet.id == job.job_params["score_set_id"])).one() # type: ignore - correlation_id = job.job_params["correlation_id"] # type: ignore - - # Setup initial context and progress - job_manager.save_to_context( - { - "application": "mavedb-worker", - "function": "populate_hgvs_for_score_set", - "resource": score_set.urn, - "correlation_id": correlation_id, - } - ) - job_manager.update_progress(0, 100, "Starting mapped HGVS population.") - logger.info(msg="Started mapped HGVS population", extra=job_manager.logging_context()) - - # Determine target info; multi-target score sets are not yet supported - try: - target_is_coding, transcript_accession = get_target_coding_info(score_set) - except NotImplementedError: - logger.warning( - msg="Multi-target score sets not supported for HGVS population. Skipping.", - extra=job_manager.logging_context(), - ) - job_manager.db.flush() - return JobExecutionOutcome.skipped(data={"reason": "Multi-target score sets not supported"}) - - job_manager.save_to_context({"target_is_coding": target_is_coding, "transcript_accession": transcript_accession}) - logger.info( - msg=f"Target info resolved: coding={target_is_coding}, transcript={transcript_accession}", - extra=job_manager.logging_context(), - ) - - # Fetch current mapped variants for the score set - variant_rows = job_manager.db.execute( - select(Variant.id, MappedVariant) - .join(Variant) - .join(ScoreSet) - .where(ScoreSet.id == score_set.id) - .where(MappedVariant.current.is_(True)) - ).all() - - total_variants = len(variant_rows) - job_manager.save_to_context({"total_variants": total_variants}) - - if not variant_rows: - logger.warning( - msg="No current mapped variants found for this score set. Skipping HGVS population.", - extra=job_manager.logging_context(), - ) - job_manager.db.flush() - return JobExecutionOutcome.succeeded(data={"populated_count": 0, "skipped_count": 0, "failed_count": 0}) - - job_manager.update_progress(5, 100, f"Processing {total_variants} mapped variants for HGVS population.") - - annotation_manager = AnnotationStatusManager(job_manager.db, job_run_id=job_manager.job_id) - populated_count = 0 - skipped_count = 0 - failed_count = 0 - - for index, (variant_id, mapped_variant) in enumerate(variant_rows): - # Periodic progress updates - if total_variants > 0 and index % max(total_variants // 20, 1) == 0: - progress = 5 + int((index / total_variants) * 90) - job_manager.update_progress(progress, 100, f"Processing HGVS for variant {index + 1}/{total_variants}.") - logger.info( - "Processing variant %s/%s: variant_id=%s", - index + 1, - total_variants, - variant_id, - extra=job_manager.logging_context(), - ) - - hgvs_g: Optional[str] = None - hgvs_c: Optional[str] = None - hgvs_p: Optional[str] = None - - clingen_id = mapped_variant.clingen_allele_id - - job_manager.save_to_context( - { - "mapped_variant_id": mapped_variant.id, - "clingen_allele_id": clingen_id, - "progress_index": index, - } - ) - - if not clingen_id: - annotation_manager.add_annotation( - variant_id=variant_id, - annotation_type=AnnotationType.MAPPED_HGVS, - version=None, - status=AnnotationStatus.SKIPPED, - failure_category=AnnotationFailureCategory.MISSING_IDENTIFIER, - annotation_data={ - "error_message": "No ClinGen allele ID available for ClinGen HGVS lookup.", - }, - current=True, - ) - logger.debug( - "Skipping variant %s: no ClinGen allele ID.", - variant_id, - extra=job_manager.logging_context(), - ) - skipped_count += 1 - continue - - # Skip multi-variant allele IDs (comma-separated) - if "," in clingen_id: - annotation_manager.add_annotation( - variant_id=variant_id, - annotation_type=AnnotationType.MAPPED_HGVS, - version=None, - status=AnnotationStatus.SKIPPED, - failure_category=AnnotationFailureCategory.UNSUPPORTED_IDENTIFIER, - annotation_data={ - "error_message": "Multi-variant ClinGen allele IDs not supported for HGVS lookup.", - }, - current=True, - ) - logger.debug( - "Skipping variant %s: multi-variant ClinGen allele ID.", - variant_id, - extra=job_manager.logging_context(), - ) - skipped_count += 1 - continue - - # Query ClinGen API for allele data - try: - allele_data = await get_clingen_allele_data(clingen_id) - except requests.exceptions.RequestException as exc: - annotation_manager.add_annotation( - variant_id=variant_id, - annotation_type=AnnotationType.MAPPED_HGVS, - version=None, - status=AnnotationStatus.FAILED, - failure_category=AnnotationFailureCategory.EXTERNAL_API_ERROR, - annotation_data={ - "error_message": f"Failed to fetch ClinGen allele data: {str(exc)}", - }, - current=True, - ) - logger.error( - "ClinGen API request failed for allele %s.", - clingen_id, - extra=job_manager.logging_context(), - exc_info=exc, - ) - failed_count += 1 - continue - - if allele_data is None: - annotation_manager.add_annotation( - variant_id=variant_id, - annotation_type=AnnotationType.MAPPED_HGVS, - version=None, - status=AnnotationStatus.SKIPPED, - failure_category=AnnotationFailureCategory.EXTERNAL_REFERENCE_NOT_FOUND, - annotation_data={ - "error_message": f"ClinGen allele {clingen_id} not found in the registry.", - }, - current=True, - ) - logger.debug( - "ClinGen allele %s not found in registry. Skipping variant %s.", - clingen_id, - variant_id, - extra=job_manager.logging_context(), - ) - skipped_count += 1 - continue - - # Extract HGVS based on allele type - if clingen_id.startswith("CA"): - hgvs_g, hgvs_c, hgvs_p = extract_hgvs_from_ca_allele_data( - allele_data, target_is_coding, transcript_accession - ) - elif clingen_id.startswith("PA"): - hgvs_g, hgvs_c, hgvs_p = extract_hgvs_from_pa_allele_data(allele_data) - - # Update mapped variant - mapped_variant.hgvs_g = hgvs_g - mapped_variant.hgvs_c = hgvs_c - mapped_variant.hgvs_p = hgvs_p - job_manager.db.add(mapped_variant) - - annotation_manager.add_annotation( - variant_id=variant_id, - annotation_type=AnnotationType.MAPPED_HGVS, - version=None, - status=AnnotationStatus.SUCCESS, - annotation_data={ - "annotation_metadata": { - "hgvs_g": hgvs_g, - "hgvs_c": hgvs_c, - "hgvs_p": hgvs_p, - }, - }, - current=True, - ) - populated_count += 1 - - annotation_manager.flush() - job_manager.db.flush() - - job_manager.save_to_context( - { - "populated_count": populated_count, - "skipped_count": skipped_count, - "failed_count": failed_count, - } - ) - logger.info( - msg=f"Completed mapped HGVS population: {populated_count} populated, {skipped_count} skipped, {failed_count} failed.", - extra=job_manager.logging_context(), - ) - - if failed_count > 0 and populated_count == 0: - log_and_send_slack_message( - f"All {failed_count} variants failed HGVS population for score set {score_set.urn}. Possible ClinGen API outage.", - job_manager.logging_context(), - logging.ERROR, - ) - - job_manager.db.flush() - return JobExecutionOutcome.succeeded( - data={ - "populated_count": populated_count, - "skipped_count": skipped_count, - "failed_count": failed_count, - } - ) diff --git a/src/mavedb/worker/jobs/external_services/variant_translation.py b/src/mavedb/worker/jobs/external_services/variant_translation.py deleted file mode 100644 index ddec4b731..000000000 --- a/src/mavedb/worker/jobs/external_services/variant_translation.py +++ /dev/null @@ -1,371 +0,0 @@ -"""ClinGen allele variant translation jobs for mapping PA<->CA allele relationships. - -This module populates the variant_translations table with relationships between -protein allele (PA) and nucleotide allele (CA) ClinGen IDs. For CA alleles, it -looks up MANE canonical PA IDs and their matching registered transcript CA IDs. -For PA alleles, it looks up matching registered transcript CA IDs directly. -""" - -import logging - -import requests -from sqlalchemy import select - -from mavedb.lib.annotation_status_manager import AnnotationStatusManager -from mavedb.lib.clingen.allele_registry import ( - expand_allele_ids, - get_canonical_pa_ids, - get_matching_registered_ca_ids, -) -from mavedb.lib.types.workflow import JobExecutionOutcome -from mavedb.lib.variant_translations import upsert_variant_translations -from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationFailureCategory, AnnotationStatus, FailureCategory -from mavedb.models.mapped_variant import MappedVariant -from mavedb.models.score_set import ScoreSet -from mavedb.models.variant import Variant -from mavedb.worker.jobs.utils.setup import validate_job_params -from mavedb.worker.lib.decorators.pipeline_management import with_pipeline_management -from mavedb.worker.lib.managers.job_manager import JobManager - -logger = logging.getLogger(__name__) - - -@with_pipeline_management -async def populate_variant_translations_for_score_set( - ctx: dict, job_id: int, job_manager: JobManager -) -> JobExecutionOutcome: - """Populate variant translations (PA<->CA relationships) for a score set. - - Queries the ClinGen Allele Registry to discover relationships between protein - allele (PA) and nucleotide allele (CA) ClinGen IDs, then stores them in the - variant_translations table. Each unique allele ID is processed once even if - shared across multiple mapped variants. - - Required job_params in the JobRun: - - score_set_id (int): ID of the ScoreSet to process - - correlation_id (str): Correlation ID for tracking - """ - job = job_manager.get_job() - - _job_required_params = ["score_set_id", "correlation_id"] - validate_job_params(_job_required_params, job) - - score_set = job_manager.db.scalars(select(ScoreSet).where(ScoreSet.id == job.job_params["score_set_id"])).one() # type: ignore - correlation_id = job.job_params["correlation_id"] # type: ignore - - job_manager.save_to_context( - { - "application": "mavedb-worker", - "function": "populate_variant_translations_for_score_set", - "resource": score_set.urn, - "correlation_id": correlation_id, - } - ) - job_manager.update_progress(0, 100, "Starting variant translation population.") - logger.info(msg="Started variant translation population.", extra=job_manager.logging_context()) - - # Fetch all current mapped variants with their ClinGen allele IDs - variant_rows = job_manager.db.execute( - select(Variant.id, MappedVariant.clingen_allele_id) - .join(MappedVariant, MappedVariant.variant_id == Variant.id) - .join(ScoreSet, Variant.score_set_id == ScoreSet.id) - .where(ScoreSet.id == score_set.id) - .where(MappedVariant.current.is_(True)) - ).all() - - if not variant_rows: - logger.warning( - msg="No current mapped variants found for this score set.", - extra=job_manager.logging_context(), - ) - job_manager.db.flush() - return JobExecutionOutcome.succeeded( - data={"translations_created": 0, "alleles_skipped": 0, "alleles_failed": 0} - ) - - # Deduplicate: multiple mapped variants can share the same allele ID, but we only - # need to query the ClinGen API once per unique ID. Track which variants map to each - # allele so we can record annotations for all of them after a single lookup. - allele_to_variants: dict[str, list[int]] = {} - for variant_id, clingen_allele_id in variant_rows: - if not clingen_allele_id: - continue - - for individual_id in expand_allele_ids([clingen_allele_id]): - allele_to_variants.setdefault(individual_id, []).append(variant_id) - - unique_allele_ids = list(allele_to_variants.keys()) - total_alleles = len(unique_allele_ids) - job_manager.save_to_context({"total_variants": len(variant_rows), "unique_allele_ids": total_alleles}) - - if not unique_allele_ids: - logger.warning( - msg="No ClinGen allele IDs found on mapped variants.", - extra=job_manager.logging_context(), - ) - job_manager.db.flush() - return JobExecutionOutcome.succeeded( - data={"translations_created": 0, "alleles_skipped": 0, "alleles_failed": 0} - ) - - job_manager.update_progress(5, 100, f"Processing {total_alleles} unique allele IDs for variant translations.") - logger.info( - "Processing %s unique allele IDs for variant translations.", - total_alleles, - extra=job_manager.logging_context(), - ) - - total_created = 0 - total_skipped = 0 - total_failed = 0 - annotation_manager = AnnotationStatusManager(job_manager.db, job_run_id=job_manager.job_id) - - for index, allele_id in enumerate(unique_allele_ids): - if total_alleles > 0 and index % max(total_alleles // 20, 1) == 0: - progress = 5 + int((index / total_alleles) * 90) - job_manager.update_progress(progress, 100, f"Processing allele {index + 1}/{total_alleles}.") - logger.info( - "Processing allele %s/%s: %s", - index + 1, - total_alleles, - allele_id, - extra=job_manager.logging_context(), - ) - - job_manager.save_to_context( - { - "current_allele_id": allele_id, - "progress_index": index, - } - ) - - variant_ids = allele_to_variants[allele_id] - - if allele_id.startswith("CA"): - # CA (nucleotide) alleles: look up the MANE canonical protein alleles (PAs) for - # this CA, then for each PA discover all registered transcript-level CAs. This - # CA -> PA -> CA expansion builds the full translation graph so we can link - # nucleotide variants to their protein equivalents and vice versa. - try: - canonical_pa_ids = await get_canonical_pa_ids(allele_id) - except requests.exceptions.RequestException as exc: - logger.error( - "ClinGen API request failed for canonical PA lookup of %s.", - allele_id, - extra=job_manager.logging_context(), - exc_info=exc, - ) - for vid in variant_ids: - annotation_manager.add_annotation( - variant_id=vid, - annotation_type=AnnotationType.VARIANT_TRANSLATION, - version=None, - status=AnnotationStatus.FAILED, - failure_category=AnnotationFailureCategory.EXTERNAL_API_ERROR, - annotation_data={ - "error_message": f"ClinGen API error looking up PA IDs for {allele_id}: {exc}", - }, - current=True, - ) - total_failed += len(variant_ids) - continue - - if not canonical_pa_ids: - # Noncoding variants won't have protein alleles — this is expected and not an error. - logger.debug( - "No canonical PA IDs found for %s (may be noncoding).", - allele_id, - extra=job_manager.logging_context(), - ) - for vid in variant_ids: - annotation_manager.add_annotation( - variant_id=vid, - annotation_type=AnnotationType.VARIANT_TRANSLATION, - version=None, - status=AnnotationStatus.SKIPPED, - failure_category=AnnotationFailureCategory.NO_LINKED_ALLELE, - annotation_data={ - "error_message": f"No canonical PA IDs for {allele_id}.", - }, - current=True, - ) - total_skipped += len(variant_ids) - continue - - created = 0 - failed = 0 - translation_pairs: set[tuple[str, str]] = set() - for pa_id in canonical_pa_ids: - # Record the direct PA <-> original CA relationship. - translation_pairs.add((pa_id, allele_id)) - - # Then expand: find all other CAs registered under this PA so we capture - # alternate transcript-level representations of the same protein change. - try: - ca_ids = await get_matching_registered_ca_ids(pa_id) - except requests.exceptions.RequestException as exc: - logger.error( - "ClinGen API request failed for registered CA lookup of %s.", - pa_id, - extra=job_manager.logging_context(), - exc_info=exc, - ) - failed += 1 - continue - - for ca_id in ca_ids: - translation_pairs.add((pa_id, ca_id)) - - created, existing = upsert_variant_translations(job_manager.db, list(translation_pairs)) - for vid in variant_ids: - annotation_manager.add_annotation( - variant_id=vid, - annotation_type=AnnotationType.VARIANT_TRANSLATION, - version=None, - status=AnnotationStatus.FAILED if failed > 0 else AnnotationStatus.SUCCESS, - annotation_data={ - "annotation_metadata": { - "allele_id": allele_id, - "translation_pairs": [[pa, ca] for pa, ca in translation_pairs], - "translations_new": created, - "translations_existing": existing, - "pa_lookups_failed": failed, - "pa_lookups_total": len(canonical_pa_ids), - }, - }, - current=True, - ) - - total_created += created - total_failed += failed - - elif allele_id.startswith("PA"): - # PA (protein) alleles: directly look up all registered transcript-level CAs. - # This is simpler than the CA path since we already have the protein allele. - try: - ca_ids = await get_matching_registered_ca_ids(allele_id) - except requests.exceptions.RequestException as exc: - logger.error( - "ClinGen API request failed for registered CA lookup of %s.", - allele_id, - extra=job_manager.logging_context(), - exc_info=exc, - ) - for vid in variant_ids: - annotation_manager.add_annotation( - variant_id=vid, - annotation_type=AnnotationType.VARIANT_TRANSLATION, - version=None, - status=AnnotationStatus.FAILED, - failure_category=AnnotationFailureCategory.EXTERNAL_API_ERROR, - annotation_data={ - "error_message": f"ClinGen API error for {allele_id}: {exc}", - }, - current=True, - ) - total_failed += len(variant_ids) - continue - - if not ca_ids: - logger.warning( - "No matching registered transcript CA IDs for PA allele %s. This is unexpected.", - allele_id, - extra=job_manager.logging_context(), - ) - for vid in variant_ids: - annotation_manager.add_annotation( - variant_id=vid, - annotation_type=AnnotationType.VARIANT_TRANSLATION, - version=None, - status=AnnotationStatus.SKIPPED, - failure_category=AnnotationFailureCategory.NO_LINKED_ALLELE, - annotation_data={ - "error_message": f"No registered transcript CA IDs for {allele_id}.", - }, - current=True, - ) - total_skipped += len(variant_ids) - continue - - translation_pairs = set([(allele_id, ca_id) for ca_id in ca_ids]) - created, existing = upsert_variant_translations(job_manager.db, list(translation_pairs)) - for vid in variant_ids: - annotation_manager.add_annotation( - variant_id=vid, - annotation_type=AnnotationType.VARIANT_TRANSLATION, - version=None, - status=AnnotationStatus.SUCCESS, - annotation_data={ - "annotation_metadata": { - "allele_id": allele_id, - "translation_pairs": [[pa, ca] for pa, ca in translation_pairs], - "translations_new": created, - "translations_existing": existing, - }, - }, - current=True, - ) - - total_created += created - - else: - logger.warning( - "Unrecognized ClinGen allele ID format: %s. Skipping.", - allele_id, - extra=job_manager.logging_context(), - ) - for vid in variant_ids: - annotation_manager.add_annotation( - variant_id=vid, - annotation_type=AnnotationType.VARIANT_TRANSLATION, - version=None, - status=AnnotationStatus.SKIPPED, - failure_category=AnnotationFailureCategory.UNSUPPORTED_IDENTIFIER, - annotation_data={ - "error_message": f"Unrecognized allele ID format: {allele_id}", - }, - current=True, - ) - total_skipped += len(variant_ids) - - annotation_manager.flush() - job_manager.db.flush() - - job_manager.save_to_context( - { - "translations_created": total_created, - "alleles_skipped": total_skipped, - "alleles_failed": total_failed, - } - ) - logger.info( - "Completed variant translation population: %s created, %s skipped, %s failed.", - total_created, - total_skipped, - total_failed, - extra=job_manager.logging_context(), - ) - - if total_failed > 0 and total_created == 0: - error_message = f"All {total_failed} variant translation lookups failed for score set {score_set.urn}. Possible ClinGen API outage." - logger.error(error_message, extra=job_manager.logging_context()) - job_manager.db.flush() - return JobExecutionOutcome.failed( - reason=error_message, - data={ - "translations_created": 0, - "alleles_skipped": total_skipped, - "alleles_failed": total_failed, - }, - failure_category=FailureCategory.DEPENDENCY_FAILURE, - ) - - job_manager.db.flush() - return JobExecutionOutcome.succeeded( - data={ - "translations_created": total_created, - "alleles_skipped": total_skipped, - "alleles_failed": total_failed, - } - ) diff --git a/src/mavedb/worker/jobs/registry.py b/src/mavedb/worker/jobs/registry.py index c445098ef..8b2b712f8 100644 --- a/src/mavedb/worker/jobs/registry.py +++ b/src/mavedb/worker/jobs/registry.py @@ -18,8 +18,6 @@ from mavedb.worker.jobs.external_services import ( link_gnomad_variants, poll_uniprot_mapping_jobs_for_score_set, - populate_hgvs_for_score_set, - populate_variant_translations_for_score_set, populate_vep_for_score_set, refresh_clinvar_controls, submit_score_set_mappings_to_car, @@ -49,8 +47,6 @@ submit_uniprot_mapping_jobs_for_score_set, poll_uniprot_mapping_jobs_for_score_set, link_gnomad_variants, - populate_hgvs_for_score_set, - populate_variant_translations_for_score_set, populate_vep_for_score_set, # Data management jobs refresh_materialized_views, @@ -160,20 +156,6 @@ "key": "link_gnomad_variants", "type": JobType.MAPPED_VARIANT_ANNOTATION, }, - populate_hgvs_for_score_set: { - "dependencies": [], - "params": {"score_set_id": None, "correlation_id": None}, - "function": "populate_hgvs_for_score_set", - "key": "populate_hgvs_for_score_set", - "type": JobType.MAPPED_VARIANT_ANNOTATION, - }, - populate_variant_translations_for_score_set: { - "dependencies": [], - "params": {"score_set_id": None, "correlation_id": None}, - "function": "populate_variant_translations_for_score_set", - "key": "populate_variant_translations_for_score_set", - "type": JobType.MAPPED_VARIANT_ANNOTATION, - }, populate_vep_for_score_set: { "dependencies": [], "params": {"score_set_id": None, "correlation_id": None}, diff --git a/tests/worker/jobs/conftest.py b/tests/worker/jobs/conftest.py index 6b7309fd2..873d09ff0 100644 --- a/tests/worker/jobs/conftest.py +++ b/tests/worker/jobs/conftest.py @@ -1011,198 +1011,6 @@ def with_cleanup_job(session, sample_cleanup_job_run): session.commit() -## HGVS Population Job Fixtures ## - - -@pytest.fixture -def populate_hgvs_sample_params(with_populated_domain_data, sample_score_set): - """Provide sample parameters for populate_hgvs_for_score_set job.""" - - return { - "correlation_id": "sample-correlation-id", - "score_set_id": sample_score_set.id, - } - - -@pytest.fixture -def sample_populate_hgvs_pipeline(): - """Create a pipeline instance for populate_hgvs_for_score_set job.""" - - return Pipeline( - urn="test:populate_hgvs_pipeline", - name="Populate HGVS Pipeline", - ) - - -@pytest.fixture -def sample_populate_hgvs_run(populate_hgvs_sample_params): - """Create a JobRun instance for populate_hgvs_for_score_set job.""" - - return JobRun( - urn="test:populate_hgvs_for_score_set", - job_type="populate_hgvs_for_score_set", - job_function="populate_hgvs_for_score_set", - max_retries=3, - retry_count=0, - job_params=populate_hgvs_sample_params, - ) - - -@pytest.fixture -def with_populate_hgvs_job(session, sample_populate_hgvs_run): - """Add a populate_hgvs_for_score_set job run to the session.""" - - session.add(sample_populate_hgvs_run) - session.commit() - - -@pytest.fixture -def with_populate_hgvs_pipeline(session, sample_populate_hgvs_pipeline): - """Add a populate_hgvs pipeline to the session.""" - - session.add(sample_populate_hgvs_pipeline) - session.commit() - - -@pytest.fixture -def sample_populate_hgvs_run_pipeline( - session, - with_populate_hgvs_job, - with_populate_hgvs_pipeline, - sample_populate_hgvs_run, - sample_populate_hgvs_pipeline, -): - """Provide a context with a populate_hgvs job run and pipeline.""" - - sample_populate_hgvs_run.pipeline_id = sample_populate_hgvs_pipeline.id - session.commit() - return sample_populate_hgvs_run - - -@pytest.fixture -def setup_sample_variants_with_caid_for_hgvs( - session, with_populated_domain_data, mock_worker_ctx, sample_populate_hgvs_run -): - """Setup variants and mapped variants in the database for HGVS population testing.""" - score_set = session.get(ScoreSet, sample_populate_hgvs_run.job_params["score_set_id"]) - - variant = Variant( - urn="urn:variant:test-variant-with-caid-hgvs", - score_set_id=score_set.id, - hgvs_nt="NM_000000.1:c.1A>G", - hgvs_pro="NP_000000.1:p.Met1Val", - data={"hgvs_c": "NM_000000.1:c.1A>G", "hgvs_p": "NP_000000.1:p.Met1Val"}, - ) - session.add(variant) - session.commit() - mapped_variant = MappedVariant( - variant_id=variant.id, - clingen_allele_id=VALID_CAID, - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant) - session.commit() - return variant, mapped_variant - - -# --- Variant Translation Fixtures --- - - -@pytest.fixture -def populate_variant_translations_sample_params(with_populated_domain_data, sample_score_set): - """Provide sample parameters for populate_variant_translations_for_score_set job.""" - - return { - "correlation_id": "sample-correlation-id", - "score_set_id": sample_score_set.id, - } - - -@pytest.fixture -def sample_populate_variant_translations_pipeline(): - """Create a pipeline instance for populate_variant_translations_for_score_set job.""" - - return Pipeline( - urn="test:populate_variant_translations_pipeline", - name="Populate Variant Translations Pipeline", - ) - - -@pytest.fixture -def sample_populate_variant_translations_run(populate_variant_translations_sample_params): - """Create a JobRun instance for populate_variant_translations_for_score_set job.""" - - return JobRun( - urn="test:populate_variant_translations_for_score_set", - job_type="populate_variant_translations_for_score_set", - job_function="populate_variant_translations_for_score_set", - max_retries=3, - retry_count=0, - job_params=populate_variant_translations_sample_params, - ) - - -@pytest.fixture -def with_populate_variant_translations_job(session, sample_populate_variant_translations_run): - """Add a populate_variant_translations_for_score_set job run to the session.""" - - session.add(sample_populate_variant_translations_run) - session.commit() - - -@pytest.fixture -def with_populate_variant_translations_pipeline(session, sample_populate_variant_translations_pipeline): - """Add a populate_variant_translations pipeline to the session.""" - - session.add(sample_populate_variant_translations_pipeline) - session.commit() - - -@pytest.fixture -def sample_populate_variant_translations_run_pipeline( - session, - with_populate_variant_translations_job, - with_populate_variant_translations_pipeline, - sample_populate_variant_translations_run, - sample_populate_variant_translations_pipeline, -): - """Provide a context with a populate_variant_translations job run and pipeline.""" - - sample_populate_variant_translations_run.pipeline_id = sample_populate_variant_translations_pipeline.id - session.commit() - return sample_populate_variant_translations_run - - -@pytest.fixture -def setup_sample_variants_with_caid_for_translation( - session, with_populated_domain_data, mock_worker_ctx, sample_populate_variant_translations_run -): - """Setup variants and mapped variants in the database for variant translation testing.""" - score_set = session.get(ScoreSet, sample_populate_variant_translations_run.job_params["score_set_id"]) - - variant = Variant( - urn="urn:variant:test-variant-with-caid-translation", - score_set_id=score_set.id, - hgvs_nt="NM_000000.1:c.1A>G", - hgvs_pro="NP_000000.1:p.Met1Val", - data={"hgvs_c": "NM_000000.1:c.1A>G", "hgvs_p": "NP_000000.1:p.Met1Val"}, - ) - session.add(variant) - session.commit() - mapped_variant = MappedVariant( - variant_id=variant.id, - clingen_allele_id=VALID_CAID, - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant) - session.commit() - return variant, mapped_variant - - ## ClinGen Cache Warming Job Fixtures ## diff --git a/tests/worker/jobs/external_services/network/test_hgvs.py b/tests/worker/jobs/external_services/network/test_hgvs.py deleted file mode 100644 index 56f100e0b..000000000 --- a/tests/worker/jobs/external_services/network/test_hgvs.py +++ /dev/null @@ -1,54 +0,0 @@ -# ruff: noqa: E402 - -import pytest - -pytest.importorskip("arq") - -from sqlalchemy import select - -from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationStatus, JobStatus, PipelineStatus -from mavedb.models.variant_annotation_status import VariantAnnotationStatus - -pytestmark = pytest.mark.usefixtures("patch_db_session_ctxmgr") - - -@pytest.mark.asyncio -@pytest.mark.integration -@pytest.mark.network -@pytest.mark.slow -class TestE2EPopulateHgvsForScoreSet: - """End-to-end test for HGVS population against the real ClinGen API.""" - - async def test_populate_hgvs_e2e( - self, - session, - arq_redis, - arq_worker, - sample_populate_hgvs_run_pipeline, - sample_populate_hgvs_pipeline, - setup_sample_variants_with_caid_for_hgvs, - ): - """Enqueue the HGVS population job, run the worker, and verify HGVS fields are populated.""" - _, mapped_variant = setup_sample_variants_with_caid_for_hgvs - - await arq_redis.enqueue_job("populate_hgvs_for_score_set", sample_populate_hgvs_run_pipeline.id) - await arq_worker.async_run() - await arq_worker.run_check() - - session.refresh(sample_populate_hgvs_run_pipeline) - assert sample_populate_hgvs_run_pipeline.status == JobStatus.SUCCEEDED - - session.refresh(sample_populate_hgvs_pipeline) - assert sample_populate_hgvs_pipeline.status == PipelineStatus.SUCCEEDED - - session.refresh(mapped_variant) - annotation = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.variant_id == mapped_variant.variant_id, - VariantAnnotationStatus.annotation_type == AnnotationType.MAPPED_HGVS, - VariantAnnotationStatus.current.is_(True), - ) - ).one_or_none() - assert annotation is not None - assert annotation.status in (AnnotationStatus.SUCCESS, AnnotationStatus.SKIPPED) diff --git a/tests/worker/jobs/external_services/network/test_variant_translations.py b/tests/worker/jobs/external_services/network/test_variant_translations.py deleted file mode 100644 index b45087dd9..000000000 --- a/tests/worker/jobs/external_services/network/test_variant_translations.py +++ /dev/null @@ -1,56 +0,0 @@ -# ruff: noqa: E402 - -import pytest - -pytest.importorskip("arq") - -from sqlalchemy import select - -from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationStatus, JobStatus, PipelineStatus -from mavedb.models.variant_annotation_status import VariantAnnotationStatus - -pytestmark = pytest.mark.usefixtures("patch_db_session_ctxmgr") - - -@pytest.mark.asyncio -@pytest.mark.integration -@pytest.mark.network -@pytest.mark.slow -class TestE2EPopulateVariantTranslationsForScoreSet: - """End-to-end test for variant translation population against the real ClinGen API.""" - - async def test_populate_variant_translations_e2e( - self, - session, - arq_redis, - arq_worker, - sample_populate_variant_translations_run_pipeline, - sample_populate_variant_translations_pipeline, - setup_sample_variants_with_caid_for_translation, - ): - """Enqueue the variant translation job, run the worker, and verify translations are created.""" - _, mapped_variant = setup_sample_variants_with_caid_for_translation - - await arq_redis.enqueue_job( - "populate_variant_translations_for_score_set", - sample_populate_variant_translations_run_pipeline.id, - ) - await arq_worker.async_run() - await arq_worker.run_check() - - session.refresh(sample_populate_variant_translations_run_pipeline) - assert sample_populate_variant_translations_run_pipeline.status == JobStatus.SUCCEEDED - - session.refresh(sample_populate_variant_translations_pipeline) - assert sample_populate_variant_translations_pipeline.status == PipelineStatus.SUCCEEDED - - annotation = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.variant_id == mapped_variant.variant_id, - VariantAnnotationStatus.annotation_type == AnnotationType.VARIANT_TRANSLATION, - VariantAnnotationStatus.current.is_(True), - ) - ).one_or_none() - assert annotation is not None - assert annotation.status in (AnnotationStatus.SUCCESS, AnnotationStatus.SKIPPED) diff --git a/tests/worker/jobs/external_services/test_hgvs.py b/tests/worker/jobs/external_services/test_hgvs.py deleted file mode 100644 index 946724cc5..000000000 --- a/tests/worker/jobs/external_services/test_hgvs.py +++ /dev/null @@ -1,544 +0,0 @@ -# ruff: noqa: E402 - -import pytest - -pytest.importorskip("arq") - -from unittest.mock import patch - -from mavedb.lib.types.workflow import JobExecutionOutcome -from mavedb.models.enums.job_pipeline import JobStatus, PipelineStatus -from mavedb.models.mapped_variant import MappedVariant -from mavedb.models.variant_annotation_status import VariantAnnotationStatus -from mavedb.worker.jobs.external_services.hgvs import populate_hgvs_for_score_set -from mavedb.worker.lib.managers.job_manager import JobManager - -pytestmark = pytest.mark.usefixtures("patch_db_session_ctxmgr") - -SAMPLE_CA_ALLELE_DATA = { - "genomicAlleles": [ - { - "referenceGenome": "GRCh38", - "hgvs": ["NC_000001.11:g.12345A>G"], - } - ], - "transcriptAlleles": [ - { - "hgvs": ["NM_000000.1:c.1A>G"], - "proteinEffect": {"hgvs": "NP_000000.1:p.Met1Val"}, - "MANE": { - "nucleotide": {"RefSeq": {"hgvs": "NM_000000.1:c.1A>G"}}, - "protein": {"RefSeq": {"hgvs": "NP_000000.1:p.Met1Val"}}, - }, - } - ], -} - -SAMPLE_PA_ALLELE_DATA = { - "aminoAcidAlleles": [ - { - "hgvs": ["NP_000000.1:p.Met1Val"], - } - ], -} - - -@pytest.mark.asyncio -@pytest.mark.unit -class TestPopulateHgvsForScoreSetUnit: - """Unit tests for the populate_hgvs_for_score_set job.""" - - async def test_no_mapped_variants( - self, - session, - with_populated_domain_data, - with_populate_hgvs_job, - mock_worker_ctx, - sample_populate_hgvs_run, - ): - """Test populating HGVS when no mapped variants exist.""" - result = await populate_hgvs_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_hgvs_run.id), - ) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - async def test_variant_without_caid_skipped( - self, - session, - with_populated_domain_data, - with_populate_hgvs_job, - mock_worker_ctx, - sample_populate_hgvs_run, - setup_sample_variants_with_caid_for_hgvs, - ): - """Test that a variant without a CAID gets a skipped annotation.""" - _, mapped_variant = setup_sample_variants_with_caid_for_hgvs - mapped_variant.clingen_allele_id = None - session.commit() - - result = await populate_hgvs_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_hgvs_run.id), - ) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - assert result.data["skipped_count"] == 1 - - async def test_variant_with_multi_caid_skipped( - self, - session, - with_populated_domain_data, - with_populate_hgvs_job, - mock_worker_ctx, - sample_populate_hgvs_run, - setup_sample_variants_with_caid_for_hgvs, - ): - """Test that a variant with a multi-variant CAID gets a skipped annotation.""" - _, mapped_variant = setup_sample_variants_with_caid_for_hgvs - mapped_variant.clingen_allele_id = "CA123,CA456" - session.commit() - - result = await populate_hgvs_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_hgvs_run.id), - ) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - assert result.data["skipped_count"] == 1 - - async def test_successful_ca_allele_hgvs_population( - self, - session, - with_populated_domain_data, - with_populate_hgvs_job, - mock_worker_ctx, - sample_populate_hgvs_run, - setup_sample_variants_with_caid_for_hgvs, - ): - """Test successful HGVS population for a CA allele.""" - with ( - patch( - "mavedb.worker.jobs.external_services.hgvs.get_clingen_allele_data", - return_value=SAMPLE_CA_ALLELE_DATA, - ), - ): - result = await populate_hgvs_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_hgvs_run.id), - ) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - assert result.data["populated_count"] == 1 - - _, mapped_variant = setup_sample_variants_with_caid_for_hgvs - session.refresh(mapped_variant) - assert mapped_variant.hgvs_g == "NC_000001.11:g.12345A>G" - - async def test_clingen_api_error_recorded_as_failed( - self, - session, - with_populated_domain_data, - with_populate_hgvs_job, - mock_worker_ctx, - sample_populate_hgvs_run, - setup_sample_variants_with_caid_for_hgvs, - ): - """Test that ClinGen API errors are recorded as failed annotations.""" - import requests - - with ( - patch( - "mavedb.worker.jobs.external_services.hgvs.get_clingen_allele_data", - side_effect=requests.exceptions.ConnectionError("Connection refused"), - ), - ): - result = await populate_hgvs_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_hgvs_run.id), - ) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - assert result.data["failed_count"] == 1 - - async def test_total_api_failure_sends_slack_alert( - self, - session, - with_populated_domain_data, - with_populate_hgvs_job, - mock_worker_ctx, - sample_populate_hgvs_run, - setup_sample_variants_with_caid_for_hgvs, - ): - """Test that a Slack alert is sent when all variants fail HGVS population.""" - import requests - - with ( - patch( - "mavedb.worker.jobs.external_services.hgvs.get_clingen_allele_data", - side_effect=requests.exceptions.ConnectionError("Connection refused"), - ), - patch("mavedb.worker.jobs.external_services.hgvs.log_and_send_slack_message") as mock_slack, - ): - result = await populate_hgvs_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_hgvs_run.id), - ) - - assert result.status == JobStatus.SUCCEEDED - assert result.data["failed_count"] == 1 - assert result.data["populated_count"] == 0 - mock_slack.assert_called_once() - - async def test_clingen_allele_not_found_skipped( - self, - session, - with_populated_domain_data, - with_populate_hgvs_job, - mock_worker_ctx, - sample_populate_hgvs_run, - setup_sample_variants_with_caid_for_hgvs, - ): - """Test that a 404 from ClinGen results in a skipped annotation.""" - with ( - patch( - "mavedb.worker.jobs.external_services.hgvs.get_clingen_allele_data", - return_value=None, - ), - ): - result = await populate_hgvs_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_hgvs_run.id), - ) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - assert result.data["skipped_count"] == 1 - - async def test_propagates_exceptions( - self, - session, - with_populated_domain_data, - with_populate_hgvs_job, - mock_worker_ctx, - sample_populate_hgvs_run, - setup_sample_variants_with_caid_for_hgvs, - ): - """Test that unexpected exceptions are propagated.""" - with patch( - "mavedb.worker.jobs.external_services.hgvs.get_clingen_allele_data", - side_effect=Exception("Test exception"), - ): - with pytest.raises(Exception) as exc_info: - await populate_hgvs_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_hgvs_run.id), - ) - - assert str(exc_info.value) == "Test exception" - - -@pytest.mark.asyncio -@pytest.mark.integration -class TestPopulateHgvsForScoreSetIntegration: - """Integration tests for the populate_hgvs_for_score_set job.""" - - async def test_no_mapped_variants( - self, - session, - with_populated_domain_data, - with_populate_hgvs_job, - mock_worker_ctx, - sample_populate_hgvs_run, - ): - """Test end-to-end when no mapped variants exist.""" - result = await populate_hgvs_for_score_set(mock_worker_ctx, sample_populate_hgvs_run.id) - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 0 - - session.refresh(sample_populate_hgvs_run) - assert sample_populate_hgvs_run.status == JobStatus.SUCCEEDED - - async def test_successful_hgvs_population( - self, - session, - with_populated_domain_data, - with_populate_hgvs_job, - mock_worker_ctx, - sample_populate_hgvs_run, - setup_sample_variants_with_caid_for_hgvs, - ): - """Test end-to-end successful HGVS population.""" - with patch( - "mavedb.worker.jobs.external_services.hgvs.get_clingen_allele_data", - return_value=SAMPLE_CA_ALLELE_DATA, - ): - result = await populate_hgvs_for_score_set(mock_worker_ctx, sample_populate_hgvs_run.id) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - # Verify mapped variant was updated with HGVS - mapped_variant = session.query(MappedVariant).first() - assert mapped_variant.hgvs_g == "NC_000001.11:g.12345A>G" - - # Verify annotation status was rendered - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == "success" - assert annotation_statuses[0].annotation_type == "mapped_hgvs" - - session.refresh(sample_populate_hgvs_run) - assert sample_populate_hgvs_run.status == JobStatus.SUCCEEDED - - async def test_successful_hgvs_population_pipeline( - self, - session, - with_populated_domain_data, - mock_worker_ctx, - sample_populate_hgvs_run_pipeline, - sample_populate_hgvs_pipeline, - setup_sample_variants_with_caid_for_hgvs, - ): - """Test end-to-end HGVS population in a pipeline.""" - with patch( - "mavedb.worker.jobs.external_services.hgvs.get_clingen_allele_data", - return_value=SAMPLE_CA_ALLELE_DATA, - ): - result = await populate_hgvs_for_score_set(mock_worker_ctx, sample_populate_hgvs_run_pipeline.id) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - # Verify mapped variant was updated - mapped_variant = session.query(MappedVariant).first() - assert mapped_variant.hgvs_g == "NC_000001.11:g.12345A>G" - - # Verify annotation status - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == "success" - assert annotation_statuses[0].annotation_type == "mapped_hgvs" - - # Verify job and pipeline status - session.refresh(sample_populate_hgvs_run_pipeline) - assert sample_populate_hgvs_run_pipeline.status == JobStatus.SUCCEEDED - - session.refresh(sample_populate_hgvs_pipeline) - assert sample_populate_hgvs_pipeline.status == PipelineStatus.SUCCEEDED - - async def test_variant_without_caid_creates_skipped_annotation( - self, - session, - with_populated_domain_data, - with_populate_hgvs_job, - mock_worker_ctx, - sample_populate_hgvs_run, - setup_sample_variants_with_caid_for_hgvs, - ): - """Test that variants without CAIDs get a skipped annotation status.""" - _, mapped_variant = setup_sample_variants_with_caid_for_hgvs - mapped_variant.clingen_allele_id = None - session.commit() - - result = await populate_hgvs_for_score_set(mock_worker_ctx, sample_populate_hgvs_run.id) - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == "skipped" - assert annotation_statuses[0].annotation_type == "mapped_hgvs" - - session.refresh(sample_populate_hgvs_run) - assert sample_populate_hgvs_run.status == JobStatus.SUCCEEDED - - async def test_exceptions_handled_by_decorators( - self, - session, - with_populated_domain_data, - with_populate_hgvs_job, - mock_worker_ctx, - sample_populate_hgvs_run, - setup_sample_variants_with_caid_for_hgvs, - ): - """Test that unexpected exceptions are handled by decorators.""" - with ( - patch( - "mavedb.worker.jobs.external_services.hgvs.get_clingen_allele_data", - side_effect=Exception("Test exception"), - ), - patch("mavedb.worker.lib.decorators.job_management.send_slack_job_error") as mock_send_slack_job_error, - ): - result = await populate_hgvs_for_score_set( - mock_worker_ctx, - sample_populate_hgvs_run.id, - ) - - mock_send_slack_job_error.assert_called_once() - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.ERRORED - assert isinstance(result.exception, Exception) - - session.refresh(sample_populate_hgvs_run) - assert sample_populate_hgvs_run.status == JobStatus.ERRORED - - -@pytest.mark.asyncio -@pytest.mark.integration -class TestPopulateHgvsForScoreSetArqContext: - """Tests for populate_hgvs_for_score_set job using the ARQ context fixture.""" - - async def test_with_arq_context_independent( - self, - arq_redis, - arq_worker, - session, - with_populated_domain_data, - with_populate_hgvs_job, - sample_populate_hgvs_run, - setup_sample_variants_with_caid_for_hgvs, - ): - """Test that the job works with the ARQ context fixture.""" - with patch( - "mavedb.worker.jobs.external_services.hgvs.get_clingen_allele_data", - return_value=SAMPLE_CA_ALLELE_DATA, - ): - await arq_redis.enqueue_job("populate_hgvs_for_score_set", sample_populate_hgvs_run.id) - await arq_worker.async_run() - await arq_worker.run_check() - - # Verify mapped variant was updated - mapped_variant = session.query(MappedVariant).first() - assert mapped_variant.hgvs_g == "NC_000001.11:g.12345A>G" - - # Verify annotation status - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == "success" - assert annotation_statuses[0].annotation_type == "mapped_hgvs" - - # Verify job completed - session.refresh(sample_populate_hgvs_run) - assert sample_populate_hgvs_run.status == JobStatus.SUCCEEDED - - async def test_with_arq_context_pipeline( - self, - arq_redis, - arq_worker, - session, - with_populated_domain_data, - sample_populate_hgvs_run_pipeline, - sample_populate_hgvs_pipeline, - setup_sample_variants_with_caid_for_hgvs, - ): - """Test that the job works with the ARQ context fixture in a pipeline.""" - with patch( - "mavedb.worker.jobs.external_services.hgvs.get_clingen_allele_data", - return_value=SAMPLE_CA_ALLELE_DATA, - ): - await arq_redis.enqueue_job("populate_hgvs_for_score_set", sample_populate_hgvs_run_pipeline.id) - await arq_worker.async_run() - await arq_worker.run_check() - - # Verify mapped variant was updated - mapped_variant = session.query(MappedVariant).first() - assert mapped_variant.hgvs_g == "NC_000001.11:g.12345A>G" - - # Verify annotation status - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == "success" - assert annotation_statuses[0].annotation_type == "mapped_hgvs" - - # Verify job and pipeline status - session.refresh(sample_populate_hgvs_run_pipeline) - assert sample_populate_hgvs_run_pipeline.status == JobStatus.SUCCEEDED - - session.refresh(sample_populate_hgvs_pipeline) - assert sample_populate_hgvs_pipeline.status == PipelineStatus.SUCCEEDED - - async def test_with_arq_context_exception_handling_independent( - self, - arq_redis, - arq_worker, - session, - with_populated_domain_data, - with_populate_hgvs_job, - sample_populate_hgvs_run, - setup_sample_variants_with_caid_for_hgvs, - ): - """Test that exceptions are handled with the ARQ context fixture.""" - with ( - patch( - "mavedb.worker.jobs.external_services.hgvs.get_clingen_allele_data", - side_effect=Exception("Test exception"), - ), - patch("mavedb.worker.lib.decorators.job_management.send_slack_job_error") as mock_send_slack_job_error, - ): - await arq_redis.enqueue_job("populate_hgvs_for_score_set", sample_populate_hgvs_run.id) - await arq_worker.async_run() - await arq_worker.run_check() - - mock_send_slack_job_error.assert_called_once() - - # Verify no annotations were rendered - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 0 - - # Verify job errored - session.refresh(sample_populate_hgvs_run) - assert sample_populate_hgvs_run.status == JobStatus.ERRORED - - async def test_with_arq_context_exception_handling_pipeline( - self, - arq_redis, - arq_worker, - session, - with_populated_domain_data, - sample_populate_hgvs_pipeline, - sample_populate_hgvs_run_pipeline, - setup_sample_variants_with_caid_for_hgvs, - ): - """Test that exceptions in pipeline context are handled.""" - with ( - patch( - "mavedb.worker.jobs.external_services.hgvs.get_clingen_allele_data", - side_effect=Exception("Test exception"), - ), - patch("mavedb.worker.lib.decorators.job_management.send_slack_job_error") as mock_send_slack_job_error, - ): - await arq_redis.enqueue_job("populate_hgvs_for_score_set", sample_populate_hgvs_run_pipeline.id) - await arq_worker.async_run() - await arq_worker.run_check() - - mock_send_slack_job_error.assert_called_once() - - # Verify no annotations were rendered - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 0 - - # Verify job errored - session.refresh(sample_populate_hgvs_run_pipeline) - assert sample_populate_hgvs_run_pipeline.status == JobStatus.ERRORED - - # Verify pipeline failed - session.refresh(sample_populate_hgvs_pipeline) - assert sample_populate_hgvs_pipeline.status == PipelineStatus.FAILED diff --git a/tests/worker/jobs/external_services/test_variant_translation.py b/tests/worker/jobs/external_services/test_variant_translation.py deleted file mode 100644 index 3e1f364ea..000000000 --- a/tests/worker/jobs/external_services/test_variant_translation.py +++ /dev/null @@ -1,770 +0,0 @@ -# ruff: noqa: E402 - -import pytest - -pytest.importorskip("arq") - -from unittest.mock import patch - -from sqlalchemy import select - -from mavedb.lib.types.workflow import JobExecutionOutcome -from mavedb.models.enums.job_pipeline import FailureCategory, JobStatus, PipelineStatus -from mavedb.models.mapped_variant import MappedVariant -from mavedb.models.variant import Variant -from mavedb.models.variant_annotation_status import VariantAnnotationStatus -from mavedb.models.variant_translation import VariantTranslation -from mavedb.worker.jobs.external_services.variant_translation import populate_variant_translations_for_score_set -from mavedb.worker.lib.managers.job_manager import JobManager - -pytestmark = pytest.mark.usefixtures("patch_db_session_ctxmgr") - - -# --- Unit Tests --- - - -@pytest.mark.asyncio -@pytest.mark.unit -class TestPopulateVariantTranslationsUnit: - """Unit tests for the populate_variant_translations_for_score_set job.""" - - async def test_no_mapped_variants( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - ): - """Test that the job succeeds with zero translations when no mapped variants exist.""" - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_variant_translations_run.id), - ) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - assert result.data["translations_created"] == 0 - - async def test_variant_without_caid_no_translations( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that a variant without a CAID results in no translations.""" - _, mapped_variant = setup_sample_variants_with_caid_for_translation - mapped_variant.clingen_allele_id = None - session.commit() - - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_variant_translations_run.id), - ) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - assert result.data["translations_created"] == 0 - - async def test_ca_allele_creates_translations( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that a CA allele creates translations via PA lookup.""" - with ( - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_canonical_pa_ids", - return_value=["PA00001"], - ), - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_matching_registered_ca_ids", - return_value=["CA11111", "CA22222"], - ), - ): - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_variant_translations_run.id), - ) - - assert result.status == JobStatus.SUCCEEDED - # 1 for PA00001->CA9765210 (the original CA), 2 for PA00001->CA11111 and PA00001->CA22222 - assert result.data["translations_created"] == 3 - - translations = session.scalars(select(VariantTranslation)).all() - assert len(translations) == 3 - - annotation = session.scalars(select(VariantAnnotationStatus)).one() - assert annotation is not None - - async def test_pa_allele_creates_translations( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that a PA allele creates translations via CA lookup.""" - _, mapped_variant = setup_sample_variants_with_caid_for_translation - mapped_variant.clingen_allele_id = "PA99999" - session.commit() - - with ( - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_matching_registered_ca_ids", - return_value=["CA33333", "CA44444"], - ), - ): - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_variant_translations_run.id), - ) - - assert result.status == JobStatus.SUCCEEDED - assert result.data["translations_created"] == 2 - - translations = session.scalars(select(VariantTranslation)).all() - assert len(translations) == 2 - aa_ids = {t.aa_clingen_id for t in translations} - assert aa_ids == {"PA99999"} - - async def test_multi_variant_caid_expanded( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that comma-separated CAIDs are expanded and each processed independently.""" - _, mapped_variant = setup_sample_variants_with_caid_for_translation - mapped_variant.clingen_allele_id = "CA55555,CA66666" - session.commit() - - with ( - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_canonical_pa_ids", - return_value=["PA00002"], - ), - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_matching_registered_ca_ids", - return_value=[], - ), - ): - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_variant_translations_run.id), - ) - - assert result.status == JobStatus.SUCCEEDED - # PA00002->CA55555 and PA00002->CA66666 - assert result.data["translations_created"] == 2 - - async def test_ca_allele_no_pa_ids_skipped( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that a CA allele with no canonical PA IDs results in a skip.""" - with ( - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_canonical_pa_ids", - return_value=[], - ), - ): - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_variant_translations_run.id), - ) - - assert result.status == JobStatus.SUCCEEDED - assert result.data["alleles_skipped"] == 1 - assert result.data["translations_created"] == 0 - - annotation = session.scalars(select(VariantAnnotationStatus)).one() - assert annotation.status == "skipped" - - async def test_pa_allele_no_ca_ids_skipped( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that a PA allele with no registered CA IDs results in a skip.""" - _, mapped_variant = setup_sample_variants_with_caid_for_translation - mapped_variant.clingen_allele_id = "PA88888" - session.commit() - - with ( - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_matching_registered_ca_ids", - return_value=[], - ), - ): - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_variant_translations_run.id), - ) - - assert result.status == JobStatus.SUCCEEDED - assert result.data["alleles_skipped"] == 1 - assert result.data["translations_created"] == 0 - - async def test_ca_allele_api_failure_records_failed_annotation( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that a ClinGen API failure for CA allele records a failed annotation.""" - import requests - - with ( - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_canonical_pa_ids", - side_effect=requests.exceptions.ConnectionError("Connection failed"), - ), - ): - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_variant_translations_run.id), - ) - - assert result.status == JobStatus.FAILED - assert result.failure_category == FailureCategory.DEPENDENCY_FAILURE - assert result.data["alleles_failed"] == 1 - - annotation = session.scalars(select(VariantAnnotationStatus)).one() - assert annotation.status == "failed" - - async def test_unrecognized_allele_format_skipped( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that an unrecognized allele ID format is skipped.""" - _, mapped_variant = setup_sample_variants_with_caid_for_translation - mapped_variant.clingen_allele_id = "XX12345" - session.commit() - - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_variant_translations_run.id), - ) - - assert result.status == JobStatus.SUCCEEDED - assert result.data["alleles_skipped"] == 1 - - async def test_duplicate_translations_not_created( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that duplicate translations are not created on re-run.""" - # Pre-populate a translation - session.add(VariantTranslation(aa_clingen_id="PA00003", nt_clingen_id="CA9765210")) - session.commit() - - with ( - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_canonical_pa_ids", - return_value=["PA00003"], - ), - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_matching_registered_ca_ids", - return_value=[], - ), - ): - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_variant_translations_run.id), - ) - - assert result.status == JobStatus.SUCCEEDED - assert result.data["translations_created"] == 0 - - translations = session.scalars( - select(VariantTranslation).where(VariantTranslation.aa_clingen_id == "PA00003") - ).all() - assert len(translations) == 1 - - async def test_propagates_exceptions( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that unexpected exceptions are propagated.""" - with patch( - "mavedb.worker.jobs.external_services.variant_translation.get_canonical_pa_ids", - side_effect=Exception("Test exception"), - ): - with pytest.raises(Exception) as exc_info: - await populate_variant_translations_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_variant_translations_run.id), - ) - - assert str(exc_info.value) == "Test exception" - - async def test_multiple_alleles_sharing_pa_no_duplicate_error( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that two CA alleles mapping to the same PA don't cause a UniqueViolation. - - This is a regression test for a bug where the SELECT-then-INSERT upsert pattern - failed to detect in-session duplicates: both alleles' iterations called - upsert_variant_translations with overlapping (PA, CA) pairs, the SELECT found - no committed row, both staged db.add() for the same pair, and the subsequent - update_progress commit raised a UniqueViolation. - """ - # Add a second variant with a different CA allele under the same score set. - score_set_id = sample_populate_variant_translations_run.job_params["score_set_id"] - - variant2 = Variant( - urn="urn:variant:test-second-ca-allele", - score_set_id=score_set_id, - hgvs_nt="NM_000000.1:c.2T>G", - hgvs_pro="NP_000000.1:p.Val2Gly", - data={}, - ) - session.add(variant2) - session.commit() - mapped_variant2 = MappedVariant( - variant_id=variant2.id, - clingen_allele_id="CA_SECOND", - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant2) - session.commit() - - # Both CA alleles resolve to the same PA. The PA then returns the same set of - # registered CAs for both iterations, producing fully overlapping translation pairs. - with ( - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_canonical_pa_ids", - return_value=["PA_SHARED"], - ), - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_matching_registered_ca_ids", - return_value=["CA9765210", "CA_SECOND"], - ), - ): - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_variant_translations_run.id), - ) - - assert result.status == JobStatus.SUCCEEDED - - translations = session.scalars(select(VariantTranslation)).all() - pairs = {(t.aa_clingen_id, t.nt_clingen_id) for t in translations} - # PA_SHARED paired with each CA: CA9765210 (original from allele 1), - # CA_SECOND (original from allele 2), plus both as registered CAs. - assert ("PA_SHARED", "CA9765210") in pairs - assert ("PA_SHARED", "CA_SECOND") in pairs - - async def test_total_api_failure_returns_failed( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that the job returns FAILED when all variant translation lookups fail.""" - import requests - - with patch( - "mavedb.worker.jobs.external_services.variant_translation.get_canonical_pa_ids", - side_effect=requests.exceptions.ConnectionError("Connection failed"), - ): - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_variant_translations_run.id), - ) - - assert result.status == JobStatus.FAILED - assert result.failure_category == FailureCategory.DEPENDENCY_FAILURE - assert result.data["alleles_failed"] == 1 - assert result.data["translations_created"] == 0 - - -# --- Integration Tests --- - - -@pytest.mark.asyncio -@pytest.mark.integration -class TestPopulateVariantTranslationsIntegration: - """Integration tests that exercise the full decorator stack.""" - - async def test_no_mapped_variants( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - ): - """Test end-to-end when no mapped variants exist.""" - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, sample_populate_variant_translations_run.id - ) - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 0 - - session.refresh(sample_populate_variant_translations_run) - assert sample_populate_variant_translations_run.status == JobStatus.SUCCEEDED - - async def test_successful_job_updates_status( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that a successful job run updates the job status to SUCCEEDED.""" - with ( - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_canonical_pa_ids", - return_value=["PA00004"], - ), - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_matching_registered_ca_ids", - return_value=["CA77777"], - ), - ): - await populate_variant_translations_for_score_set( - mock_worker_ctx, - sample_populate_variant_translations_run.id, - ) - - session.refresh(sample_populate_variant_translations_run) - assert sample_populate_variant_translations_run.status == JobStatus.SUCCEEDED - - translations = session.scalars(select(VariantTranslation)).all() - assert len(translations) == 2 # PA00004->CA9765210 and PA00004->CA77777 - - async def test_job_with_pipeline_updates_pipeline_status( - self, - session, - with_populated_domain_data, - mock_worker_ctx, - sample_populate_variant_translations_run_pipeline, - sample_populate_variant_translations_pipeline, - setup_sample_variants_with_caid_for_translation, - ): - """Test that a job in a pipeline updates the pipeline status on success.""" - with ( - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_canonical_pa_ids", - return_value=["PA00005"], - ), - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_matching_registered_ca_ids", - return_value=[], - ), - ): - await populate_variant_translations_for_score_set( - mock_worker_ctx, - sample_populate_variant_translations_run_pipeline.id, - ) - - session.refresh(sample_populate_variant_translations_run_pipeline) - session.refresh(sample_populate_variant_translations_pipeline) - assert sample_populate_variant_translations_run_pipeline.status == JobStatus.SUCCEEDED - assert sample_populate_variant_translations_pipeline.status == PipelineStatus.SUCCEEDED - - async def test_variant_without_caid_creates_skipped_annotation( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that variants without CAIDs produce no annotations (filtered before processing).""" - _, mapped_variant = setup_sample_variants_with_caid_for_translation - mapped_variant.clingen_allele_id = None - session.commit() - - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, sample_populate_variant_translations_run.id - ) - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - assert result.data["translations_created"] == 0 - - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 0 - - session.refresh(sample_populate_variant_translations_run) - assert sample_populate_variant_translations_run.status == JobStatus.SUCCEEDED - - async def test_unrecognized_allele_creates_skipped_annotation( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that unrecognized allele formats create skipped annotations through the full stack.""" - _, mapped_variant = setup_sample_variants_with_caid_for_translation - mapped_variant.clingen_allele_id = "XX12345" - session.commit() - - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, sample_populate_variant_translations_run.id - ) - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == "skipped" - assert annotation_statuses[0].annotation_type == "variant_translation" - - session.refresh(sample_populate_variant_translations_run) - assert sample_populate_variant_translations_run.status == JobStatus.SUCCEEDED - - async def test_exceptions_handled_by_decorators( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that unexpected exceptions are handled by decorators.""" - with ( - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_canonical_pa_ids", - side_effect=Exception("Test exception"), - ), - patch("mavedb.worker.lib.decorators.job_management.send_slack_job_error") as mock_send_slack_job_error, - ): - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, - sample_populate_variant_translations_run.id, - ) - - mock_send_slack_job_error.assert_called_once() - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.ERRORED - assert isinstance(result.exception, Exception) - - session.refresh(sample_populate_variant_translations_run) - assert sample_populate_variant_translations_run.status == JobStatus.ERRORED - - -# --- ARQ Context Tests --- - - -@pytest.mark.asyncio -@pytest.mark.integration -class TestPopulateVariantTranslationsArqContext: - """Tests for populate_variant_translations_for_score_set job using the ARQ context fixture.""" - - async def test_with_arq_context_independent( - self, - arq_redis, - arq_worker, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that the job works with the ARQ context fixture.""" - with ( - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_canonical_pa_ids", - return_value=["PA00006"], - ), - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_matching_registered_ca_ids", - return_value=["CA88888"], - ), - ): - await arq_redis.enqueue_job( - "populate_variant_translations_for_score_set", - sample_populate_variant_translations_run.id, - ) - await arq_worker.async_run() - await arq_worker.run_check() - - session.refresh(sample_populate_variant_translations_run) - assert sample_populate_variant_translations_run.status == JobStatus.SUCCEEDED - - translations = session.scalars(select(VariantTranslation)).all() - assert len(translations) == 2 # PA00006->CA9765210 and PA00006->CA88888 - - async def test_with_arq_context_pipeline( - self, - arq_redis, - arq_worker, - session, - with_populated_domain_data, - sample_populate_variant_translations_run_pipeline, - sample_populate_variant_translations_pipeline, - setup_sample_variants_with_caid_for_translation, - ): - """Test that the job works with the ARQ context fixture in a pipeline.""" - with ( - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_canonical_pa_ids", - return_value=["PA00007"], - ), - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_matching_registered_ca_ids", - return_value=[], - ), - ): - await arq_redis.enqueue_job( - "populate_variant_translations_for_score_set", - sample_populate_variant_translations_run_pipeline.id, - ) - await arq_worker.async_run() - await arq_worker.run_check() - - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == "success" - assert annotation_statuses[0].annotation_type == "variant_translation" - - session.refresh(sample_populate_variant_translations_run_pipeline) - assert sample_populate_variant_translations_run_pipeline.status == JobStatus.SUCCEEDED - - session.refresh(sample_populate_variant_translations_pipeline) - assert sample_populate_variant_translations_pipeline.status == PipelineStatus.SUCCEEDED - - async def test_with_arq_context_exception_handling_independent( - self, - arq_redis, - arq_worker, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that exceptions are handled with the ARQ context fixture.""" - with ( - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_canonical_pa_ids", - side_effect=Exception("Test exception"), - ), - patch("mavedb.worker.lib.decorators.job_management.send_slack_job_error") as mock_send_slack_job_error, - ): - await arq_redis.enqueue_job( - "populate_variant_translations_for_score_set", - sample_populate_variant_translations_run.id, - ) - await arq_worker.async_run() - await arq_worker.run_check() - - mock_send_slack_job_error.assert_called_once() - - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 0 - - session.refresh(sample_populate_variant_translations_run) - assert sample_populate_variant_translations_run.status == JobStatus.ERRORED - - async def test_with_arq_context_exception_handling_pipeline( - self, - arq_redis, - arq_worker, - session, - with_populated_domain_data, - sample_populate_variant_translations_pipeline, - sample_populate_variant_translations_run_pipeline, - setup_sample_variants_with_caid_for_translation, - ): - """Test that exceptions in pipeline context are handled.""" - with ( - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_canonical_pa_ids", - side_effect=Exception("Test exception"), - ), - patch("mavedb.worker.lib.decorators.job_management.send_slack_job_error") as mock_send_slack_job_error, - ): - await arq_redis.enqueue_job( - "populate_variant_translations_for_score_set", - sample_populate_variant_translations_run_pipeline.id, - ) - await arq_worker.async_run() - await arq_worker.run_check() - - mock_send_slack_job_error.assert_called_once() - - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 0 - - session.refresh(sample_populate_variant_translations_run_pipeline) - assert sample_populate_variant_translations_run_pipeline.status == JobStatus.ERRORED - - session.refresh(sample_populate_variant_translations_pipeline) - assert sample_populate_variant_translations_pipeline.status == PipelineStatus.FAILED From c936dd4735556a74d0174eb5450a6b5eb772ca97 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 23 Jun 2026 09:27:43 -0700 Subject: [PATCH 033/252] feat(alleles): cross-layer allele equivalence query with as_of support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit get_allele_translations(db, allele_id, *, as_of=None) returns an allele's full cross-layer equivalence set (genomic/coding/protein) by traversing the MappingRecordAllele link graph: allele -> its live links -> mapping record(s) -> all co-linked alleles. The relation is co-membership in a MappingRecord's allele set, not a shared identifier — ClinGen's CAID spans only the nucleotide layers (the protein allele carries a distinct PA), so the link graph is the only thing tying all layers together. This replaces what the retired variant_translations PA<->CA table provided. Forward-compatible with temporal reads: the same half-open valid-time predicate is applied at both the anchor and fan-out hops, so passing as_of reconstructs the equivalence set as of any instant. Defaults to the currently-live set. --- src/mavedb/lib/alleles.py | 54 +++++++++++++++++++ tests/worker/jobs/test_allele_translations.py | 41 ++++++++++++++ 2 files changed, 95 insertions(+) create mode 100644 src/mavedb/lib/alleles.py create mode 100644 tests/worker/jobs/test_allele_translations.py diff --git a/src/mavedb/lib/alleles.py b/src/mavedb/lib/alleles.py new file mode 100644 index 000000000..ddf51565b --- /dev/null +++ b/src/mavedb/lib/alleles.py @@ -0,0 +1,54 @@ +"""Allele-graph queries over the deduplicated allele model. + +These traverse the ``MappingRecordAllele`` link graph rather than any external identifier, because an +allele's cross-layer equivalence (its genomic / coding / protein representations) is established by the +mapping + reverse-translation process and materialized as co-membership in a ``MappingRecord``'s allele +set. No single identifier spans the layers: ClinGen's canonical allele id (CAID) covers only the +nucleotide layers, the protein allele carries a distinct PA, so the link graph is the only thing that +ties all three together. +""" + +from datetime import datetime +from typing import Optional + +from sqlalchemy import select +from sqlalchemy.orm import Session + +from mavedb.models.allele import Allele +from mavedb.models.mapping_record_allele import MappingRecordAllele + + +def get_allele_translations(db: Session, allele_id: int, *, as_of: Optional[datetime] = None) -> list[Allele]: + """Return the cross-layer equivalence set of ``allele_id``: every allele co-linked to a + ``MappingRecord`` that links it (the anchor allele itself included), spanning the genomic, coding, + and protein layers. + + The relation is co-membership in a record's allele set, not a shared identifier — see the module + docstring. Because alleles are deduplicated by ``vrs_digest`` and shared across variants/score sets, + the anchor may belong to several ``MappingRecord``s; the result is the union of their allele sets + (normally the same biological equivalence class). Scope by record or score set upstream if a single + context is required. + + Temporal: by default returns the currently-live set (``valid_to IS NULL``). Pass ``as_of`` to + reconstruct the set as it stood at a past instant — both the anchor hop and the fan-out hop apply + the same half-open ``[valid_from, valid_to)`` predicate, so the whole set is evaluated at one + instant. The retire-cascade invariant (a live link implies a live record) holds under ``as_of`` too, + so filtering the links alone is sufficient. + """ + link_live = MappingRecordAllele.as_of(as_of) if as_of is not None else MappingRecordAllele.current + + record_ids = db.scalars( + select(MappingRecordAllele.mapping_record_id).where(MappingRecordAllele.allele_id == allele_id).where(link_live) + ).all() + if not record_ids: + return [] + + return list( + db.scalars( + select(Allele) + .join(MappingRecordAllele, MappingRecordAllele.allele_id == Allele.id) + .where(MappingRecordAllele.mapping_record_id.in_(record_ids)) + .where(link_live) + .distinct() + ).all() + ) diff --git a/tests/worker/jobs/test_allele_translations.py b/tests/worker/jobs/test_allele_translations.py new file mode 100644 index 000000000..0e1c1b12c --- /dev/null +++ b/tests/worker/jobs/test_allele_translations.py @@ -0,0 +1,41 @@ +# ruff: noqa: E402 + +import pytest + +pytest.importorskip("arq") + +from datetime import datetime, timezone + +from mavedb.lib.alleles import get_allele_translations + + +@pytest.mark.unit +class TestGetAlleleTranslations: + """The cross-layer equivalence query traverses the MappingRecordAllele link graph. + + Uses the RT-derived fixture, which links an authoritative allele and a derived (cross-layer) + allele to one MappingRecord — exactly the co-membership the query resolves. + """ + + def test_returns_full_equivalence_set_from_any_member(self, session, setup_rt_derived_allele_with_caid): + _variant, authoritative_allele, rt_allele = setup_rt_derived_allele_with_caid + + from_authoritative = {a.id for a in get_allele_translations(session, authoritative_allele.id)} + from_rt = {a.id for a in get_allele_translations(session, rt_allele.id)} + + # Reachable from either member, and includes the anchor itself — the full equivalence set. + assert from_authoritative == {authoritative_allele.id, rt_allele.id} + assert from_rt == {authoritative_allele.id, rt_allele.id} + + def test_as_of_before_links_existed_is_empty(self, session, setup_rt_derived_allele_with_caid): + _variant, authoritative_allele, _rt = setup_rt_derived_allele_with_caid + + past = datetime(2000, 1, 1, tzinfo=timezone.utc) + assert get_allele_translations(session, authoritative_allele.id, as_of=past) == [] + + def test_as_of_after_links_existed_returns_set(self, session, setup_rt_derived_allele_with_caid): + _variant, authoritative_allele, rt_allele = setup_rt_derived_allele_with_caid + + future = datetime(2999, 1, 1, tzinfo=timezone.utc) + ids = {a.id for a in get_allele_translations(session, authoritative_allele.id, as_of=future)} + assert ids == {authoritative_allele.id, rt_allele.id} From d397398ee8e4bb5697755df312cdfc941770b83d Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 30 Jun 2026 08:29:21 -0700 Subject: [PATCH 034/252] feat(annotation): add append-only AnnotationEvent log and status vocabulary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduces the AnnotationEvent log: a single append-only event table whose subjects are Variant and Allele, selected by annotation_type via a polymorphic CHECK. "Current" is derived (DISTINCT ON … id DESC), never stored. Adds the shared Disposition (present/absent/not_applicable/failed) and EventReason vocabulary, the v_current_annotation_events view, and the supporting migrations. --- .../a7e1c4f9b3d2_add_annotation_event.py | 79 ++++++++++ ...e4_add_v_current_annotation_events_view.py | 30 ++++ src/mavedb/models/__init__.py | 2 + src/mavedb/models/annotation_event.py | 138 ++++++++++++++++ src/mavedb/models/annotation_event_view.py | 96 ++++++++++++ src/mavedb/models/enums/disposition.py | 19 +++ src/mavedb/models/enums/event_reason.py | 42 +++++ tests/models/conftest.py | 80 ++++++++++ tests/models/test_annotation_event_model.py | 103 ++++++++++++ tests/models/test_annotation_event_view.py | 147 ++++++++++++++++++ 10 files changed, 736 insertions(+) create mode 100644 alembic/versions/a7e1c4f9b3d2_add_annotation_event.py create mode 100644 alembic/versions/b8f2c5a1d3e4_add_v_current_annotation_events_view.py create mode 100644 src/mavedb/models/annotation_event.py create mode 100644 src/mavedb/models/annotation_event_view.py create mode 100644 src/mavedb/models/enums/disposition.py create mode 100644 src/mavedb/models/enums/event_reason.py create mode 100644 tests/models/conftest.py create mode 100644 tests/models/test_annotation_event_model.py create mode 100644 tests/models/test_annotation_event_view.py diff --git a/alembic/versions/a7e1c4f9b3d2_add_annotation_event.py b/alembic/versions/a7e1c4f9b3d2_add_annotation_event.py new file mode 100644 index 000000000..cf8d77e35 --- /dev/null +++ b/alembic/versions/a7e1c4f9b3d2_add_annotation_event.py @@ -0,0 +1,79 @@ +"""add annotation_event log + +Revision ID: a7e1c4f9b3d2 +Revises: d4e5f6a7b8c9 +Create Date: 2026-06-25 16:00:00.000000 + +""" + +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "a7e1c4f9b3d2" +down_revision = "d4e5f6a7b8c9" +branch_labels = None +depends_on = None + +_VARIANT_SUBJECT_TYPES = "'vrs_mapping', 'cross_level_translation', 'variant_translation', 'ldh_submission'" +_ALLELE_SUBJECT_TYPES = ( + "'clingen_allele_id', 'gnomad_allele_frequency', 'vep_functional_consequence', 'clinvar_control', 'mapped_hgvs'" +) + + +def upgrade(): + op.create_table( + "annotation_event", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("annotation_type", sa.String(length=50), nullable=False), + sa.Column("variant_id", sa.Integer(), nullable=True), + sa.Column("allele_id", sa.Integer(), nullable=True), + sa.Column("disposition", sa.String(length=50), nullable=False), + sa.Column("reason", sa.String(length=50), nullable=False), + sa.Column("source_version", sa.String(length=50), nullable=True), + sa.Column("metadata", postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column("job_run_id", sa.Integer(), nullable=True), + sa.Column("score_set_id", sa.Integer(), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.CheckConstraint( + f"(annotation_type IN ({_VARIANT_SUBJECT_TYPES}) " + "AND variant_id IS NOT NULL AND allele_id IS NULL) " + f"OR (annotation_type IN ({_ALLELE_SUBJECT_TYPES}) " + "AND allele_id IS NOT NULL AND variant_id IS NULL)", + name="ck_annotation_event_subject", + ), + sa.ForeignKeyConstraint(["variant_id"], ["variants.id"], ondelete="RESTRICT"), + sa.ForeignKeyConstraint(["allele_id"], ["alleles.id"], ondelete="RESTRICT"), + sa.ForeignKeyConstraint(["job_run_id"], ["job_runs.id"], ondelete="SET NULL"), + sa.ForeignKeyConstraint(["score_set_id"], ["scoresets.id"], ondelete="SET NULL"), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + "ix_annotation_event_allele_type_id", + "annotation_event", + ["allele_id", "annotation_type", sa.text("id DESC")], + unique=False, + ) + op.create_index( + "ix_annotation_event_variant_type_id", + "annotation_event", + ["variant_id", "annotation_type", sa.text("id DESC")], + unique=False, + ) + op.create_index( + "ix_annotation_event_allele_type_version", + "annotation_event", + ["allele_id", "annotation_type", "source_version"], + unique=False, + ) + op.create_index("ix_annotation_event_job_run_id", "annotation_event", ["job_run_id"], unique=False) + + +def downgrade(): + op.drop_index("ix_annotation_event_job_run_id", table_name="annotation_event") + op.drop_index("ix_annotation_event_allele_type_version", table_name="annotation_event") + op.drop_index("ix_annotation_event_variant_type_id", table_name="annotation_event") + op.drop_index("ix_annotation_event_allele_type_id", table_name="annotation_event") + op.drop_table("annotation_event") diff --git a/alembic/versions/b8f2c5a1d3e4_add_v_current_annotation_events_view.py b/alembic/versions/b8f2c5a1d3e4_add_v_current_annotation_events_view.py new file mode 100644 index 000000000..73843866c --- /dev/null +++ b/alembic/versions/b8f2c5a1d3e4_add_v_current_annotation_events_view.py @@ -0,0 +1,30 @@ +"""add v_current_annotation_events view + +Revision ID: b8f2c5a1d3e4 +Revises: a7e1c4f9b3d2 +Create Date: 2026-06-26 + +Creates the v_current_annotation_events view: the latest AnnotationEvent per (subject, annotation_type), +with ClinVar partitioned additionally by source_version (multi-live, one current row per release). +Replaces the per-variant v_variant_annotations as the current-state projection over the new +allele-model annotation log. Intended for operator queries, BI, and the annotation CLI scripts. +""" + +from alembic import op + +from mavedb.db.view import CreateView, DropView +from mavedb.models.annotation_event_view import definition, signature + +# revision identifiers, used by Alembic. +revision = "b8f2c5a1d3e4" +down_revision = "a7e1c4f9b3d2" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.execute(CreateView(signature, definition, materialized=False)) + + +def downgrade() -> None: + op.execute(DropView(signature, materialized=False)) diff --git a/src/mavedb/models/__init__.py b/src/mavedb/models/__init__.py index c181e4099..1cec88e97 100644 --- a/src/mavedb/models/__init__.py +++ b/src/mavedb/models/__init__.py @@ -42,6 +42,8 @@ "uniprot_offset", "user", "variant_annotation_status", + "annotation_event", + "annotation_event_view", "variant", "variant_translation", "vep_allele_consequence", diff --git a/src/mavedb/models/annotation_event.py b/src/mavedb/models/annotation_event.py new file mode 100644 index 000000000..eae8877a0 --- /dev/null +++ b/src/mavedb/models/annotation_event.py @@ -0,0 +1,138 @@ +""" +SQLAlchemy model for the annotation event log. + +One append-only log spanning the whole pipeline (mapping, reverse translation, +annotation). Its subjects are exactly two persistent entities — ``Variant`` and +``Allele`` — selected by ``annotation_type`` via the polymorphic-subject CHECK. +Everything else the pipeline touches (``MappingRecord``, ``MappingRecordAllele``, +the external value tables) is a vehicle or resolution path, never a status subject. + +This is deliberately **not** a ``ValidTime`` table. A SCD-2 state table is a lossy +projection of an event log — it discards the skip/reconfirm/no-op events that are +the audit point. "Current" is derived (``DISTINCT ON (subject, annotation_type) +… id DESC``), never a stored ``current`` flag. +""" + +from datetime import datetime +from typing import TYPE_CHECKING, Any, Dict, Optional + +from sqlalchemy import CheckConstraint, DateTime, ForeignKey, Index, Integer, String, func, text +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.ext.mutable import MutableDict +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from mavedb.db.base import Base +from mavedb.models.enums.annotation_type import AnnotationType +from mavedb.models.enums.disposition import Disposition +from mavedb.models.enums.event_reason import EventReason + +if TYPE_CHECKING: + from mavedb.models.allele import Allele + from mavedb.models.job_run import JobRun + from mavedb.models.score_set import ScoreSet + from mavedb.models.variant import Variant + +VARIANT_SUBJECT_TYPES = ( + AnnotationType.VRS_MAPPING.value, + AnnotationType.CROSS_LEVEL_TRANSLATION.value, + AnnotationType.VARIANT_TRANSLATION.value, + AnnotationType.LDH_SUBMISSION.value, +) +"""annotation_type values whose subject is the variant (variant_id set, allele_id null)""" + +ALLELE_SUBJECT_TYPES = ( + AnnotationType.CLINGEN_ALLELE_ID.value, + AnnotationType.GNOMAD_ALLELE_FREQUENCY.value, + AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE.value, + AnnotationType.CLINVAR_CONTROL.value, + AnnotationType.MAPPED_HGVS.value, +) +"""annotation_type values whose subject is the allele (allele_id set, variant_id null)""" + + +def _sql_in_list(values: tuple[str, ...]) -> str: + return ", ".join(f"'{v}'" for v in values) + + +class AnnotationEvent(Base): + """An append-only event recording the *status* (not value) of one pipeline + observation about a ``Variant`` or an ``Allele``. + + The value (frequency, consequence, CAID, control) lives in the domain + ValidTime tables; this log records disposition + why + when. Reading the + domain tables alone cannot distinguish confirmed-absence from never-checked — + that gap is the whole reason the log exists. + + NOTE: JSONB ``event_metadata`` is tracked as a mutable object via MutableDict, + which only catches top-level mutations. Mutating a nested object + requires ``flag_modified(instance, "event_metadata")``. + """ + + __tablename__ = "annotation_event" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + + annotation_type: Mapped[AnnotationType] = mapped_column(String(50), nullable=False) + + # Exactly one is set, per ck_annotation_event_subject. + variant_id: Mapped[Optional[int]] = mapped_column( + Integer, ForeignKey("variants.id", ondelete="RESTRICT"), nullable=True + ) + allele_id: Mapped[Optional[int]] = mapped_column( + Integer, ForeignKey("alleles.id", ondelete="RESTRICT"), nullable=True + ) + + disposition: Mapped[Disposition] = mapped_column(String(50), nullable=False) + + # Domain-specific code reusing in-code vocabularies (EventReason, plus MappingOutcome and RT + # skip_category for those two jobs); disposition is the public axis. + reason: Mapped[EventReason] = mapped_column(String(50), nullable=False) + + # gnomAD db_version / Ensembl release / ClinVar release / mapper version. + source_version: Mapped[Optional[str]] = mapped_column(String(50), nullable=True) + + # DB column is "metadata"; the attribute avoids the reserved Declarative name. + event_metadata: Mapped[Optional[Dict[str, Any]]] = mapped_column( + "metadata", MutableDict.as_mutable(JSONB), nullable=True + ) + + job_run_id: Mapped[Optional[int]] = mapped_column( + Integer, ForeignKey("job_runs.id", ondelete="SET NULL"), nullable=True + ) + score_set_id: Mapped[Optional[int]] = mapped_column( + Integer, ForeignKey("scoresets.id", ondelete="SET NULL"), nullable=True + ) + + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now()) + + # One-directional (no back-ref) — match the annotation-link convention. + variant: Mapped[Optional["Variant"]] = relationship("Variant") + allele: Mapped[Optional["Allele"]] = relationship("Allele") + job_run: Mapped[Optional["JobRun"]] = relationship("JobRun") + score_set: Mapped[Optional["ScoreSet"]] = relationship("ScoreSet") + + __table_args__ = ( + # Polymorphic subject: the type picks exactly one subject column. + CheckConstraint( + f"(annotation_type IN ({_sql_in_list(VARIANT_SUBJECT_TYPES)}) " + "AND variant_id IS NOT NULL AND allele_id IS NULL) " + f"OR (annotation_type IN ({_sql_in_list(ALLELE_SUBJECT_TYPES)}) " + "AND allele_id IS NOT NULL AND variant_id IS NULL)", + name="ck_annotation_event_subject", + ), + # latest-per-allele / latest-per-variant (the DISTINCT ON … id DESC projections) + Index("ix_annotation_event_allele_type_id", "allele_id", "annotation_type", text("id DESC")), + Index("ix_annotation_event_variant_type_id", "variant_id", "annotation_type", text("id DESC")), + # version-keyed skip + Index("ix_annotation_event_allele_type_version", "allele_id", "annotation_type", "source_version"), + # audit by run + Index("ix_annotation_event_job_run_id", "job_run_id"), + ) + + def __repr__(self) -> str: + subject = f"variant_id={self.variant_id}" if self.variant_id is not None else f"allele_id={self.allele_id}" + return ( + f"" + ) diff --git a/src/mavedb/models/annotation_event_view.py b/src/mavedb/models/annotation_event_view.py new file mode 100644 index 000000000..5c8547807 --- /dev/null +++ b/src/mavedb/models/annotation_event_view.py @@ -0,0 +1,96 @@ +"""``v_current_annotation_events`` — the current-state projection over the AnnotationEvent log. + +The log is append-only; "current" is derived, never stored. This view exposes the latest event per +``(subject, annotation_type)`` — one row per allele/variant per annotation type — so operators, BI, +and app code can read current status with a plain ``SELECT`` instead of re-deriving the +``DISTINCT ON`` everywhere. It mirrors the existing ``v_variant_annotations`` pattern. + +ClinVar is **multi-live**: an allele accumulates one live link per archival release, so its current +status is one row *per release*. The window partition folds ``source_version`` in **only** for +``clinvar_control`` (via a CASE that is constant-NULL for every other type, collapsing those back to +one row per subject+type). + +The subject is polymorphic — exactly one of ``variant_id`` / ``allele_id`` is set per row (enforced by +the log's CHECK), and the other is constant-NULL within a partition, so partitioning by both keys each +row to its real subject. + +There is deliberately **no ``score_set_id`` axis**: an allele-subject status (CAID, gnomAD, ClinVar, +VEP) is a *shared* allele-level fact, not a property of any one score set. A consumer that wants a +score set's annotation status resolves the score set's current alleles (and variants) through the live +mapping links — e.g. ``lib.clingen.alleles.get_alleles_for_score_set`` — and then looks those subjects +up here. Filtering by the *run's* score set would wrongly drop an allele last (re-)annotated by another +score set's run. The derived "current-for-variant" walk (resolving an allele-subject fact down to a +variant at the level a type keys on) is intentionally **not** built here — that is the deferred +consumer surface; see ``docs/design/allele-annotation-status.md``. +""" + +from sqlalchemy import case, func, select + +from mavedb.db.base import Base +from mavedb.db.view import view +from mavedb.models.annotation_event import AnnotationEvent +from mavedb.models.enums.annotation_type import AnnotationType + +signature = "v_current_annotation_events" + +# ClinVar is the lone multi-live type: split "current" by release. For every other type this is NULL, +# so the partition collapses to (allele_id, variant_id, annotation_type) — one current row per subject. +_clinvar_release_key = case( + (AnnotationEvent.annotation_type == AnnotationType.CLINVAR_CONTROL.value, AnnotationEvent.source_version), + else_=None, +) + +_ranked = select( + AnnotationEvent.id.label("id"), + AnnotationEvent.annotation_type.label("annotation_type"), + AnnotationEvent.variant_id.label("variant_id"), + AnnotationEvent.allele_id.label("allele_id"), + AnnotationEvent.disposition.label("disposition"), + AnnotationEvent.reason.label("reason"), + AnnotationEvent.source_version.label("source_version"), + AnnotationEvent.event_metadata.label("event_metadata"), + AnnotationEvent.job_run_id.label("job_run_id"), + AnnotationEvent.created_at.label("created_at"), + func.row_number() + .over( + partition_by=[ + AnnotationEvent.allele_id, + AnnotationEvent.variant_id, + AnnotationEvent.annotation_type, + _clinvar_release_key, + ], + order_by=AnnotationEvent.id.desc(), + ) + .label("row_number"), +).subquery("ranked_annotation_events") + +definition = select( + _ranked.c.id, + _ranked.c.annotation_type, + _ranked.c.variant_id, + _ranked.c.allele_id, + _ranked.c.disposition, + _ranked.c.reason, + _ranked.c.source_version, + _ranked.c.event_metadata, + _ranked.c.job_run_id, + _ranked.c.created_at, +).where(_ranked.c.row_number == 1) + + +class CurrentAnnotationEventView(Base): + __table__ = view(signature, definition, materialized=False) + # Each surviving event id is unique across the view, so it is a valid mapping key for the + # otherwise-PK-less view (standard SQLAlchemy view-mapping idiom). + __mapper_args__ = {"primary_key": [__table__.c.id]} + + id = __table__.c.id + annotation_type = __table__.c.annotation_type + variant_id = __table__.c.variant_id + allele_id = __table__.c.allele_id + disposition = __table__.c.disposition + reason = __table__.c.reason + source_version = __table__.c.source_version + event_metadata = __table__.c.event_metadata + job_run_id = __table__.c.job_run_id + created_at = __table__.c.created_at diff --git a/src/mavedb/models/enums/disposition.py b/src/mavedb/models/enums/disposition.py new file mode 100644 index 000000000..94438f292 --- /dev/null +++ b/src/mavedb/models/enums/disposition.py @@ -0,0 +1,19 @@ +from enum import Enum + + +class Disposition(str, Enum): + """The stable, consumer-facing status axis of a variant event. + + Defined by *what the consumer may conclude* — not by the domain-specific + operation that produced it (that lives in ``reason``). + + - ``present`` — we hold the result / the step succeeded + - ``absent`` — the source or biology has nothing — an informative negative + - ``not_applicable`` — we could not ask — a pipeline/structural gap, not a statement about the source + - ``failed`` — errored, might retry; failure_category carries transient vs permanent + """ + + PRESENT = "present" + ABSENT = "absent" + NOT_APPLICABLE = "not_applicable" + FAILED = "failed" diff --git a/src/mavedb/models/enums/event_reason.py b/src/mavedb/models/enums/event_reason.py new file mode 100644 index 000000000..f7fd4b56b --- /dev/null +++ b/src/mavedb/models/enums/event_reason.py @@ -0,0 +1,42 @@ +from enum import Enum + + +class EventReason(str, Enum): + """Pipeline-wide vocabulary for a variant event's ``reason`` — the single field that says + *what happened*, spanning present / absent / not_applicable / failed. + + Reasons are shared across jobs wherever they mean the same thing (the ``annotation_type`` + already says *which* job), and job-specific only where a case is genuinely unique. One job + contributes its own pre-existing domain enum instead of duplicating here: ``mapping`` uses + ``MappingOutcome`` (mapped / intronic / no_protein_consequence / failed), which mirrors the + external dcd-mapping vocabulary. The full ``reason`` vocabulary is this enum plus that one. + """ + + # present — we hold the result / the step succeeded + CREATED = "created" # linked/registered this run (gnomAD, ClinVar, CAR) + PREEXISTING = "preexisting" # already held before this run (gnomAD, ClinVar, CAR) + RECONFIRMED = "reconfirmed" # re-verified unchanged (gnomAD, CAR force) + SKIPPED = "skipped" # version-skip: already current at this source version (gnomAD, VEP) + SUPERSEDED = "superseded" # re-resolved within a release, newest wins (ClinVar) + SUBMITTED = "submitted" # LDH + TRANSLATED = "translated" # reverse translation + RECONFIRMATION_SKIPPED = "reconfirmation_skipped" # HGVS no longer buildable, existing CAID kept (CAR) + + # absent — the source or biology has nothing (informative negative) + NO_RECORD = "no_record" # source queried, returned nothing (gnomAD, ClinVar, VEP) + NO_CODING_TRANSCRIPT = "no_coding_transcript" # non-coding target has no protein consequence (RT) + + # not_applicable — we could not ask (structural gap) + NO_CAID = "no_caid" # no ClinGen allele id to key on (gnomAD, ClinVar) + NO_HGVS = "no_hgvs" # no HGVS to submit/resolve (CAR, VEP) + MULTI_VARIANT_CAID = "multi_variant_caid" # cis-block CAID cannot be used (ClinVar) + NO_ASSAY_LEVEL_HGVS = "no_assay_level_hgvs" # no assay-level HGVS to translate (RT) + + # failed — errored + API_ERROR = "api_error" # network/timeout/upstream error (ClinVar, VEP, LDH, CAR no-response) + SERVICE_REJECTED = "service_rejected" # external service refused the input (CAR) + MALFORMED_RESPONSE = "malformed_response" # unparseable/contractless response (CAR) + CAID_CONFLICT = "caid_conflict" # returned identifier conflicts with the stored one (CAR) + TRANSLATION_FAILED = "translation_failed" # all candidate HGVS failed translation (RT) + TRANSLATION_ERROR = "translation_error" # the translation engine errored (RT) + TRANSCRIPT_UNRESOLVED = "transcript_unresolved" # protein-coding target with no resolvable transcript (RT) diff --git a/tests/models/conftest.py b/tests/models/conftest.py new file mode 100644 index 000000000..5edfe06ee --- /dev/null +++ b/tests/models/conftest.py @@ -0,0 +1,80 @@ +import pytest + +from mavedb.models.enums import JobStatus +from mavedb.models.experiment import Experiment +from mavedb.models.experiment_set import ExperimentSet +from mavedb.models.job_run import JobRun +from mavedb.models.score_set import ScoreSet +from mavedb.models.user import User +from mavedb.models.variant import Variant +from tests.helpers.constants import ( + TEST_EXPERIMENT, + TEST_EXPERIMENT_SET, + TEST_LICENSE, + TEST_MINIMAL_VARIANT, + TEST_SEQ_SCORESET, + TEST_USER, + VALID_EXPERIMENT_SET_URN, + VALID_EXPERIMENT_URN, + VALID_SCORE_SET_URN, +) + + +@pytest.fixture +def setup_lib_db_with_score_set(session, setup_lib_db): + """Build an experiment set, experiment, and score set on top of the base lib db (users/licenses).""" + user = session.query(User).filter(User.username == TEST_USER["username"]).first() + + experiment_set = ExperimentSet(**TEST_EXPERIMENT_SET, urn=VALID_EXPERIMENT_SET_URN) + experiment_set.created_by = user + experiment_set.modified_by = user + session.add(experiment_set) + session.commit() + session.refresh(experiment_set) + + experiment = Experiment(**TEST_EXPERIMENT, urn=VALID_EXPERIMENT_URN, experiment_set_id=experiment_set.id) + experiment.created_by = user + experiment.modified_by = user + session.add(experiment) + session.commit() + session.refresh(experiment) + + score_set_scaffold = TEST_SEQ_SCORESET.copy() + score_set_scaffold.pop("target_genes") + score_set = ScoreSet( + **score_set_scaffold, urn=VALID_SCORE_SET_URN, experiment_id=experiment.id, licence_id=TEST_LICENSE["id"] + ) + score_set.created_by = user + score_set.modified_by = user + session.add(score_set) + session.commit() + session.refresh(score_set) + + return score_set + + +@pytest.fixture +def setup_lib_db_with_variant(session, setup_lib_db_with_score_set): + """Add a single variant to the score set, for variant-subject event tests.""" + variant = Variant( + **TEST_MINIMAL_VARIANT, urn=f"{setup_lib_db_with_score_set.urn}#1", score_set_id=setup_lib_db_with_score_set.id + ) + session.add(variant) + session.commit() + session.refresh(variant) + + return variant + + +@pytest.fixture +def job_run(session): + """Create a persisted JobRun to anchor an event's provenance.""" + job = JobRun( + job_type="test_annotation_job", + job_function="test_function", + status=JobStatus.RUNNING, + ) + session.add(job) + session.commit() + session.refresh(job) + return job diff --git a/tests/models/test_annotation_event_model.py b/tests/models/test_annotation_event_model.py new file mode 100644 index 000000000..336536479 --- /dev/null +++ b/tests/models/test_annotation_event_model.py @@ -0,0 +1,103 @@ +# ruff: noqa: E402 + +import pytest + +pytest.importorskip("psycopg2") + +from sqlalchemy.exc import IntegrityError + +from mavedb.models.allele import Allele +from mavedb.models.annotation_event import AnnotationEvent +from mavedb.models.enums.annotation_type import AnnotationType +from mavedb.models.enums.disposition import Disposition + + +@pytest.fixture +def allele(session): + allele = Allele(vrs_digest="test-variant-event-allele-digest", level="genomic") + session.add(allele) + session.commit() + session.refresh(allele) + return allele + + +def _expect_rejected(session, event): + """Adding `event` must violate a CHECK constraint on flush.""" + session.add(event) + with pytest.raises(IntegrityError): + session.flush() + session.rollback() + + +class TestSubjectConstraint: + def test_variant_subject_event_inserts(self, session, setup_lib_db_with_variant, job_run): + event = AnnotationEvent( + annotation_type=AnnotationType.VRS_MAPPING, + variant_id=setup_lib_db_with_variant.id, + disposition=Disposition.PRESENT, + reason="mapped", + job_run_id=job_run.id, + score_set_id=setup_lib_db_with_variant.score_set_id, + ) + session.add(event) + session.commit() + assert event.id is not None + assert event.allele_id is None + + def test_allele_subject_event_inserts(self, session, allele, job_run): + event = AnnotationEvent( + annotation_type=AnnotationType.GNOMAD_ALLELE_FREQUENCY, + allele_id=allele.id, + disposition=Disposition.ABSENT, + reason="no_record", + source_version="4.1.0", + job_run_id=job_run.id, + ) + session.add(event) + session.commit() + assert event.id is not None + assert event.variant_id is None + + def test_variant_subject_type_with_allele_id_rejected(self, session, allele): + _expect_rejected( + session, + AnnotationEvent( + annotation_type=AnnotationType.VRS_MAPPING, + allele_id=allele.id, + disposition=Disposition.PRESENT, + reason="mapped", + ), + ) + + def test_allele_subject_type_with_variant_id_rejected(self, session, setup_lib_db_with_variant): + _expect_rejected( + session, + AnnotationEvent( + annotation_type=AnnotationType.GNOMAD_ALLELE_FREQUENCY, + variant_id=setup_lib_db_with_variant.id, + disposition=Disposition.PRESENT, + reason="created", + ), + ) + + def test_neither_subject_set_rejected(self, session): + _expect_rejected( + session, + AnnotationEvent( + annotation_type=AnnotationType.VRS_MAPPING, + disposition=Disposition.PRESENT, + reason="mapped", + ), + ) + + def test_both_subjects_set_rejected(self, session, setup_lib_db_with_variant, allele): + _expect_rejected( + session, + AnnotationEvent( + annotation_type=AnnotationType.VRS_MAPPING, + variant_id=setup_lib_db_with_variant.id, + allele_id=allele.id, + disposition=Disposition.PRESENT, + reason="mapped", + ), + ) diff --git a/tests/models/test_annotation_event_view.py b/tests/models/test_annotation_event_view.py new file mode 100644 index 000000000..14c74f802 --- /dev/null +++ b/tests/models/test_annotation_event_view.py @@ -0,0 +1,147 @@ +# ruff: noqa: E402 +"""Tests for the v_current_annotation_events view (current event per subject + type).""" + +import pytest + +pytest.importorskip("psycopg2") + +from sqlalchemy import select + +from mavedb.models.allele import Allele +from mavedb.models.annotation_event import AnnotationEvent +from mavedb.models.annotation_event_view import CurrentAnnotationEventView +from mavedb.models.enums.annotation_type import AnnotationType +from mavedb.models.enums.disposition import Disposition + + +def _allele(session, digest, level="genomic"): + allele = Allele(vrs_digest=digest, level=level, post_mapped={"type": "Allele"}) + session.add(allele) + session.commit() + session.refresh(allele) + return allele + + +def _event(session, job_run, annotation_type, disposition, reason, *, allele_id=None, variant_id=None, **kw): + event = AnnotationEvent( + annotation_type=annotation_type, + allele_id=allele_id, + variant_id=variant_id, + disposition=disposition, + reason=reason, + job_run_id=job_run.id, + **kw, + ) + session.add(event) + session.commit() + session.refresh(event) + return event + + +def _view_rows(session, **filters): + stmt = select(CurrentAnnotationEventView) + for col, val in filters.items(): + stmt = stmt.where(getattr(CurrentAnnotationEventView, col) == val) + return list(session.scalars(stmt).all()) + + +def test_returns_only_latest_event_per_subject_and_type(session, job_run): + allele = _allele(session, "view-latest") + _event( + session, job_run, AnnotationType.GNOMAD_ALLELE_FREQUENCY, Disposition.ABSENT, "no_record", allele_id=allele.id + ) + latest = _event( + session, job_run, AnnotationType.GNOMAD_ALLELE_FREQUENCY, Disposition.PRESENT, "created", allele_id=allele.id + ) + + rows = _view_rows(session, allele_id=allele.id, annotation_type=AnnotationType.GNOMAD_ALLELE_FREQUENCY.value) + + assert [r.id for r in rows] == [latest.id] + assert rows[0].disposition == Disposition.PRESENT + + +def test_clinvar_is_multi_live_one_row_per_release(session, job_run): + allele = _allele(session, "view-clinvar") + e_2025 = _event( + session, + job_run, + AnnotationType.CLINVAR_CONTROL, + Disposition.PRESENT, + "created", + allele_id=allele.id, + source_version="01_2025", + ) + _event( + session, + job_run, + AnnotationType.CLINVAR_CONTROL, + Disposition.PRESENT, + "created", + allele_id=allele.id, + source_version="01_2026", + ) + e_2026_latest = _event( + session, + job_run, + AnnotationType.CLINVAR_CONTROL, + Disposition.PRESENT, + "superseded", + allele_id=allele.id, + source_version="01_2026", + ) + + rows = _view_rows(session, allele_id=allele.id, annotation_type=AnnotationType.CLINVAR_CONTROL.value) + + by_version = {r.source_version: r.id for r in rows} + assert by_version == {"01_2025": e_2025.id, "01_2026": e_2026_latest.id} + + +def test_non_clinvar_collapses_across_versions(session, job_run): + """gnomAD/VEP supersede to a single current state: a re-fetch at a new source_version yields one + row, not one per version (the CASE folds source_version in only for ClinVar).""" + allele = _allele(session, "view-gnomad-version") + _event( + session, + job_run, + AnnotationType.GNOMAD_ALLELE_FREQUENCY, + Disposition.PRESENT, + "created", + allele_id=allele.id, + source_version="4.0.0", + ) + latest = _event( + session, + job_run, + AnnotationType.GNOMAD_ALLELE_FREQUENCY, + Disposition.PRESENT, + "reconfirmed", + allele_id=allele.id, + source_version="4.1.0", + ) + + rows = _view_rows(session, allele_id=allele.id, annotation_type=AnnotationType.GNOMAD_ALLELE_FREQUENCY.value) + + assert [r.id for r in rows] == [latest.id] + assert rows[0].source_version == "4.1.0" + + +def test_variant_subject_events_present_and_keyed_by_variant(session, setup_lib_db_with_variant, job_run): + variant = setup_lib_db_with_variant + _event(session, job_run, AnnotationType.VRS_MAPPING, Disposition.FAILED, "failed", variant_id=variant.id) + latest = _event(session, job_run, AnnotationType.VRS_MAPPING, Disposition.PRESENT, "mapped", variant_id=variant.id) + + rows = _view_rows(session, variant_id=variant.id, annotation_type=AnnotationType.VRS_MAPPING.value) + + assert [r.id for r in rows] == [latest.id] + assert rows[0].allele_id is None + + +def test_distinct_subjects_yield_distinct_rows(session, job_run): + a1 = _allele(session, "view-a1") + a2 = _allele(session, "view-a2") + _event(session, job_run, AnnotationType.CLINGEN_ALLELE_ID, Disposition.PRESENT, "created", allele_id=a1.id) + _event(session, job_run, AnnotationType.CLINGEN_ALLELE_ID, Disposition.NOT_APPLICABLE, "no_hgvs", allele_id=a2.id) + + rows = _view_rows(session, annotation_type=AnnotationType.CLINGEN_ALLELE_ID.value) + + assert {r.allele_id for r in rows} == {a1.id, a2.id} From 5773aa5a6df132d5d1b0ed3f539a22da435bbf17 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 30 Jun 2026 08:30:07 -0700 Subject: [PATCH 035/252] refactor(alleles): collapse to one work-unit per allele group_alleles_for_annotation now returns one job-specific payload per allele instead of an AlleleAnnotationGroup. Drops the authoritative_variant_ids bandaid seam (and is_authoritative from ScoreSetAlleleRow): annotation is an allele-level fact, and with the AnnotationEvent log the per-variant status fan-out goes away. --- src/mavedb/lib/clingen/alleles.py | 63 ++++++++++--------------------- tests/lib/clingen/test_alleles.py | 44 +++++++-------------- 2 files changed, 34 insertions(+), 73 deletions(-) diff --git a/src/mavedb/lib/clingen/alleles.py b/src/mavedb/lib/clingen/alleles.py index a2a962765..58cb6d7d0 100644 --- a/src/mavedb/lib/clingen/alleles.py +++ b/src/mavedb/lib/clingen/alleles.py @@ -5,8 +5,7 @@ score set. A single definition here prevents the two jobs from drifting apart. """ -from dataclasses import dataclass, field -from typing import Callable, Generic, Iterable, NamedTuple, Optional, TypeVar +from typing import Callable, Iterable, NamedTuple, Optional, TypeVar from sqlalchemy import select from sqlalchemy.orm import Session @@ -20,11 +19,9 @@ class ScoreSetAlleleRow(NamedTuple): - """One (allele, variant) link for a score set. An allele shared by multiple variants - appears once per variant so callers can fan annotation statuses out correctly. - - ``is_authoritative`` is a property of the link, not the allele: the same VRS allele can be - the authoritative measurement for one variant and an RT-derived equivalence for another. + """One (allele, variant) link for a score set. An allele shared by multiple variants appears once + per variant; :func:`group_alleles_for_annotation` collapses those duplicates into one work-unit + per allele. ``hgvs_g``/``hgvs_c``/``hgvs_p`` are allele-level (stable by construction), carried here so the VEP job can build its HGVS payload without a second query. They are optional with a ``None`` @@ -35,7 +32,6 @@ class ScoreSetAlleleRow(NamedTuple): post_mapped: dict | None clingen_allele_id: str | None variant_id: int - is_authoritative: bool hgvs_g: str | None = None hgvs_c: str | None = None hgvs_p: str | None = None @@ -57,7 +53,6 @@ def get_alleles_for_score_set(db: Session, score_set_id: int) -> list[ScoreSetAl Allele.post_mapped, Allele.clingen_allele_id, Variant.id.label("variant_id"), - MappingRecordAllele.is_authoritative, Allele.hgvs_g, Allele.hgvs_c, Allele.hgvs_p, @@ -72,41 +67,21 @@ def get_alleles_for_score_set(db: Session, score_set_id: int) -> list[ScoreSetAl ).all() return [ - ScoreSetAlleleRow( - r.id, r.post_mapped, r.clingen_allele_id, r.variant_id, r.is_authoritative, r.hgvs_g, r.hgvs_c, r.hgvs_p - ) + ScoreSetAlleleRow(r.id, r.post_mapped, r.clingen_allele_id, r.variant_id, r.hgvs_g, r.hgvs_c, r.hgvs_p) for r in rows ] -@dataclass -class AlleleAnnotationGroup(Generic[P]): - """One allele's annotation work-unit: a job-specific ``payload`` plus the variants that receive a - per-variant annotation-status (VAS) row for it. - - ``authoritative_variant_ids`` is the interim **bandaid seam** (see - docs/design/allele-annotation-status.md). An annotation is an allele-level fact, but VAS is - per-variant; fanning it to every variant the allele backs would write multiple ``current`` rows - for one variant. So status is fanned only to the variants for which this allele is the - *authoritative* measurement. At the AnnotationEvent migration the per-variant fan-out goes away - and this list is no longer needed. The allele is still grouped (and linked/annotated at the - allele level) regardless — a purely RT-derived allele simply has an empty list here. - """ - - payload: P - authoritative_variant_ids: list[int] = field(default_factory=list) - - def group_alleles_for_annotation( rows: Iterable[ScoreSetAlleleRow], payload: Callable[[ScoreSetAlleleRow], Optional[P]], -) -> dict[int, AlleleAnnotationGroup[P]]: +) -> dict[int, P]: """Collapse the per-(allele, variant) rows from :func:`get_alleles_for_score_set` into one - work-unit per allele, keyed by ``allele_id``. + job-specific payload per allele, keyed by ``allele_id``. - The same allele recurs once per variant that links it (and can be authoritative for one variant - while RT-derived for another). This groups those rows so every annotation job shares one shape: - one entry per allele, carrying the authoritative-variant fan-out set for the bandaid. + The same allele recurs once per variant that links it, so this dedups those rows down to one + entry per allele — the shape every allele-keyed annotation job wants now that annotation events + are allele-keyed (one event per allele, never fanned per-variant). ``payload`` builds the job-specific payload from the first row seen for an allele — the CAID for gnomAD/ClinVar, the HGVS for VEP, etc. Returning ``None`` skips the allele entirely (e.g. it @@ -118,13 +93,15 @@ def group_alleles_for_annotation( is the permanent, never-reused surrogate. If annotation storage later keys on the digest, carry it on the row and store against it — the grouping contract here does not change. """ - groups: dict[int, AlleleAnnotationGroup[P]] = {} + groups: dict[int, P] = {} for row in rows: - if row.allele_id not in groups: - built = payload(row) - if built is None: - continue - groups[row.allele_id] = AlleleAnnotationGroup(payload=built) - if row.is_authoritative: - groups[row.allele_id].authoritative_variant_ids.append(row.variant_id) + if row.allele_id in groups: + continue + + built = payload(row) + if built is None: + continue + + groups[row.allele_id] = built + return groups diff --git a/tests/lib/clingen/test_alleles.py b/tests/lib/clingen/test_alleles.py index fbcfaa986..440646244 100644 --- a/tests/lib/clingen/test_alleles.py +++ b/tests/lib/clingen/test_alleles.py @@ -3,66 +3,50 @@ from mavedb.lib.clingen.alleles import ScoreSetAlleleRow, group_alleles_for_annotation -def _row(allele_id, variant_id, *, is_authoritative, caid="CA1"): +def _row(allele_id, variant_id, *, caid="CA1"): return ScoreSetAlleleRow( allele_id=allele_id, post_mapped={"type": "Allele"}, clingen_allele_id=caid, variant_id=variant_id, - is_authoritative=is_authoritative, ) -def test_collapses_rows_to_one_group_per_allele(): +def test_collapses_rows_to_one_payload_per_allele(): rows = [ - _row(1, 10, is_authoritative=True), - _row(1, 11, is_authoritative=True), - _row(2, 12, is_authoritative=True, caid="CA2"), + _row(1, 10), + _row(1, 11), + _row(2, 12, caid="CA2"), ] groups = group_alleles_for_annotation(rows, payload=lambda r: r.clingen_allele_id) - assert set(groups) == {1, 2} - assert groups[1].payload == "CA1" - assert groups[1].authoritative_variant_ids == [10, 11] - assert groups[2].authoritative_variant_ids == [12] + assert groups == {1: "CA1", 2: "CA2"} -def test_only_authoritative_links_contribute_variant_ids(): - """An allele that is authoritative for one variant and RT-derived for another fans status only to - the authoritative variant — the bandaid invariant.""" +def test_shared_allele_yields_a_single_entry(): + """An allele linked by multiple variants is deduped to one work-unit (events are allele-keyed).""" rows = [ - _row(1, 10, is_authoritative=True), - _row(1, 11, is_authoritative=False), + _row(1, 10), + _row(1, 11), ] groups = group_alleles_for_annotation(rows, payload=lambda r: r.clingen_allele_id) - assert groups[1].authoritative_variant_ids == [10] - - -def test_purely_rt_derived_allele_is_grouped_with_empty_fan_out(): - """A non-authoritative-only allele is still grouped (so it gets linked/annotated at the allele - level) but contributes no per-variant VAS row.""" - rows = [_row(1, 10, is_authoritative=False)] - - groups = group_alleles_for_annotation(rows, payload=lambda r: r.clingen_allele_id) - - assert set(groups) == {1} - assert groups[1].authoritative_variant_ids == [] + assert groups == {1: "CA1"} def test_payload_returning_none_skips_the_allele(): """Returning None from payload drops the allele entirely — replacing each job's ad-hoc 'no CAID / no HGVS -> continue' filter.""" rows = [ - _row(1, 10, is_authoritative=True, caid=None), - _row(2, 11, is_authoritative=True, caid="CA2"), + _row(1, 10, caid=None), + _row(2, 11, caid="CA2"), ] groups = group_alleles_for_annotation(rows, payload=lambda r: r.clingen_allele_id) - assert set(groups) == {2} + assert groups == {2: "CA2"} def test_empty_rows_yield_empty_grouping(): From 5d35ed095fb3f2d491cb1325f4f0369b0033949f Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 30 Jun 2026 08:30:22 -0700 Subject: [PATCH 036/252] refactor(annotation): rewrite AnnotationStatusManager to emit events AnnotationStatusManager now writes AnnotationEvent rows via record_event and derives current status from the event log instead of maintaining per-variant status rows. Resolves subjects through VARIANT_SUBJECT_TYPES/ALLELE_SUBJECT_TYPES and the Disposition vocabulary. --- src/mavedb/lib/annotation_status_manager.py | 253 ++-- tests/lib/test_annotation_status_manager.py | 1456 ++----------------- 2 files changed, 242 insertions(+), 1467 deletions(-) diff --git a/src/mavedb/lib/annotation_status_manager.py b/src/mavedb/lib/annotation_status_manager.py index 6598bfabd..4022cb4d2 100644 --- a/src/mavedb/lib/annotation_status_manager.py +++ b/src/mavedb/lib/annotation_status_manager.py @@ -1,207 +1,170 @@ -"""Manage annotation statuses for variants. - -This module provides functionality to insert and retrieve annotation statuses -for genetic variants, ensuring that only one current status exists per -(variant, annotation type, version) combination. +"""Append-only writer for the annotation event log. + +Buffers :class:`AnnotationEvent` rows and flushes them in batches. This is an +**append-only** log, not a state table: there is no ``current`` flag to +maintain and no retire-on-write step. "Current" is derived at read time +(``DISTINCT ON (subject, annotation_type) … ORDER BY id DESC``) — exposed as +the ``v_current_annotation_events`` view (``mavedb.models.annotation_event_view``). + +Each event's *subject* is either a variant or an allele, chosen by +``annotation_type``. The writer validates the subject/type pairing up front so +a mis-subjected event fails with a clear ``ValueError`` rather than a deferred +DB ``CHECK`` violation at flush. """ import logging from typing import Optional -from sqlalchemy import select, update +from sqlalchemy import select from sqlalchemy.orm import Session from sqlalchemy.sql import desc from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationFailureCategory, AnnotationStatus -from mavedb.models.variant_annotation_status import VariantAnnotationStatus +from mavedb.models.enums.disposition import Disposition +from mavedb.models.annotation_event import ALLELE_SUBJECT_TYPES, VARIANT_SUBJECT_TYPES, AnnotationEvent logger = logging.getLogger(__name__) -# Default number of pending annotations to accumulate before auto-flushing. +# Default number of pending events to accumulate before auto-flushing. DEFAULT_BATCH_SIZE = 500 class AnnotationStatusManager: - """ - Manager for handling variant annotation statuses with batched writes. + """Buffered, append-only writer for :class:`AnnotationEvent` rows. - Annotations are accumulated in memory and flushed to the database in - batches (default 500) to reduce round-trips. Callers **must** call - :meth:`flush` after the last ``add_annotation`` to persist any remainder. + Events are accumulated in memory and flushed in batches (default 500) to + reduce round-trips. Callers **must** call :meth:`flush` after the last + :meth:`record_event` to persist any remainder. """ - def __init__(self, session: Session, job_run_id: Optional[int] = None, *, batch_size: int = DEFAULT_BATCH_SIZE): + def __init__( + self, + session: Session, + job_run_id: Optional[int] = None, + *, + score_set_id: Optional[int] = None, + batch_size: int = DEFAULT_BATCH_SIZE, + ): self.session = session self.job_run_id = job_run_id + self.score_set_id = score_set_id self.batch_size = batch_size - self._pending: list[VariantAnnotationStatus] = [] - self._retirement_filters: list[dict] = [] + self._pending: list[AnnotationEvent] = [] - def add_annotation( + def record_event( self, - variant_id: int, annotation_type: AnnotationType, - status: AnnotationStatus, - version: Optional[str] = None, - failure_category: Optional[AnnotationFailureCategory] = None, - annotation_data: dict = {}, - current: bool = True, - replace_all_versions: bool = True, + *, + disposition: Disposition, + reason: str, + variant_id: Optional[int] = None, + allele_id: Optional[int] = None, + source_version: Optional[str] = None, + metadata: Optional[dict] = None, + score_set_id: Optional[int] = None, ) -> None: - """ - Stage a new annotation and schedule retirement of previous current rows. - - By default (``replace_all_versions=True``), all existing current annotations for - (variant, type) are retired regardless of version. + """Buffer one terminal observation about a variant or an allele. - When ``replace_all_versions=False``, only existing current annotations matching - (variant, type, version) are retired. + Exactly one of ``variant_id`` / ``allele_id`` must be set, and it must + match the subject the ``annotation_type`` keys on. ``reason`` is the + single "what happened" axis across all dispositions (see EventReason). - Writes are accumulated in memory and flushed to the database when - ``batch_size`` is reached. Call :meth:`flush` after the last add to - persist any remaining annotations. - - NOTE: - This method does not commit the session. The caller is responsible - for persisting changes (e.g., via ``session.commit()``). + Writes are accumulated in memory and flushed when ``batch_size`` is + reached. Call :meth:`flush` after the last call to persist the + remainder. Does not commit — the caller owns the transaction. """ - self._retirement_filters.append( - { - "variant_id": variant_id, - "annotation_type": annotation_type, - "replace_all_versions": replace_all_versions, - "version": version, - } - ) + self._validate_subject(annotation_type, variant_id, allele_id) self._pending.append( - VariantAnnotationStatus( - variant_id=variant_id, + AnnotationEvent( annotation_type=annotation_type, - status=status, - version=version, - failure_category=failure_category, - current=current, + variant_id=variant_id, + allele_id=allele_id, + disposition=disposition, + reason=reason, + source_version=source_version, + event_metadata=metadata, job_run_id=self.job_run_id, - **annotation_data, + score_set_id=score_set_id if score_set_id is not None else self.score_set_id, ) # type: ignore[call-arg] ) if len(self._pending) >= self.batch_size: self.flush() - def flush(self) -> None: - """Flush all pending annotations to the database. + @staticmethod + def _validate_subject(annotation_type: AnnotationType, variant_id: Optional[int], allele_id: Optional[int]) -> None: + if (variant_id is None) == (allele_id is None): + raise ValueError("Exactly one of variant_id or allele_id must be set") - Retires old ``current=True`` rows in bulk, then inserts all pending - new rows in a single ``add_all`` + ``flush``. This replaces the - previous pattern of 2 flushes per ``add_annotation`` call. - """ + type_value = annotation_type.value if isinstance(annotation_type, AnnotationType) else annotation_type + if variant_id is not None and type_value not in VARIANT_SUBJECT_TYPES: + raise ValueError(f"annotation_type {type_value!r} is allele-subject; pass allele_id, not variant_id") + if allele_id is not None and type_value not in ALLELE_SUBJECT_TYPES: + raise ValueError(f"annotation_type {type_value!r} is variant-subject; pass variant_id, not allele_id") + + def flush(self) -> None: + """Insert all pending events in a single ``add_all`` + ``flush``.""" if not self._pending: return - self._retire_existing() self.session.add_all(self._pending) self.session.flush() - logger.debug(f"Flushed {len(self._pending)} annotation statuses") + logger.debug(f"Flushed {len(self._pending)} variant events") self._pending.clear() - self._retirement_filters.clear() - - def _retire_existing(self) -> None: - """Bulk-retire existing current annotations for all pending writes. - - Groups retirement filters by (annotation_type, replace_all_versions, version) - and issues one UPDATE per group, minimizing round-trips. - """ - # Group filters to minimize UPDATE statements. - # Key: (annotation_type, replace_all_versions, version) -> list of variant_ids - groups: dict[tuple, list[int]] = {} - for f in self._retirement_filters: - key = (f["annotation_type"], f["replace_all_versions"], f["version"]) - groups.setdefault(key, []).append(f["variant_id"]) - - for (annotation_type, replace_all_versions, version), variant_ids in groups.items(): - conditions = [ - VariantAnnotationStatus.variant_id.in_(variant_ids), - VariantAnnotationStatus.annotation_type == annotation_type, - VariantAnnotationStatus.current.is_(True), - ] - if not replace_all_versions: - conditions.append(VariantAnnotationStatus.version == version) - - stmt = update(VariantAnnotationStatus).where(*conditions).values(current=False) - self.session.execute(stmt) def get_current_annotation( - self, variant_id: int, annotation_type: AnnotationType, version: Optional[str] = None - ) -> Optional[VariantAnnotationStatus]: - """ - Retrieve the current annotation for a given variant/type/version. - - Flushes pending annotations first to ensure the result is up to date. - """ - self.flush() - - stmt = select(VariantAnnotationStatus).where( - VariantAnnotationStatus.variant_id == variant_id, - VariantAnnotationStatus.annotation_type == annotation_type, - VariantAnnotationStatus.current.is_(True), - ) - - if version is not None: - stmt = stmt.where(VariantAnnotationStatus.version == version) - - result = self.session.execute(stmt) - return result.scalar_one_or_none() - - def get_annotation_history( self, - variant_id: int, annotation_type: AnnotationType, - version: Optional[str] = None, - ) -> list[VariantAnnotationStatus]: - """ - Return the full annotation timeline for a variant/type, newest first. - - Includes both current and retired rows — useful for debugging and - support investigations. + *, + variant_id: Optional[int] = None, + allele_id: Optional[int] = None, + source_version: Optional[str] = None, + ) -> Optional[AnnotationEvent]: + """Latest event for a single ``(subject, annotation_type)`` key. + + Current status is the newest event by ``id`` — there is no ``current`` + flag to filter on. Flushes pending events first so the result reflects + buffered writes. """ self.flush() + self._validate_subject(annotation_type, variant_id, allele_id) - stmt = ( - select(VariantAnnotationStatus) - .where( - VariantAnnotationStatus.variant_id == variant_id, - VariantAnnotationStatus.annotation_type == annotation_type, - ) - .order_by(desc(VariantAnnotationStatus.id)) - ) - - if version is not None: - stmt = stmt.where(VariantAnnotationStatus.version == version) + stmt = select(AnnotationEvent).where(AnnotationEvent.annotation_type == annotation_type) + if variant_id is not None: + stmt = stmt.where(AnnotationEvent.variant_id == variant_id) + else: + stmt = stmt.where(AnnotationEvent.allele_id == allele_id) + if source_version is not None: + stmt = stmt.where(AnnotationEvent.source_version == source_version) - return list(self.session.scalars(stmt).all()) + stmt = stmt.order_by(desc(AnnotationEvent.id)).limit(1) + return self.session.scalars(stmt).first() - def get_all_current_annotations( + def get_event_history( self, - variant_id: int, - ) -> list[VariantAnnotationStatus]: - """ - Return all current annotations for a variant, across all types and versions. - - Useful for a quick overview of what annotations are active for a given variant. + annotation_type: AnnotationType, + *, + variant_id: Optional[int] = None, + allele_id: Optional[int] = None, + source_version: Optional[str] = None, + ) -> list[AnnotationEvent]: + """Full event timeline for a ``(subject, annotation_type)`` key, newest first. + + The append-only log retains every observation — skips, reconfirms, + no-ops — so this is the complete audit trail, not just the current row. """ self.flush() + self._validate_subject(annotation_type, variant_id, allele_id) - stmt = ( - select(VariantAnnotationStatus) - .where( - VariantAnnotationStatus.variant_id == variant_id, - VariantAnnotationStatus.current.is_(True), - ) - .order_by(VariantAnnotationStatus.annotation_type, VariantAnnotationStatus.version) - ) + stmt = select(AnnotationEvent).where(AnnotationEvent.annotation_type == annotation_type) + if variant_id is not None: + stmt = stmt.where(AnnotationEvent.variant_id == variant_id) + else: + stmt = stmt.where(AnnotationEvent.allele_id == allele_id) + if source_version is not None: + stmt = stmt.where(AnnotationEvent.source_version == source_version) + stmt = stmt.order_by(desc(AnnotationEvent.id)) return list(self.session.scalars(stmt).all()) diff --git a/tests/lib/test_annotation_status_manager.py b/tests/lib/test_annotation_status_manager.py index 52771b6bf..b616fbf34 100644 --- a/tests/lib/test_annotation_status_manager.py +++ b/tests/lib/test_annotation_status_manager.py @@ -5,1362 +5,174 @@ pytest.importorskip("psycopg2") from mavedb.lib.annotation_status_manager import AnnotationStatusManager +from mavedb.models.allele import Allele from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationStatus -from mavedb.models.variant import Variant +from mavedb.models.enums.disposition import Disposition +from mavedb.models.enums.event_reason import EventReason @pytest.fixture def annotation_status_manager(session, job_run): - """Fixture to provide an AnnotationStatusManager instance.""" return AnnotationStatusManager(session, job_run_id=job_run.id) @pytest.fixture -def existing_annotation_status(session, annotation_status_manager, setup_lib_db_with_variant): - """Fixture to create an existing annotation status in the database.""" - - # Add initial annotation - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - annotation_status_manager.flush() +def allele(session): + allele = Allele(vrs_digest="asm-test-allele-digest", level="genomic") + session.add(allele) session.commit() + session.refresh(allele) + return allele - annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - - assert annotation.id is not None - assert annotation.current is True - - return annotation - - -@pytest.fixture -def existing_unversioned_annotation_status(session, annotation_status_manager, setup_lib_db_with_variant): - """Fixture to create an existing annotation status in the database.""" - - # Add initial annotation - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version=None, - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - ) - - assert annotation.id is not None - assert annotation.current is True - - return annotation - - -@pytest.mark.unit -class TestAnnotationStatusManagerCreateAnnotationUnit: - """Unit tests for AnnotationStatusManager.add_annotation method.""" - - @pytest.mark.parametrize( - "annotation_type", - AnnotationType._member_map_.values(), - ) - @pytest.mark.parametrize( - "status", - AnnotationStatus._member_map_.values(), - ) - def test_add_annotation_creates_entry_with_annotation_type_version_status( - self, session, annotation_status_manager, annotation_type, status, setup_lib_db_with_variant - ): - """Test that adding an annotation creates a new entry with correct type and version.""" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=annotation_type, - version="v1.0", - annotation_data={}, - current=True, - status=status, - ) - annotation_status_manager.flush() - session.commit() - - annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=annotation_type, - version="v1.0", - ) - - assert annotation is not None - assert annotation.annotation_type == annotation_type - assert annotation.status == status - assert annotation.version == "v1.0" - - def test_add_annotation_stores_job_run_id( - self, session, annotation_status_manager, job_run, setup_lib_db_with_variant - ): - """Test that every annotation is created with the job_run_id from the manager.""" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - status=AnnotationStatus.SUCCESS, - version="v1.0", - annotation_data={}, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1.0", - ) - - assert annotation is not None - assert annotation.job_run_id == job_run.id - - def test_add_annotation_persists_annotation_data( - self, session, annotation_status_manager, setup_lib_db_with_variant - ): - """Test that adding an annotation persists the provided annotation data.""" - annotation_data = { - "annotation_metadata": {"some_key": "some_value"}, - "error_message": None, - } - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - status=AnnotationStatus.SUCCESS, - version="v1.0", - failure_category=None, - annotation_data=annotation_data, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1.0", - ) - - assert annotation is not None - assert annotation.failure_category is None - for key, value in annotation_data.items(): - assert getattr(annotation, key) == value - - def test_add_annotation_creates_entry_and_marks_previous_not_current( - self, session, job_run, existing_annotation_status, setup_lib_db_with_variant - ): - """Test that adding an annotation creates a new entry and marks previous ones as not current.""" - manager = AnnotationStatusManager(session, job_run_id=job_run.id) - - # Add second annotation for same (variant, type, version) - manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.FAILED, - current=True, - ) - manager.flush() - session.commit() - - annotation = manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - - assert annotation is not None - assert annotation.id is not None - assert annotation.current is True - - # Refresh first annotation from DB - session.refresh(existing_annotation_status) - assert existing_annotation_status.current is False - - def test_add_annotation_with_different_version_keeps_previous_current( - self, session, job_run, existing_annotation_status, setup_lib_db_with_variant - ): - """Test that adding an annotation with a different version keeps previous current.""" - manager = AnnotationStatusManager(session, job_run_id=job_run.id) - - # Add second annotation for same (variant, type) but different version - manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v2", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - replace_all_versions=False, - ) - manager.flush() - session.commit() - - annotation = manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v2", - ) - - assert annotation is not None - assert annotation.id is not None - assert annotation.current is True - - # Refresh first annotation from DB - session.refresh(existing_annotation_status) - assert existing_annotation_status.current is True - - def test_add_annotation_with_different_type_keeps_previous_current( - self, session, job_run, existing_annotation_status, setup_lib_db_with_variant - ): - """Test that adding an annotation with a different type keeps previous current.""" - manager = AnnotationStatusManager(session, job_run_id=job_run.id) - - # Add second annotation for same variant but different type - manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINGEN_ALLELE_ID, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - manager.flush() - session.commit() - - annotation = manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINGEN_ALLELE_ID, - version="v1", - ) - - assert annotation is not None - assert annotation.id is not None - assert annotation.current is True - - # Refresh first annotation from DB - session.refresh(existing_annotation_status) - assert existing_annotation_status.current is True - - def test_add_annotation_without_version(self, session, annotation_status_manager, setup_lib_db_with_variant): - """Test that adding an annotation without specifying version works correctly.""" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, - version=None, - annotation_data={}, - status=AnnotationStatus.SKIPPED, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, - ) - - assert annotation is not None - assert annotation.id is not None - assert annotation.version is None - assert annotation.current is True - - def test_add_annotation_multiple_without_version_marks_previous_not_current( - self, session, annotation_status_manager, existing_unversioned_annotation_status, setup_lib_db_with_variant - ): - """Test that adding multiple annotations without version marks previous ones as not current.""" - - # Add second annotation without version - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version=None, - annotation_data={}, - status=AnnotationStatus.FAILED, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - second_annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - ) - - assert second_annotation is not None - assert second_annotation.id is not None - assert second_annotation.current is True - - # Refresh first annotation from DB - session.refresh(existing_unversioned_annotation_status) - assert existing_unversioned_annotation_status.current is False - - def test_add_annotation_different_type_without_version_keeps_previous_current( - self, session, annotation_status_manager, existing_unversioned_annotation_status, setup_lib_db_with_variant - ): - """Test that adding an annotation of different type without version keeps previous current.""" - - # Add second annotation of different type without version - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINGEN_ALLELE_ID, - version=None, - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - second_annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINGEN_ALLELE_ID, - ) - - assert second_annotation is not None - assert second_annotation.id is not None - assert second_annotation.current is True - - # Refresh first annotation from DB - session.refresh(existing_unversioned_annotation_status) - assert existing_unversioned_annotation_status.current is True - - def test_add_annotation_multiple_variants_independent_current_flags( - self, session, annotation_status_manager, setup_lib_db_with_score_set - ): - """Test that adding annotations for different variants maintains independent current flags.""" - - variant1 = Variant(score_set_id=1, hgvs_nt="NM_000000.1:c.1A>G", hgvs_pro="NP_000000.1:p.Met1Val", data={}) - variant2 = Variant(score_set_id=1, hgvs_nt="NM_000000.1:c.2A>T", hgvs_pro="NP_000000.1:p.Met2Val", data={}) - session.add_all([variant1, variant2]) - session.commit() - session.refresh(variant1) - session.refresh(variant2) - - # Add annotation for variant 1 - annotation_status_manager.add_annotation( - variant_id=variant1.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - - # Add annotation for variant 2 - annotation_status_manager.add_annotation( - variant_id=variant2.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - annotation1 = annotation_status_manager.get_current_annotation( - variant_id=variant1.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - annotation2 = annotation_status_manager.get_current_annotation( - variant_id=variant2.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - - assert annotation1 is not None - assert annotation1.id is not None - assert annotation1.current is True - - assert annotation2 is not None - assert annotation2.id is not None - assert annotation2.current is True - - -class TestAnnotationStatusManagerGetCurrentAnnotationUnit: - """Unit tests for AnnotationStatusManager.get_current_annotation method.""" - - def test_get_current_annotation_returns_none_when_no_entry( - self, annotation_status_manager, setup_lib_db_with_variant - ): - """Test that getting current annotation returns None when no entry exists.""" - annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - assert annotation is None - - def test_get_current_annotation_returns_correct_entry( - self, session, annotation_status_manager, existing_annotation_status, setup_lib_db_with_variant - ): - """Test that getting current annotation returns the correct entry.""" - annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - assert annotation.id == existing_annotation_status.id - assert annotation.current is True - - def test_get_current_annotation_returns_none_for_non_current( - self, session, annotation_status_manager, existing_annotation_status, setup_lib_db_with_variant - ): - """Test that getting current annotation returns None when the entry is not current.""" - # Mark existing annotation as not current - existing_annotation_status.current = False - session.commit() - - annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - assert annotation is None - - def test_get_current_annotation_with_different_version_returns_none( - self, session, annotation_status_manager, existing_annotation_status, setup_lib_db_with_variant - ): - """Test that getting current annotation with different version returns None.""" - annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v2", - ) - assert annotation is None - - def test_get_current_annotation_with_different_type_returns_none( - self, session, annotation_status_manager, existing_annotation_status, setup_lib_db_with_variant - ): - """Test that getting current annotation with different type returns None.""" - annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINGEN_ALLELE_ID, - version="v1", - ) - assert annotation is None - - def test_get_current_annotation_without_version_returns_correct_entry( - self, session, annotation_status_manager, existing_unversioned_annotation_status, setup_lib_db_with_variant - ): - """Test that getting current annotation without version returns the correct entry.""" - annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version=None, - ) - assert annotation.id == existing_unversioned_annotation_status.id - assert annotation.current is True - - -class TestAnnotationStatusManagerIntegration: - """Integration tests for AnnotationStatusManager methods.""" - - def test_add_and_get_current_annotation_work_together( - self, session, annotation_status_manager, setup_lib_db_with_variant - ): - """Test that adding and getting current annotation work together correctly.""" - # Add annotation - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - # Get current annotation - retrieved_annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - - assert retrieved_annotation is not None - assert retrieved_annotation.current is True - assert retrieved_annotation.status == AnnotationStatus.SUCCESS - - @pytest.mark.parametrize( - "version", - ["v1.0", "v2.0", None], - ) - def test_add_multiple_and_get_current_returns_latest( - self, session, annotation_status_manager, version, setup_lib_db_with_variant - ): - """Test that adding multiple annotations and getting current returns the latest one.""" - # Add first annotation - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version=version, - annotation_data={}, - status=AnnotationStatus.FAILED, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - # Add second annotation - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version=version, - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - # Get current annotation - retrieved_annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version=version, - ) - - assert retrieved_annotation is not None - assert retrieved_annotation.current is True - assert retrieved_annotation.version == version - assert retrieved_annotation.status == AnnotationStatus.SUCCESS - - @pytest.mark.parametrize( - "version", - ["v1.0", "v2.0", None], - ) - def test_add_annotations_for_different_variants_and_get_current_independent( - self, session, annotation_status_manager, version, setup_lib_db_with_score_set - ): - """Test that adding annotations for different variants and getting current works independently.""" - - variant1 = Variant(score_set_id=1, hgvs_nt="NM_000000.1:c.1A>G", hgvs_pro="NP_000000.1:p.Met1Val", data={}) - variant2 = Variant(score_set_id=1, hgvs_nt="NM_000000.1:c.2A>T", hgvs_pro="NP_000000.1:p.Met2Val", data={}) - session.add_all([variant1, variant2]) - session.commit() - session.refresh(variant1) - session.refresh(variant2) - - # Add annotation for variant 1 - annotation_status_manager.add_annotation( - variant_id=variant1.id, - annotation_type=AnnotationType.VRS_MAPPING, - version=version, - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - - # Add annotation for variant 2 - annotation_status_manager.add_annotation( - variant_id=variant2.id, - annotation_type=AnnotationType.VRS_MAPPING, - version=version, - annotation_data={}, - status=AnnotationStatus.FAILED, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - # Get current annotation for variant 1 - retrieved_annotation1 = annotation_status_manager.get_current_annotation( - variant_id=variant1.id, - annotation_type=AnnotationType.VRS_MAPPING, - version=version, - ) - - assert retrieved_annotation1 is not None - assert retrieved_annotation1.current is True - assert retrieved_annotation1.status == AnnotationStatus.SUCCESS - assert retrieved_annotation1.version == version - - # Get current annotation for variant 2 - retrieved_annotation2 = annotation_status_manager.get_current_annotation( - variant_id=variant2.id, - annotation_type=AnnotationType.VRS_MAPPING, - version=version, - ) - - assert retrieved_annotation2 is not None - assert retrieved_annotation2.current is True - assert retrieved_annotation2.status == AnnotationStatus.FAILED - assert retrieved_annotation2.version == version - - -@pytest.mark.unit -class TestAnnotationStatusManagerReplaceAllVersionsUnit: - """Unit tests for the replace_all_versions parameter of AnnotationStatusManager.add_annotation.""" - - def test_replace_all_versions_false_keeps_different_version_current( - self, session, annotation_status_manager, existing_annotation_status, setup_lib_db_with_variant - ): - """Default behavior: a new annotation only retires the same version, not others.""" - # existing_annotation_status is version "v1", current=True - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v2", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - replace_all_versions=False, - ) - annotation_status_manager.flush() - session.commit() - - new_annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v2", - ) - assert new_annotation is not None - assert new_annotation.current is True - - session.refresh(existing_annotation_status) - assert existing_annotation_status.current is True - - def test_replace_all_versions_true_retires_all_versions( - self, session, annotation_status_manager, setup_lib_db_with_variant - ): - """replace_all_versions=True retires all current records for (variant, type) regardless of version.""" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - replace_all_versions=False, - ) - annotation_status_manager.flush() - - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v2", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - replace_all_versions=False, - ) - annotation_status_manager.flush() - session.commit() - - # Both v1 and v2 are current at this point (replace_all_versions=False) - v1 = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - v2 = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v2", - ) - assert v1 is not None and v1.current is True - assert v2 is not None and v2.current is True - # Now add v3 with replace_all_versions=True — should retire both v1 and v2 - annotation_status_manager.add_annotation( +class TestRecordEvent: + def test_records_variant_subject_event(self, session, annotation_status_manager, setup_lib_db_with_variant): + annotation_status_manager.record_event( + AnnotationType.LDH_SUBMISSION, variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v3", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - replace_all_versions=True, + disposition=Disposition.PRESENT, + reason=EventReason.SUBMITTED, ) annotation_status_manager.flush() session.commit() - session.refresh(v1) - session.refresh(v2) - assert v1.current is False - assert v2.current is False - - v3 = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v3", + event = annotation_status_manager.get_current_annotation( + AnnotationType.LDH_SUBMISSION, variant_id=setup_lib_db_with_variant.id ) - assert v3 is not None and v3.current is True + assert event is not None + assert event.disposition == Disposition.PRESENT + assert event.allele_id is None + # No current flag exists on the event log. + assert not hasattr(event, "current") - def test_replace_all_versions_true_only_affects_matching_type( - self, session, annotation_status_manager, setup_lib_db_with_variant - ): - """replace_all_versions=True only retires records for the same annotation_type.""" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINVAR_CONTROL, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, + def test_records_allele_subject_event_with_metadata(self, session, annotation_status_manager, allele): + annotation_status_manager.record_event( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, + allele_id=allele.id, + disposition=Disposition.PRESENT, + reason=EventReason.CREATED, + source_version="4.1.0", + metadata={"gnomad_variant_id": "1-55051215-G-A"}, ) annotation_status_manager.flush() session.commit() - vrs = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - clinvar = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINVAR_CONTROL, - version="v1", - ) - - # replace VRS_MAPPING only - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v2", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - replace_all_versions=True, + event = annotation_status_manager.get_current_annotation( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, allele_id=allele.id ) - annotation_status_manager.flush() - session.commit() - - session.refresh(vrs) - session.refresh(clinvar) - assert vrs.current is False - assert clinvar.current is True + assert event.event_metadata == {"gnomad_variant_id": "1-55051215-G-A"} + assert event.source_version == "4.1.0" + assert event.variant_id is None - new_vrs = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v2", - ) - assert new_vrs is not None and new_vrs.current is True - def test_replace_all_versions_true_only_affects_matching_variant( - self, session, annotation_status_manager, setup_lib_db_with_score_set - ): - """replace_all_versions=True only retires records for the same variant_id.""" - variant1 = Variant(score_set_id=1, hgvs_nt="NM_000000.1:c.1A>G", hgvs_pro="NP_000000.1:p.Met1Val", data={}) - variant2 = Variant(score_set_id=1, hgvs_nt="NM_000000.1:c.2A>T", hgvs_pro="NP_000000.1:p.Met2Val", data={}) - session.add_all([variant1, variant2]) - session.commit() - session.refresh(variant1) - session.refresh(variant2) - - annotation_status_manager.add_annotation( - variant_id=variant1.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - annotation_status_manager.add_annotation( - variant_id=variant2.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - ann1 = annotation_status_manager.get_current_annotation( - variant_id=variant1.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - ann2 = annotation_status_manager.get_current_annotation( - variant_id=variant2.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - - # replace variant1 only - annotation_status_manager.add_annotation( - variant_id=variant1.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v2", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - replace_all_versions=True, - ) - annotation_status_manager.flush() - session.commit() - - session.refresh(ann1) - session.refresh(ann2) - assert ann1.current is False - assert ann2.current is True # untouched - - new_ann1 = annotation_status_manager.get_current_annotation( - variant_id=variant1.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v2", - ) - assert new_ann1 is not None and new_ann1.current is True - - def test_replace_all_versions_true_same_version_also_retired( - self, session, annotation_status_manager, existing_annotation_status, setup_lib_db_with_variant - ): - """replace_all_versions=True retires a same-version record just as replace_all_versions=False would.""" - # existing_annotation_status is version "v1" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.FAILED, - current=True, - replace_all_versions=True, - ) +class TestAppendOnlyLatestWins: + def test_latest_event_by_id_is_current(self, session, annotation_status_manager, allele): + for reason in (EventReason.CREATED, EventReason.RECONFIRMED, EventReason.SKIPPED): + annotation_status_manager.record_event( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, + allele_id=allele.id, + disposition=Disposition.PRESENT, + reason=reason, + source_version="4.1.0", + ) annotation_status_manager.flush() session.commit() - session.refresh(existing_annotation_status) - assert existing_annotation_status.current is False - - new_annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", + # All three events persist (append-only — no retire of prior rows). + history = annotation_status_manager.get_event_history( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, allele_id=allele.id ) - assert new_annotation is not None - assert new_annotation.current is True - assert new_annotation.status == AnnotationStatus.FAILED - - -@pytest.mark.unit -class TestAnnotationStatusManagerBatchingUnit: - """Unit tests for batching and flush behavior.""" - - def test_flush_noop_when_empty(self, annotation_status_manager): - """flush() with no pending annotations does nothing and does not error.""" - annotation_status_manager.flush() # should not raise - - def test_auto_flush_at_batch_size(self, session, setup_lib_db_with_score_set): - """Annotations are auto-flushed to the DB when batch_size is reached.""" - variants = [ - Variant(score_set_id=1, hgvs_nt=f"NM_000000.1:c.{i}A>G", hgvs_pro=f"NP_000000.1:p.Met{i}Val", data={}) - for i in range(3) + assert [e.reason for e in history] == [ # newest first + EventReason.SKIPPED, + EventReason.RECONFIRMED, + EventReason.CREATED, ] - session.add_all(variants) - session.commit() - for v in variants: - session.refresh(v) - - manager = AnnotationStatusManager(session, batch_size=2) - - # Add first — stays pending (below threshold) - manager.add_annotation( - variant_id=variants[0].id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - assert len(manager._pending) == 1 - - # Add second — triggers auto-flush (reaches batch_size=2) - manager.add_annotation( - variant_id=variants[1].id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - assert len(manager._pending) == 0 # flushed - - # Verify the auto-flushed rows are visible in the DB - ann = manager.get_current_annotation( - variant_id=variants[0].id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - assert ann is not None and ann.current is True - - # Add a third — stays pending (below threshold again) - manager.add_annotation( - variant_id=variants[2].id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - assert len(manager._pending) == 1 - - # Explicit flush persists the remainder - manager.flush() - assert len(manager._pending) == 0 - - ann3 = manager.get_current_annotation( - variant_id=variants[2].id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - assert ann3 is not None and ann3.current is True - - def test_get_current_annotation_auto_flushes_pending( - self, session, annotation_status_manager, setup_lib_db_with_variant - ): - """get_current_annotation() flushes pending writes before querying.""" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - # No explicit flush — get_current_annotation should auto-flush - annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - assert annotation is not None - assert annotation.current is True - assert len(annotation_status_manager._pending) == 0 - def test_flush_clears_internal_buffers(self, session, annotation_status_manager, setup_lib_db_with_variant): - """flush() clears both _pending and _retirement_filters.""" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) + # Current = newest by id. + current = annotation_status_manager.get_current_annotation( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, allele_id=allele.id + ) + assert current.reason == EventReason.SKIPPED + + def test_source_version_scopes_current(self, session, annotation_status_manager, allele): + annotation_status_manager.record_event( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, + allele_id=allele.id, + disposition=Disposition.PRESENT, + reason=EventReason.CREATED, + source_version="4.0.0", + ) + annotation_status_manager.record_event( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, + allele_id=allele.id, + disposition=Disposition.ABSENT, + reason=EventReason.NO_RECORD, + source_version="4.1.0", + ) + annotation_status_manager.flush() + session.commit() + + v40 = annotation_status_manager.get_current_annotation( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, allele_id=allele.id, source_version="4.0.0" + ) + assert v40.disposition == Disposition.PRESENT + + +class TestSubjectValidation: + def test_variant_subject_type_with_allele_id_raises(self, annotation_status_manager, allele): + with pytest.raises(ValueError, match="variant-subject"): + annotation_status_manager.record_event( + AnnotationType.VRS_MAPPING, + allele_id=allele.id, + disposition=Disposition.PRESENT, + reason=EventReason.CREATED, + ) + + def test_allele_subject_type_with_variant_id_raises(self, annotation_status_manager, setup_lib_db_with_variant): + with pytest.raises(ValueError, match="allele-subject"): + annotation_status_manager.record_event( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, + variant_id=setup_lib_db_with_variant.id, + disposition=Disposition.PRESENT, + reason=EventReason.CREATED, + ) + + def test_neither_subject_raises(self, annotation_status_manager): + with pytest.raises(ValueError, match="Exactly one"): + annotation_status_manager.record_event( + AnnotationType.VRS_MAPPING, + disposition=Disposition.PRESENT, + reason=EventReason.CREATED, + ) + + def test_both_subjects_raises(self, annotation_status_manager, setup_lib_db_with_variant, allele): + with pytest.raises(ValueError, match="Exactly one"): + annotation_status_manager.record_event( + AnnotationType.VRS_MAPPING, + variant_id=setup_lib_db_with_variant.id, + allele_id=allele.id, + disposition=Disposition.PRESENT, + reason=EventReason.CREATED, + ) + + +class TestBatching: + def test_auto_flush_at_batch_size(self, session, job_run, setup_lib_db_with_variant, annotation_status_manager): + annotation_status_manager.batch_size = 2 + annotation_status_manager.record_event( + AnnotationType.LDH_SUBMISSION, + variant_id=setup_lib_db_with_variant.id, + disposition=Disposition.PRESENT, + reason=EventReason.SUBMITTED, + ) + # Not yet flushed (1 < batch_size). assert len(annotation_status_manager._pending) == 1 - assert len(annotation_status_manager._retirement_filters) == 1 - - annotation_status_manager.flush() - assert len(annotation_status_manager._pending) == 0 - assert len(annotation_status_manager._retirement_filters) == 0 - - def test_batch_retirement_groups_by_annotation_type( - self, session, annotation_status_manager, setup_lib_db_with_variant - ): - """Multiple annotation types in one batch are retired independently.""" - # Create initial annotations for two types - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINVAR_CONTROL, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - vrs_v1 = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - clinvar_v1 = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINVAR_CONTROL, - version="v1", - ) - - # Now add replacements for both types in one batch - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v2", - annotation_data={}, - status=AnnotationStatus.FAILED, - current=True, - ) - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINVAR_CONTROL, - version="v2", - annotation_data={}, - status=AnnotationStatus.FAILED, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - session.refresh(vrs_v1) - session.refresh(clinvar_v1) - assert vrs_v1.current is False - assert clinvar_v1.current is False - - vrs_v2 = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v2", - ) - clinvar_v2 = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINVAR_CONTROL, - version="v2", - ) - assert vrs_v2 is not None and vrs_v2.current is True - assert clinvar_v2 is not None and clinvar_v2.current is True - - -@pytest.mark.unit -class TestAnnotationStatusManagerAuditHelpersUnit: - """Unit tests for audit query helpers: get_annotation_history and get_all_current_annotations.""" - - def test_get_annotation_history_returns_all_rows_newest_first( - self, session, annotation_status_manager, setup_lib_db_with_variant - ): - """get_annotation_history returns both current and retired rows, newest first.""" - # Create two annotations for the same (variant, type, version) — first gets retired - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - annotation_status_manager.flush() - - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.FAILED, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - history = annotation_status_manager.get_annotation_history( + annotation_status_manager.record_event( + AnnotationType.LDH_SUBMISSION, variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", + disposition=Disposition.PRESENT, + reason=EventReason.SUBMITTED, ) - - assert len(history) == 2 - # Newest first - assert history[0].status == AnnotationStatus.FAILED - assert history[0].current is True - assert history[1].status == AnnotationStatus.SUCCESS - assert history[1].current is False - - def test_get_annotation_history_filters_by_version( - self, session, annotation_status_manager, setup_lib_db_with_variant - ): - """get_annotation_history with version only returns matching rows.""" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINVAR_CONTROL, - version="2025-01", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - replace_all_versions=False, - ) - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINVAR_CONTROL, - version="2025-02", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - replace_all_versions=False, - ) - annotation_status_manager.flush() - session.commit() - - history_jan = annotation_status_manager.get_annotation_history( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINVAR_CONTROL, - version="2025-01", - ) - assert len(history_jan) == 1 - assert history_jan[0].version == "2025-01" - - def test_get_annotation_history_without_version_returns_all_versions( - self, session, annotation_status_manager, setup_lib_db_with_variant - ): - """get_annotation_history without version returns rows across all versions.""" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINVAR_CONTROL, - version="2025-01", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - replace_all_versions=False, - ) - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINVAR_CONTROL, - version="2025-02", - annotation_data={}, - status=AnnotationStatus.FAILED, - current=True, - replace_all_versions=False, - ) - annotation_status_manager.flush() - session.commit() - - history = annotation_status_manager.get_annotation_history( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINVAR_CONTROL, - ) - assert len(history) == 2 - - def test_get_annotation_history_empty_for_no_records(self, annotation_status_manager, setup_lib_db_with_variant): - """get_annotation_history returns empty list when no records exist.""" - history = annotation_status_manager.get_annotation_history( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - ) - assert history == [] - - def test_get_annotation_history_auto_flushes_pending( - self, session, annotation_status_manager, setup_lib_db_with_variant - ): - """get_annotation_history flushes pending writes before querying.""" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - # No explicit flush - history = annotation_status_manager.get_annotation_history( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - ) - assert len(history) == 1 + # Auto-flushed at batch_size=2. assert len(annotation_status_manager._pending) == 0 - - def test_get_all_current_annotations_returns_all_types( - self, session, annotation_status_manager, setup_lib_db_with_variant - ): - """get_all_current_annotations returns current annotations across all types.""" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINVAR_CONTROL, - version="2025-01", - annotation_data={}, - status=AnnotationStatus.FAILED, - current=True, - ) - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINGEN_ALLELE_ID, - version=None, - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - all_current = annotation_status_manager.get_all_current_annotations( - variant_id=setup_lib_db_with_variant.id, - ) - assert len(all_current) == 3 - types = {a.annotation_type for a in all_current} - assert types == { - AnnotationType.VRS_MAPPING, - AnnotationType.CLINVAR_CONTROL, - AnnotationType.CLINGEN_ALLELE_ID, - } - - def test_get_all_current_annotations_excludes_retired( - self, session, annotation_status_manager, setup_lib_db_with_variant - ): - """get_all_current_annotations does not include retired rows.""" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - annotation_status_manager.flush() - - # Replace it — v1 becomes retired - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v2", - annotation_data={}, - status=AnnotationStatus.FAILED, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - all_current = annotation_status_manager.get_all_current_annotations( - variant_id=setup_lib_db_with_variant.id, - ) - assert len(all_current) == 1 - assert all_current[0].version == "v2" - - def test_get_all_current_annotations_empty_for_no_records( - self, annotation_status_manager, setup_lib_db_with_variant - ): - """get_all_current_annotations returns empty list when no records exist.""" - result = annotation_status_manager.get_all_current_annotations( - variant_id=setup_lib_db_with_variant.id, - ) - assert result == [] - - def test_get_all_current_annotations_auto_flushes_pending( - self, session, annotation_status_manager, setup_lib_db_with_variant - ): - """get_all_current_annotations flushes pending writes before querying.""" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - # No explicit flush - result = annotation_status_manager.get_all_current_annotations( - variant_id=setup_lib_db_with_variant.id, - ) - assert len(result) == 1 - assert len(annotation_status_manager._pending) == 0 - - def test_get_all_current_annotations_ordered_by_type_then_version( - self, session, annotation_status_manager, setup_lib_db_with_variant - ): - """get_all_current_annotations returns results ordered by annotation_type, version.""" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINVAR_CONTROL, - version="2025-02", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - replace_all_versions=False, - ) - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINVAR_CONTROL, - version="2025-01", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - replace_all_versions=False, - ) - annotation_status_manager.flush() - session.commit() - - all_current = annotation_status_manager.get_all_current_annotations( - variant_id=setup_lib_db_with_variant.id, - ) - assert len(all_current) == 3 - # clinvar_control < vrs_mapping alphabetically - assert all_current[0].annotation_type == AnnotationType.CLINVAR_CONTROL - assert all_current[0].version == "2025-01" - assert all_current[1].annotation_type == AnnotationType.CLINVAR_CONTROL - assert all_current[1].version == "2025-02" - assert all_current[2].annotation_type == AnnotationType.VRS_MAPPING - - -@pytest.mark.unit -class TestVariantAnnotationStatusReprUnit: - """Unit tests for the VariantAnnotationStatus __repr__ method.""" - - def test_repr_includes_key_fields(self, session, annotation_status_manager, setup_lib_db_with_variant): - """__repr__ includes id, variant_id, type, version, status, current, and created_at.""" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - annotation_status_manager.flush() session.commit() - - annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - repr_str = repr(annotation) - - assert "VariantAnnotationStatus" in repr_str - assert f"id={annotation.id}" in repr_str - assert f"variant_id={setup_lib_db_with_variant.id}" in repr_str - assert "type='vrs_mapping'" in repr_str - assert "version='v1'" in repr_str - assert "status='success'" in repr_str - assert "current=True" in repr_str - assert "created_at=" in repr_str From aacc13aadcaaff73dff115b2a7ffe05026679961 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 30 Jun 2026 08:30:31 -0700 Subject: [PATCH 037/252] feat(gnomad): derive annotation events from link verdicts link_gnomad_variants_to_alleles now returns a GnomadLinkVerdict per touched allele (CREATED/UNCHANGED) as the single source of truth for per-allele status; the job derives annotation events directly from the verdict map rather than re-querying link state. Alleles absent from the map are read as 'gnomAD had no record'. --- src/mavedb/lib/gnomad.py | 38 ++++-- .../worker/jobs/external_services/gnomad.py | 119 +++++++++--------- tests/lib/test_gnomad.py | 41 +++--- .../jobs/external_services/test_gnomad.py | 108 +++++++++------- 4 files changed, 181 insertions(+), 125 deletions(-) diff --git a/src/mavedb/lib/gnomad.py b/src/mavedb/lib/gnomad.py index 8456ec35d..5fd4d888c 100644 --- a/src/mavedb/lib/gnomad.py +++ b/src/mavedb/lib/gnomad.py @@ -1,6 +1,7 @@ import logging import os import re +from enum import Enum from typing import Any, Sequence, Union from sqlalchemy import Connection, Row, func, select, text @@ -12,6 +13,9 @@ from mavedb.models.gnomad_allele_link import GnomadAlleleLink from mavedb.models.gnomad_variant import GnomADVariant +logger = logging.getLogger(__name__) + + GNOMAD_DB_NAME = "gnomAD" GNOMAD_DATA_VERSION = os.getenv("GNOMAD_DATA_VERSION", "v4.1") _CAID_LEADING_ZERO_RE = r"^(CA)0+([0-9])" @@ -21,7 +25,17 @@ in link_gnomad_variants_to_alleles. """ -logger = logging.getLogger(__name__) + +class GnomadLinkVerdict(str, Enum): + """Per-allele outcome of a gnomAD linking run, returned for every allele the linker touched. + + The single source of truth for what happened to an allele's link this run — the caller derives + annotation status directly from this, never by re-querying link state (which would be a second, + drift-prone source of truth). + """ + + CREATED = "created" # link created or superseded this run (a new/changed live link) + UNCHANGED = "unchanged" # a live link already pointed at the resolved variant; left untouched def gnomad_identifier(contig: str, position: Union[str, int], alleles: list[str]) -> str: @@ -150,7 +164,9 @@ def gnomad_variant_data_for_caids( return result_rows -def link_gnomad_variants_to_alleles(db: Session, gnomad_variant_data: Sequence[Row[Any]]) -> set[int]: +def link_gnomad_variants_to_alleles( + db: Session, gnomad_variant_data: Sequence[Row[Any]] +) -> dict[int, GnomadLinkVerdict]: """Link gnomAD variants to deduplicated alleles by CAID, superseding only on change. Every ``Allele`` carrying the row's ``clingen_allele_id`` (populated by CAR) is linked through a @@ -163,12 +179,16 @@ def link_gnomad_variants_to_alleles(db: Session, gnomad_variant_data: Sequence[R it. A current-version link to a *different* identifier (a CAID re-resolved within one release) is logged and superseded newest-wins, not raised. - Does not commit. Returns the allele ids whose link was created or superseded this run. + Does not commit. Returns a verdict per allele *touched* this run (matched a CAID-bearing row): + :attr:`GnomadLinkVerdict.CREATED` for a created/superseded link, :attr:`~GnomadLinkVerdict.UNCHANGED` + for a live link left in place. Alleles absent from the map were matched by no row — the caller reads + those as "gnomAD had no record". This is the single source of truth for per-allele status; callers + must not re-derive it by re-querying link state. """ save_to_logging_context({"num_gnomad_variant_rows": len(gnomad_variant_data)}) logger.debug(msg="Linking gnomAD variants to alleles", extra=logging_context()) - changed_allele_ids: set[int] = set() + verdicts: dict[int, GnomadLinkVerdict] = {} for index, row in enumerate(gnomad_variant_data, start=1): logger.info( msg=f"Processing gnomAD variant row {index}/{len(gnomad_variant_data)}: {row.caid}", extra=logging_context() @@ -241,6 +261,7 @@ def link_gnomad_variants_to_alleles(db: Session, gnomad_variant_data: Sequence[R # No change: live link already points here — leave it untouched (no spurious boundary). if live_link is not None and live_link.gnomad_variant_id == gnomad_variant.id: + verdicts.setdefault(allele.id, GnomadLinkVerdict.UNCHANGED) continue if ( @@ -265,7 +286,7 @@ def link_gnomad_variants_to_alleles(db: Session, gnomad_variant_data: Sequence[R [GnomadAlleleLink(allele_id=allele.id, gnomad_variant_id=gnomad_variant.id)], GnomadAlleleLink.allele_id == allele.id, ) - changed_allele_ids.add(allele.id) + verdicts[allele.id] = GnomadLinkVerdict.CREATED # created always wins over a same-run unchanged logger.debug( msg=f"Linked gnomAD variant {gnomad_variant.db_identifier} to allele {allele.id} ({allele.clingen_allele_id})", @@ -276,9 +297,10 @@ def link_gnomad_variants_to_alleles(db: Session, gnomad_variant_data: Sequence[R f"Processed {len(alleles_with_caid)} alleles with CAID {row.caid} for gnomAD variant {gnomad_identifier_for_variant}. ({index}/{len(gnomad_variant_data)})" ) - save_to_logging_context({"changed_allele_count": len(changed_allele_ids)}) + changed_allele_count = sum(1 for v in verdicts.values() if v is GnomadLinkVerdict.CREATED) + save_to_logging_context({"changed_allele_count": changed_allele_count}) logger.info( - msg=f"Created or superseded {len(changed_allele_ids)} allele links this run.", + msg=f"Created or superseded {changed_allele_count} allele links this run.", extra=logging_context(), ) - return changed_allele_ids + return verdicts diff --git a/src/mavedb/worker/jobs/external_services/gnomad.py b/src/mavedb/worker/jobs/external_services/gnomad.py index 8daa5c9af..3b7a48d16 100644 --- a/src/mavedb/worker/jobs/external_services/gnomad.py +++ b/src/mavedb/worker/jobs/external_services/gnomad.py @@ -7,6 +7,7 @@ """ import logging +from collections import Counter from sqlalchemy import select @@ -15,12 +16,14 @@ from mavedb.lib.clingen.alleles import get_alleles_for_score_set, group_alleles_for_annotation from mavedb.lib.gnomad import ( GNOMAD_DATA_VERSION, + GnomadLinkVerdict, gnomad_variant_data_for_caids, link_gnomad_variants_to_alleles, ) from mavedb.lib.types.workflow import JobExecutionOutcome from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationFailureCategory, AnnotationStatus +from mavedb.models.enums.disposition import Disposition +from mavedb.models.enums.event_reason import EventReason from mavedb.models.gnomad_allele_link import GnomadAlleleLink from mavedb.models.gnomad_variant import GnomADVariant from mavedb.models.score_set import ScoreSet @@ -33,34 +36,31 @@ def _annotate_gnomad( annotation_manager: AnnotationStatusManager, - variant_ids: list[int], - status: AnnotationStatus, + allele_id: int, + disposition: Disposition, + reason: EventReason, *, - failure_category: AnnotationFailureCategory | None = None, error_message: str | None = None, metadata: dict | None = None, ) -> None: - """Fan a GNOMAD_ALLELE_FREQUENCY annotation out to every variant served by an allele. + """Record one GNOMAD_ALLELE_FREQUENCY event for an allele (frequency is an allele-level fact). - AAS migration seam: the single choke point for gnomAD's per-variant VAS writes. At migration it - becomes an allele-keyed event writer; the per-variant fan-out goes away, and the variant - association narrows to provenance (which variant's mapping drove the linkage). See - docs/design/allele-annotation-status.md. + The single choke point for gnomAD's status writes. One event per allele, stamped at the current + gnomAD version; provenance (which variants drove the linkage) is derived from the live links + as-of the event, not fanned out here. """ - annotation_data: dict = {"annotation_metadata": metadata or {}} + meta = dict(metadata or {}) if error_message is not None: - annotation_data["error_message"] = error_message - - for variant_id in variant_ids: - annotation_manager.add_annotation( - variant_id=variant_id, - annotation_type=AnnotationType.GNOMAD_ALLELE_FREQUENCY, - version=GNOMAD_DATA_VERSION, - status=status, - failure_category=failure_category, - annotation_data=annotation_data, - current=True, - ) + meta["error_message"] = error_message + + annotation_manager.record_event( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, + allele_id=allele_id, + disposition=disposition, + reason=reason, + source_version=GNOMAD_DATA_VERSION, + metadata=meta or None, + ) @with_pipeline_management @@ -113,12 +113,20 @@ async def link_gnomad_variants(ctx: dict, job_id: int, job_manager: JobManager) # One work-unit per allele (payload = CAID; alleles without one are skipped). Covers ALL the score # set's alleles — authoritative and RT-derived — since the genomic allele gnomAD knows is often the - # RT-derived one; VAS still fans only to authoritative_variant_ids (the bandaid seam). + # RT-derived one. Events are allele-keyed, so every linked allele is recorded. allele_data = group_alleles_for_annotation( get_alleles_for_score_set(job_manager.db, score_set.id), payload=lambda row: row.clingen_allele_id, ) + annotation_counts: Counter[str] = Counter( + { + "created_allele_count": 0, + "preexisting_allele_count": 0, + "skipped_allele_count": 0, + } + ) + num_alleles_with_caids = len(allele_data) job_manager.save_to_context({"num_alleles_to_link_gnomad": num_alleles_with_caids}) @@ -128,9 +136,7 @@ async def link_gnomad_variants(ctx: dict, job_id: int, job_manager: JobManager) extra=job_manager.logging_context(), ) job_manager.db.flush() - return JobExecutionOutcome.succeeded( - data={"created_allele_count": 0, "preexisting_allele_count": 0, "skipped_allele_count": 0} - ) + return JobExecutionOutcome.succeeded(data=dict(annotation_counts)) job_manager.update_progress( 10, 100, f"Found {num_alleles_with_caids} alleles with CAIDs to link to gnomAD variants." @@ -150,10 +156,11 @@ def alleles_linked_at_current_version(allele_ids: set[int]) -> set[int]: ).all() ) - # Cost: skip alleles already linked at the current version (they can't change). force re-fetches + # Skip alleles already linked at the current version (they can't change). force re-fetches # all; the linker still supersedes only on change, so a forced no-op writes nothing. already_current = set() if force else alleles_linked_at_current_version(set(allele_data.keys())) - variant_caids = sorted({allele_data[aid].payload for aid in allele_data if aid not in already_current}) + variant_caids = sorted({allele_data[aid] for aid in allele_data if aid not in already_current}) + job_manager.save_to_context( { "num_alleles_already_current": len(already_current), @@ -162,7 +169,7 @@ def alleles_linked_at_current_version(allele_ids: set[int]) -> set[int]: } ) - changed_allele_ids: set[int] = set() + verdicts: dict[int, GnomadLinkVerdict] = {} if variant_caids: with athena.engine.connect() as athena_session: logger.debug("Fetching gnomAD variants from Athena.") @@ -175,7 +182,7 @@ def alleles_linked_at_current_version(allele_ids: set[int]) -> set[int]: ) logger.info(msg="Attempting to link alleles to gnomAD variants.", extra=job_manager.logging_context()) - changed_allele_ids = link_gnomad_variants_to_alleles(job_manager.db, gnomad_variant_data) + verdicts = link_gnomad_variants_to_alleles(job_manager.db, gnomad_variant_data) job_manager.db.flush() else: logger.info( @@ -184,43 +191,43 @@ def alleles_linked_at_current_version(allele_ids: set[int]) -> set[int]: ) job_manager.update_progress(75, 100, "All alleles already current at this gnomAD version.") - # Status is an audit event, written every run: SUCCESS/created if linked this run, SUCCESS/preexisting - # if already current, SKIPPED if no current-version link (gnomAD had no data for the CAID). - current_after = alleles_linked_at_current_version(set(allele_data.keys())) - - annotation_manager = AnnotationStatusManager(job_manager.db, job_run_id=job_manager.job_id) - created_count = preexisting_count = skipped_count = 0 - for allele_id, entry in allele_data.items(): - if allele_id in current_after: - action = "created" if allele_id in changed_allele_ids else "preexisting" - if action == "created": - created_count += 1 - else: - preexisting_count += 1 + annotation_manager = AnnotationStatusManager( + job_manager.db, job_run_id=job_manager.job_id, score_set_id=score_set.id + ) + for allele_id, caid in allele_data.items(): + verdict = verdicts.get(allele_id) + if verdict is GnomadLinkVerdict.CREATED: + annotation_counts["created_allele_count"] += 1 + _annotate_gnomad( + annotation_manager, + allele_id, + Disposition.PRESENT, + EventReason.CREATED, + metadata={"clingen_allele_id": caid}, + ) + elif allele_id in already_current or verdict is GnomadLinkVerdict.UNCHANGED: + annotation_counts["preexisting_allele_count"] += 1 _annotate_gnomad( annotation_manager, - entry.authoritative_variant_ids, - AnnotationStatus.SUCCESS, - metadata={"clingen_allele_id": entry.payload, "action": action}, + allele_id, + Disposition.PRESENT, + EventReason.PREEXISTING, + metadata={"clingen_allele_id": caid}, ) else: - skipped_count += 1 + annotation_counts["skipped_allele_count"] += 1 _annotate_gnomad( annotation_manager, - entry.authoritative_variant_ids, - AnnotationStatus.SKIPPED, - failure_category=AnnotationFailureCategory.EXTERNAL_REFERENCE_NOT_FOUND, + allele_id, + Disposition.ABSENT, + EventReason.NO_RECORD, error_message="No gnomAD variant could be linked for this allele.", - metadata={"clingen_allele_id": entry.payload}, + metadata={"clingen_allele_id": caid}, ) annotation_manager.flush() - outcome_data = { - "created_allele_count": created_count, - "preexisting_allele_count": preexisting_count, - "skipped_allele_count": skipped_count, - } + outcome_data = dict(annotation_counts) job_manager.save_to_context(outcome_data) logger.info(msg="Done linking gnomAD variants to alleles.", extra=job_manager.logging_context()) job_manager.db.flush() diff --git a/tests/lib/test_gnomad.py b/tests/lib/test_gnomad.py index e00b57e76..f9c8b17aa 100644 --- a/tests/lib/test_gnomad.py +++ b/tests/lib/test_gnomad.py @@ -9,6 +9,7 @@ fastapi = pytest.importorskip("fastapi") from mavedb.lib.gnomad import ( + GnomadLinkVerdict, allele_list_from_list_like_string, gnomad_identifier, gnomad_table_name, @@ -170,7 +171,7 @@ def test_links_new_gnomad_variant_to_allele(session, mocked_gnomad_variant_row): with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): result = link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) - assert result == {allele.id} + assert result == {allele.id: GnomadLinkVerdict.CREATED} session.commit() live_links = _live_links_for(session, allele.id) @@ -186,7 +187,7 @@ def test_can_link_gnomad_variants_with_none_type_faf_fields(session, mocked_gnom with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): result = link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) - assert result == {allele.id} + assert result == {allele.id: GnomadLinkVerdict.CREATED} session.commit() live_links = _live_links_for(session, allele.id) @@ -202,7 +203,7 @@ def test_links_existing_gnomad_variant(session, mocked_gnomad_variant_row): with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): result = link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) - assert result == {allele.id} + assert result == {allele.id: GnomadLinkVerdict.CREATED} session.commit() # Reused the existing gnomAD variant rather than creating a second. @@ -214,14 +215,20 @@ def test_links_existing_gnomad_variant(session, mocked_gnomad_variant_row): def test_re_running_unchanged_data_is_idempotent(session, mocked_gnomad_variant_row): """Supersede only on change: a second run with identical data writes nothing — one live link, - no retired rows, so the valid-time history records no spurious boundary.""" + no retired rows, so the valid-time history records no spurious boundary. The second run still + *reports* the allele (verdict UNCHANGED), so the caller can mark it preexisting without re-querying + link state.""" allele = _make_allele(session, mocked_gnomad_variant_row.caid, vrs_digest="vrs-1") with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): - assert link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) == {allele.id} + assert link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) == { + allele.id: GnomadLinkVerdict.CREATED + } session.commit() - # Second run sees the live link already points to this gnomAD variant → no change reported. - assert link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) == set() + # Second run sees the live link already points to this gnomAD variant → unchanged, no DB write. + assert link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) == { + allele.id: GnomadLinkVerdict.UNCHANGED + } session.commit() # One link, still live, never retired — the re-run did not churn the history. @@ -237,11 +244,15 @@ def test_version_bump_supersedes_to_single_live_link(session, mocked_gnomad_vari allele = _make_allele(session, mocked_gnomad_variant_row.caid, vrs_digest="vrs-1") with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", "v1.old"): - assert link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) == {allele.id} + assert link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) == { + allele.id: GnomadLinkVerdict.CREATED + } session.commit() with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", "v2.new"): - assert link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) == {allele.id} + assert link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) == { + allele.id: GnomadLinkVerdict.CREATED + } session.commit() live_links = _live_links_for(session, allele.id) @@ -273,7 +284,9 @@ def test_same_version_different_identifier_supersedes_newest_wins(session, mocke session.commit() with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): - assert link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) == {allele.id} + assert link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) == { + allele.id: GnomadLinkVerdict.CREATED + } session.commit() live_links = _live_links_for(session, allele.id) @@ -288,7 +301,7 @@ def test_links_one_gnomad_variant_to_multiple_alleles_sharing_a_caid(session, mo with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): result = link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) - assert result == {allele1.id, allele2.id} + assert result == {allele1.id: GnomadLinkVerdict.CREATED, allele2.id: GnomadLinkVerdict.CREATED} session.commit() for allele in (allele1, allele2): @@ -305,15 +318,15 @@ def test_links_allele_when_dump_strips_leading_zero_from_caid(session, mocked_gn with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): result = link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) - assert result == {allele.id} + assert result == {allele.id: GnomadLinkVerdict.CREATED} session.commit() assert len(_live_links_for(session, allele.id)) == 1 -def test_returns_empty_set_when_no_alleles_match(session, mocked_gnomad_variant_row): +def test_returns_empty_map_when_no_alleles_match(session, mocked_gnomad_variant_row): result = link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) - assert result == set() + assert result == {} assert len(session.scalars(select(GnomadAlleleLink)).all()) == 0 # No gnomAD variant is created when nothing matches the CAID. assert len(session.scalars(select(GnomADVariant)).all()) == 0 diff --git a/tests/worker/jobs/external_services/test_gnomad.py b/tests/worker/jobs/external_services/test_gnomad.py index 23316adb8..0b14fda21 100644 --- a/tests/worker/jobs/external_services/test_gnomad.py +++ b/tests/worker/jobs/external_services/test_gnomad.py @@ -14,7 +14,7 @@ from mavedb.models.enums.job_pipeline import JobStatus, PipelineStatus from mavedb.models.gnomad_allele_link import GnomadAlleleLink from mavedb.models.gnomad_variant import GnomADVariant -from mavedb.models.variant_annotation_status import VariantAnnotationStatus +from mavedb.models.annotation_event import AnnotationEvent from mavedb.worker.jobs.external_services.gnomad import link_gnomad_variants from mavedb.worker.lib.managers.job_manager import JobManager @@ -91,7 +91,7 @@ async def test_link_gnomad_variants_call_linking_method( ), patch( "mavedb.worker.jobs.external_services.gnomad.link_gnomad_variants_to_alleles", - return_value=set(), + return_value={}, ) as mock_linking_method, patch("mavedb.worker.jobs.external_services.gnomad.athena.engine", athena_engine), ): @@ -156,8 +156,8 @@ async def test_link_gnomad_variants_no_alleles_with_caids( assert len(session.scalars(select(GnomadAlleleLink)).all()) == 0 # Verify no annotations were rendered (since there were no alleles with CAIDs) - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 0 + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 0 # Verify job status updates session.refresh(sample_link_gnomad_variants_run) @@ -189,11 +189,13 @@ async def test_link_gnomad_variants_no_matching_caids( # Verify that no allele links were created assert len(session.scalars(select(GnomadAlleleLink)).all()) == 0 - # Verify a skipped annotation status was rendered (since there was an allele with a CAID) - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == "skipped" - assert annotation_statuses[0].annotation_type == "gnomad_allele_frequency" + # Verify an absent event was rendered: the allele's CAID was queried but gnomAD had no record. + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 1 + assert events[0].disposition == "absent" + assert events[0].reason == "no_record" + assert events[0].annotation_type == "gnomad_allele_frequency" + assert events[0].allele_id == allele.id and events[0].variant_id is None # Verify job status updates session.refresh(sample_link_gnomad_variants_run) @@ -230,16 +232,18 @@ async def test_link_gnomad_variants_successful_linking_independent( assert len(live_links) == 1 # Verify annotation status was rendered - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == "success" - assert annotation_statuses[0].annotation_type == "gnomad_allele_frequency" + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 1 + assert events[0].disposition == "present" + assert events[0].reason == "created" + assert events[0].annotation_type == "gnomad_allele_frequency" + assert events[0].allele_id is not None and events[0].variant_id is None # Verify job status updates session.refresh(sample_link_gnomad_variants_run) assert sample_link_gnomad_variants_run.status == JobStatus.SUCCEEDED - async def test_link_gnomad_variants_links_rt_derived_allele_but_annotates_only_authoritative( + async def test_link_gnomad_variants_links_and_annotates_rt_derived_allele( self, session, with_populated_domain_data, @@ -249,11 +253,12 @@ async def test_link_gnomad_variants_links_rt_derived_allele_but_annotates_only_a setup_rt_derived_allele_with_caid, athena_engine, ): - """gnomAD linkage must cover the full allele set, not just authoritative links: the - RT-derived allele carries the CAID gnomAD matches and must be linked. Per-variant VAS status, - however, is written only for the authoritative link (the interim bandaid) — so exactly one - annotation row is produced, keyed to the variant, never a second row for the RT-derived - allele.""" + """gnomAD linkage covers the full allele set, not just authoritative links: the RT-derived + allele carries the CAID gnomAD matches and is linked. Events are now allele-keyed, so the + RT-derived allele's PRESENT status is recorded directly — the limitation the per-variant + bandaid had (dropping the RT-derived allele's status) is lifted. Each allele in the score set + gets its own event: present for the linked RT allele, absent for the unmatched authoritative + one.""" variant, authoritative_allele, rt_allele = setup_rt_derived_allele_with_caid with patch("mavedb.worker.jobs.external_services.gnomad.athena.engine", athena_engine): @@ -279,13 +284,16 @@ async def test_link_gnomad_variants_links_rt_derived_allele_but_annotates_only_a == 0 ) - # Annotation status is written only for the authoritative link: exactly one VAS row, for the - # variant. (Its status is "skipped" because the variant's authoritative allele had no match — - # cross-level resolution onto the RT-derived allele is deferred to the AnnotationEvent design.) - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].variant_id == variant.id - assert annotation_statuses[0].annotation_type == "gnomad_allele_frequency" + # One allele-keyed event per allele (never per-variant): the linked RT allele is present, + # the unmatched authoritative allele is absent. The variant resolves its derived allele's + # status through the live links — no variant_id on the events. + events = session.scalars(select(AnnotationEvent)).all() + assert all(e.annotation_type == "gnomad_allele_frequency" for e in events) + assert all(e.variant_id is None for e in events) + by_allele = {e.allele_id: e for e in events} + assert by_allele[rt_allele.id].disposition == "present" + assert by_allele[authoritative_allele.id].disposition == "absent" + assert by_allele[authoritative_allele.id].reason == "no_record" async def test_link_gnomad_variants_skips_allele_already_current( self, @@ -326,11 +334,11 @@ async def test_link_gnomad_variants_skips_allele_already_current( assert len(links) == 1 assert links[0].valid_to is None - # Status is SUCCESS, marked preexisting. - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == "success" - assert annotation_statuses[0].annotation_metadata["action"] == "preexisting" + # Event is PRESENT, marked preexisting (the link was already current; not created this run). + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 1 + assert events[0].disposition == "present" + assert events[0].reason == "preexisting" async def test_link_gnomad_variants_force_refetches_without_churn( self, @@ -403,10 +411,12 @@ async def test_link_gnomad_variants_successful_linking_pipeline( assert len(session.scalars(select(GnomadAlleleLink).where(GnomadAlleleLink.current)).all()) > 0 # Verify annotation status was rendered - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == "success" - assert annotation_statuses[0].annotation_type == "gnomad_allele_frequency" + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 1 + assert events[0].disposition == "present" + assert events[0].reason == "created" + assert events[0].annotation_type == "gnomad_allele_frequency" + assert events[0].allele_id is not None and events[0].variant_id is None # Verify job status updates session.refresh(sample_link_gnomad_variants_run_pipeline) @@ -481,10 +491,12 @@ async def test_link_gnomad_variants_with_arq_context_independent( assert len(session.scalars(select(GnomadAlleleLink).where(GnomadAlleleLink.current)).all()) > 0 # Verify annotation status was rendered - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == "success" - assert annotation_statuses[0].annotation_type == "gnomad_allele_frequency" + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 1 + assert events[0].disposition == "present" + assert events[0].reason == "created" + assert events[0].annotation_type == "gnomad_allele_frequency" + assert events[0].allele_id is not None and events[0].variant_id is None # Verify that the job completed successfully session.refresh(sample_link_gnomad_variants_run) @@ -514,10 +526,12 @@ async def test_link_gnomad_variants_with_arq_context_pipeline( assert len(session.scalars(select(GnomadAlleleLink).where(GnomadAlleleLink.current)).all()) > 0 # Verify annotation status was rendered - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == "success" - assert annotation_statuses[0].annotation_type == "gnomad_allele_frequency" + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 1 + assert events[0].disposition == "present" + assert events[0].reason == "created" + assert events[0].annotation_type == "gnomad_allele_frequency" + assert events[0].allele_id is not None and events[0].variant_id is None # Verify that the job completed successfully session.refresh(sample_link_gnomad_variants_run_pipeline) @@ -557,8 +571,8 @@ async def test_link_gnomad_variants_with_arq_context_exception_handling_independ assert len(session.scalars(select(GnomadAlleleLink)).all()) == 0 # Verify no annotations were rendered - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 0 + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 0 # Verify that the job errored session.refresh(sample_link_gnomad_variants_run) @@ -594,8 +608,8 @@ async def test_link_gnomad_variants_with_arq_context_exception_handling_pipeline assert len(session.scalars(select(GnomadAlleleLink)).all()) == 0 # Verify no annotations were rendered - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 0 + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 0 # Verify that the job errored session.refresh(sample_link_gnomad_variants_run_pipeline) From 6f0f47f50e22e62430d5166d1b59b4c7de7efad7 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 30 Jun 2026 08:30:44 -0700 Subject: [PATCH 038/252] feat(vep): derive annotation events from resolution outcomes VEP resolution now returns a VepResolution (consequences vs errored) so a genuine empty is distinguished from an unknown, and linking returns a VepLinkVerdict per allele (CREATED/UNCHANGED). The job derives annotation events from these outcomes instead of re-querying consequence state; errored HGVS are retried, not recorded as negatives. --- src/mavedb/lib/vep.py | 155 ++++++----- .../worker/jobs/external_services/vep.py | 243 +++++++++++------- tests/lib/test_vep.py | 32 ++- .../external_services/network/test_vep.py | 29 ++- .../worker/jobs/external_services/test_vep.py | 143 +++++++---- 5 files changed, 369 insertions(+), 233 deletions(-) diff --git a/src/mavedb/lib/vep.py b/src/mavedb/lib/vep.py index 4b8ac0342..25c0732fa 100644 --- a/src/mavedb/lib/vep.py +++ b/src/mavedb/lib/vep.py @@ -5,9 +5,9 @@ import logging import os from datetime import date -from typing import Mapping, Optional, Sequence +from enum import Enum +from typing import Mapping, NamedTuple, Optional, Sequence -import requests from sqlalchemy import select from sqlalchemy.orm import Session @@ -17,9 +17,9 @@ logger = logging.getLogger(__name__) + ENSEMBL_API_URL = os.environ.get("ENSEMBL_API_URL", "https://rest.ensembl.org") -# List of all possible VEP consequences, in order from most to least severe VEP_CONSEQUENCES = [ "transcript_ablation", "splice_acceptor_variant", @@ -77,6 +77,40 @@ "intron_variant", "intergenic_variant", ] +""" +List of all functional consequences VEP can return, in order of severity (most severe first). +""" + + +class VepLinkVerdict(str, Enum): + """Per-allele outcome of a VEP linking run, returned for every allele whose status is decided. + + The single source of truth for what happened to an allele's consequence this run — the caller + derives annotation status from this, never by re-querying consequence state. An allele absent from + the map had no live consequence and resolved none this run (the caller reads that as "no result"). + + - ``CREATED`` — a new or changed consequence was created/superseded this run. + - ``UNCHANGED`` — a live consequence was retained (value matched, or held against a null run). + """ + + CREATED = "created" + UNCHANGED = "unchanged" + + +class VepResolution(NamedTuple): + """Outcome of resolving a set of HGVS strings, splitting the two kinds of "no consequence". + + This outcome allows us to differentiate between a genuine empty (VEP found nothing) and an + unknown (VEP failed to answer). + + - ``consequences`` — HGVS that resolved to a most-severe consequence (the hits). + - ``errored`` — HGVS whose VEP/Recoder request *failed* (HTTP/transport error after retries); the + result is unknown and the allele should be retried, not treated as a negative. + - Any queried HGVS in neither set was answered (HTTP 200) with no consequence — a genuine **empty**. + """ + + consequences: dict[str, str] + errored: set[str] async def run_variant_recoder(missing_hgvs: Sequence[str]) -> dict[str, list[str]]: @@ -86,35 +120,30 @@ async def run_variant_recoder(missing_hgvs: Sequence[str]) -> dict[str, list[str missing_hgvs (Sequence[str]): List of HGVS strings to recode. Returns: - dict[str, list[str]]: Mapping of input HGVS to list of genomic HGVS strings (hgvsg). - Returns an empty dict if Ensembl rejects the batch (e.g. 400 for - unrecognised identifiers) — callers treat missing entries as failures. + dict[str, list[str]]: Mapping of input HGVS to list of genomic HGVS strings (hgvsg). An input + with no recodable genomic mapping is simply absent (a genuine empty). + + Raises: + requests.exceptions.RequestException: if the Recoder request fails (HTTP/transport error after + retries). The caller attributes the failure to this batch's inputs so they are reported as + errored (unknown, retry) rather than silently conflated with a genuine empty. """ headers = {"Content-Type": "application/json", "Accept": "application/json"} + # request_with_backoff is synchronous (requests lib + time.sleep backoff); run_in_executor # keeps the event loop free during the full request + any retry wait time. loop = asyncio.get_running_loop() - try: - response = await loop.run_in_executor( - None, - functools.partial( - request_with_backoff, - method="POST", - url=f"{ENSEMBL_API_URL}/variant_recoder/human", - headers=headers, - json={"ids": list(missing_hgvs)}, - timeout=600, # Variant Recoder can be very slow for large batches and 504s are common; generous timeout and backoff retries are needed - ), - ) - except requests.exceptions.HTTPError as exc: - # A 4xx from Ensembl (e.g. 400 for an unrecognised identifier format) means the batch - # cannot be recoded. Return empty so callers can handle these missing entries. - logger.warning( - f"Variant Recoder returned {exc.response.status_code if exc.response is not None else 'unknown'} " - f"for batch of {len(missing_hgvs)} HGVS strings — treating as no results.", - exc_info=exc, - ) - return {} + response = await loop.run_in_executor( + None, + functools.partial( + request_with_backoff, + method="POST", + url=f"{ENSEMBL_API_URL}/variant_recoder/human", + headers=headers, + json={"ids": list(missing_hgvs)}, + timeout=600, # Variant Recoder can be very slow for large batches and 504s are common; generous timeout and backoff retries are needed + ), + ) data = response.json() # request_with_backoff handles http errors, so no need to check response status @@ -124,11 +153,13 @@ async def run_variant_recoder(missing_hgvs: Sequence[str]) -> dict[str, list[str hgvs_string = variant_data.get("input") if isinstance(variant_data, dict) else None if variant_str == "input" or not hgvs_string: continue + genomic_strings = variant_data.get("hgvsg") if isinstance(variant_data, dict) else None if genomic_strings: for genomic_hgvs in genomic_strings: if genomic_hgvs.startswith("NC_"): hgvs_to_genomic.setdefault(hgvs_string, []).append(genomic_hgvs) + return hgvs_to_genomic @@ -143,11 +174,14 @@ async def get_functional_consequence(hgvs_strings: Sequence[str]) -> dict[str, O hgvs_strings (Sequence[str]): List of HGVS strings to process (max 200 per call). Returns: - dict[str, Optional[str]]: Mapping of HGVS string to functional consequence. - If no consequence found, maps to None. Returns an empty dict - if Ensembl rejects the batch (e.g. 400 for unrecognised - identifiers) — callers treat missing entries as needing Recoder - fallback or as failures. + dict[str, Optional[str]]: Mapping of HGVS string to functional consequence. An HGVS the + successful response carried no consequence for maps to None (a genuine + miss — the caller may try Recoder, else treats it as empty). + + Raises: + requests.exceptions.RequestException: if the VEP request fails (HTTP/transport error after + retries). The caller attributes the failure to this batch's inputs so they are reported as + errored (unknown, retry) rather than silently conflated with a genuine empty. """ if len(hgvs_strings) > 200: raise ValueError( @@ -160,27 +194,17 @@ async def get_functional_consequence(hgvs_strings: Sequence[str]) -> dict[str, O # request_with_backoff is synchronous (requests lib + time.sleep backoff); run_in_executor # keeps the event loop free during the full request + any retry wait time. loop = asyncio.get_running_loop() - try: - response = await loop.run_in_executor( - None, - functools.partial( - request_with_backoff, - method="POST", - url=f"{ENSEMBL_API_URL}/vep/human/hgvs", - headers=headers, - json={"hgvs_notations": list(hgvs_strings)}, - timeout=60, # VEP can be slow for large batches. - ), - ) - except requests.exceptions.HTTPError as exc: - # A 4xx from Ensembl (e.g. 400 for an unrecognised identifier) means the batch cannot - # be resolved. Return empty so the callers can handle these missing entries. - logger.warning( - f"VEP returned {exc.response.status_code if exc.response is not None else 'unknown'} " - f"for batch of {len(hgvs_strings)} HGVS strings — treating as no results.", - exc_info=exc, - ) - return result + response = await loop.run_in_executor( + None, + functools.partial( + request_with_backoff, + method="POST", + url=f"{ENSEMBL_API_URL}/vep/human/hgvs", + headers=headers, + json={"hgvs_notations": list(hgvs_strings)}, + timeout=60, # VEP can be slow for large batches. + ), + ) data = response.json() for entry in data: @@ -222,7 +246,7 @@ def link_vep_consequences_to_alleles( *, source_version: str, access_date: date, -) -> set[int]: +) -> dict[int, VepLinkVerdict]: """Store VEP consequences against deduplicated alleles, superseding only on change. ``consequence_by_allele_id`` maps each queried allele to the consequence VEP resolved this run @@ -239,12 +263,16 @@ def link_vep_consequences_to_alleles( - **None this run**: leave any live row in place — do not overwrite a held consequence with a null result. Log a warning if VEP found no consequence for an allele which previously had a live consequence. - Does not commit. Returns the allele ids whose consequence was created or superseded this run. + Does not commit. Returns a verdict per allele whose status is decided this run: + :attr:`VepLinkVerdict.CREATED` for a created/superseded consequence, :attr:`~VepLinkVerdict.UNCHANGED` + for a live consequence retained (value matched, or held against a null run). An allele absent from + the map had no live row and resolved nothing — the caller reads that as "no result". This is the + single source of truth for per-allele status; callers must not re-derive it from consequence state. """ save_to_logging_context({"num_alleles_to_link_vep": len(consequence_by_allele_id)}) logger.debug(msg="Linking VEP consequences to alleles", extra=logging_context()) - changed_allele_ids: set[int] = set() + verdicts: dict[int, VepLinkVerdict] = {} for allele_id, consequence in consequence_by_allele_id.items(): live = db.scalar( select(VepAlleleConsequence).where( @@ -253,7 +281,9 @@ def link_vep_consequences_to_alleles( ) ) - # VEP found nothing this run. Do not overwrite a held consequence with a null result. + # TODO#780 - VEP found nothing this run. Do not overwrite a held consequence with a null result; a retained + # consequence is UNCHANGED (status preexisting), while no live row at all leaves the allele out of + # the map (the caller reads that as a no-result). if consequence is None: if live is not None: logger.warning( @@ -261,6 +291,7 @@ def link_vep_consequences_to_alleles( f"'{live.functional_consequence}' in place.", extra=logging_context(), ) + verdicts[allele_id] = VepLinkVerdict.UNCHANGED continue @@ -268,6 +299,7 @@ def link_vep_consequences_to_alleles( if live is not None and live.functional_consequence == consequence: live.source_version = source_version live.access_date = access_date + verdicts[allele_id] = VepLinkVerdict.UNCHANGED continue # New or changed consequence: retire any live row, insert the successor. @@ -283,11 +315,12 @@ def link_vep_consequences_to_alleles( ], VepAlleleConsequence.allele_id == allele_id, ) - changed_allele_ids.add(allele_id) + verdicts[allele_id] = VepLinkVerdict.CREATED - save_to_logging_context({"changed_allele_count": len(changed_allele_ids)}) + changed_allele_count = sum(1 for v in verdicts.values() if v is VepLinkVerdict.CREATED) + save_to_logging_context({"changed_allele_count": changed_allele_count}) logger.info( - msg=f"Created or superseded {len(changed_allele_ids)} VEP allele consequences this run.", + msg=f"Created or superseded {changed_allele_count} VEP allele consequences this run.", extra=logging_context(), ) - return changed_allele_ids + return verdicts diff --git a/src/mavedb/worker/jobs/external_services/vep.py b/src/mavedb/worker/jobs/external_services/vep.py index 02f6a4498..cf3757335 100644 --- a/src/mavedb/worker/jobs/external_services/vep.py +++ b/src/mavedb/worker/jobs/external_services/vep.py @@ -8,8 +8,10 @@ import asyncio import logging import os +from collections import Counter from datetime import date +import requests from sqlalchemy import select from mavedb.lib.annotation_status_manager import AnnotationStatusManager @@ -18,13 +20,16 @@ from mavedb.lib.utils import batched from mavedb.lib.vep import ( VEP_CONSEQUENCES, + VepLinkVerdict, + VepResolution, get_ensembl_release, get_functional_consequence, link_vep_consequences_to_alleles, run_variant_recoder, ) from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationFailureCategory, AnnotationStatus +from mavedb.models.enums.disposition import Disposition +from mavedb.models.enums.event_reason import EventReason from mavedb.models.score_set import ScoreSet from mavedb.models.vep_allele_consequence import VepAlleleConsequence from mavedb.worker.jobs.utils.setup import validate_job_params @@ -40,36 +45,33 @@ def _annotate_vep( annotation_manager: AnnotationStatusManager, - variant_ids: list[int], - status: AnnotationStatus, + allele_id: int, + disposition: Disposition, + reason: EventReason, *, - failure_category: AnnotationFailureCategory | None = None, + source_version: str, error_message: str | None = None, metadata: dict | None = None, ) -> None: - """Fan a VEP_FUNCTIONAL_CONSEQUENCE annotation out to every variant served by an allele. + """Record one VEP_FUNCTIONAL_CONSEQUENCE event for an allele (the consequence is an allele-level fact). - AAS migration seam: the single choke point for VEP's per-variant VAS writes. At migration it - becomes an allele-keyed event writer; the per-variant fan-out goes away, and the variant - association narrows to provenance (which variant's mapping drove the linkage). See - docs/design/allele-annotation-status.md. - - VEP carries no version string, so the VAS row is written with ``version=None`` (unlike gnomAD, - which keys on its data version). + The single choke point for VEP's status writes. One event per allele, stamped with the Ensembl + release queried; provenance (which variants drove the linkage) is derived from the live links + as-of the event, not fanned out here. The consequence value itself is not embedded — it lives in + the ``VepAlleleConsequence`` value table, joinable by ``allele_id`` + ``source_version``. """ - annotation_data: dict = {"annotation_metadata": metadata or {}} + meta = dict(metadata or {}) if error_message is not None: - annotation_data["error_message"] = error_message - - for variant_id in variant_ids: - annotation_manager.add_annotation( - variant_id=variant_id, - annotation_type=AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, - status=status, - failure_category=failure_category, - annotation_data=annotation_data, - current=True, - ) + meta["error_message"] = error_message + + annotation_manager.record_event( + AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, + allele_id=allele_id, + disposition=disposition, + reason=reason, + source_version=source_version, + metadata=meta or None, + ) def _vep_hgvs_payload(row: ScoreSetAlleleRow) -> str | None: @@ -79,22 +81,40 @@ def _vep_hgvs_payload(row: ScoreSetAlleleRow) -> str | None: return row.hgvs_g or row.hgvs_c or row.hgvs_p or None -async def _resolve_consequences(unique_hgvs: list[str], job_manager: JobManager) -> dict[str, str]: - """Resolve a set of HGVS strings to their most-severe VEP consequence. +async def _resolve_consequences(unique_hgvs: list[str], job_manager: JobManager) -> VepResolution: + """Resolve a set of HGVS strings to their most-severe VEP consequence, splitting empty from errored. Phase 1 submits the HGVS strings to VEP. Phase 2 runs Variant Recoder on the misses (a VEP entry with a null consequence is treated as a miss — VEP knew the variant but could not classify it). Phase 3 re-submits the recoded genomic strings to VEP and maps the most-severe consequence back to - the original HGVS. Returns only the HGVS strings that resolved to a consequence; absent keys are - failures the caller treats as no-result. + the original HGVS. + + Returns a :class:`VepResolution`: ``consequences`` for the hits, and ``errored`` for HGVS whose VEP + or Recoder *request failed* (HTTP/transport error after retries) — those are unknown and must be + retried, never conflated with a genuine empty. An input in neither set was queried successfully and + yielded no consequence (a genuine empty). A failed batch does not abort the run: only that batch's + inputs are marked errored and processing continues. """ all_consequences: dict[str, str] = {} + errored: set[str] = set() batches = list(batched(unique_hgvs, _VEP_BATCH_SIZE)) # --- Phase 1: initial VEP pass --- all_missing_hgvs: set[str] = set() for batch_idx, batch in enumerate(batches): - consequences = await get_functional_consequence(list(batch)) + try: + consequences = await get_functional_consequence(list(batch)) + except requests.exceptions.RequestException as exc: + # The request failed — we do not know these alleles' consequence. Mark errored and do + # not route to Recoder or count as a genuine empty. + logger.warning( + msg=f"VEP request failed for batch {batch_idx + 1}/{len(batches)} ({len(batch)} HGVS); marking errored.", + exc_info=exc, + extra=job_manager.logging_context(), + ) + errored.update(batch) + continue + hit_consequences = {h: c for h, c in consequences.items() if c is not None} all_consequences.update(hit_consequences) all_missing_hgvs.update(set(batch) - set(hit_consequences.keys())) @@ -111,7 +131,7 @@ async def _resolve_consequences(unique_hgvs: list[str], job_manager: JobManager) ) if not all_missing_hgvs: - return all_consequences + return VepResolution(all_consequences, errored) # --- Phase 2: Variant Recoder fallback for HGVS strings VEP could not resolve --- recoder_batch_list = list(batched(list(all_missing_hgvs), _RECODER_BATCH_SIZE)) @@ -136,18 +156,19 @@ async def _recoder_with_semaphore(batch: list[str], total: int) -> dict[str, lis return_exceptions=True, ) - first_exception = next((r for r in recoder_results if isinstance(r, BaseException)), None) - if first_exception is not None: - successful_batches = sum(1 for r in recoder_results if not isinstance(r, BaseException)) - logger.error( - msg=f"Variant Recoder error ({successful_batches}/{total_recoder_batches} batches succeeded): {first_exception}", - extra=job_manager.logging_context(), - ) - raise first_exception - + # A failed Recoder batch marks its inputs errored (they were misses; we still don't know them) and + # the run continues — one bad batch must not abort every allele's annotation. hgvs_to_genomic: dict[str, list[str]] = {} - for result in recoder_results: - hgvs_to_genomic.update(result) # type: ignore[arg-type] + for batch, result in zip(recoder_batch_list, recoder_results): + if isinstance(result, BaseException): + logger.warning( + msg=f"Variant Recoder request failed for a batch of {len(batch)} HGVS; marking errored.", + exc_info=result, + extra=job_manager.logging_context(), + ) + errored.update(batch) + continue + hgvs_to_genomic.update(result) logger.info( msg=f"Completed Variant Recoder processing. {len(hgvs_to_genomic)} HGVS strings successfully recoded.", @@ -158,23 +179,37 @@ async def _recoder_with_semaphore(batch: list[str], total: int) -> dict[str, lis all_recoded_genomic_hgvs = list({g for genomic_list in hgvs_to_genomic.values() for g in genomic_list}) recoded_vep_batch_list = list(batched(all_recoded_genomic_hgvs, _VEP_BATCH_SIZE)) all_recoded_consequences: dict[str, str | None] = {} + errored_genomic: set[str] = set() for recoded_idx, recoded_batch in enumerate(recoded_vep_batch_list): - all_recoded_consequences.update(await get_functional_consequence(list(recoded_batch))) + try: + all_recoded_consequences.update(await get_functional_consequence(list(recoded_batch))) + except requests.exceptions.RequestException as exc: + logger.warning( + msg=f"VEP request failed for recoded batch {recoded_idx + 1}/{len(recoded_vep_batch_list)} " + f"({len(recoded_batch)} HGVS); marking errored.", + exc_info=exc, + extra=job_manager.logging_context(), + ) + errored_genomic.update(recoded_batch) job_manager.update_progress( 66 + int((recoded_idx + 1) / len(recoded_vep_batch_list) * 33), 100, f"Processed recoded VEP batch {recoded_idx + 1}/{len(recoded_vep_batch_list)}", ) - # Map the most-severe consequence from the recoded genomic strings back to the original HGVS. + # Map the most-severe consequence from the recoded genomic strings back to the original HGVS. An + # original HGVS is a hit if any recoded form resolved; else errored if a recoded query failed and + # nothing resolved; else a genuine empty (Recoder ran, VEP found no consequence) — left out of both. for original_hgvs, recoded_hgvs_list in hgvs_to_genomic.items(): recoded_consequences = [c for h in recoded_hgvs_list if (c := all_recoded_consequences.get(h))] most_severe = next((c for c in VEP_CONSEQUENCES if c in recoded_consequences), None) if most_severe: all_consequences[original_hgvs] = most_severe + elif any(g in errored_genomic for g in recoded_hgvs_list): + errored.add(original_hgvs) - return all_consequences + return VepResolution(all_consequences, errored) @with_pipeline_management @@ -222,14 +257,22 @@ async def populate_vep_for_score_set(ctx: dict, job_id: int, job_manager: JobMan job_manager.update_progress(0, 100, "Starting VEP consequence linkage.") logger.info(msg="Started VEP consequence linkage", extra=job_manager.logging_context()) - # One work-unit per allele (payload = HGVS; alleles without one are skipped). Covers ALL the score - # set's alleles — authoritative and RT-derived — since the genomic allele VEP is most reliable on - # is often the RT-derived one; VAS still fans only to authoritative_variant_ids (the bandaid seam). + # One work-unit per allele (payload = HGVS; alleles without one are skipped). Events are allele-keyed, + # so each allele records its own event. allele_data = group_alleles_for_annotation( get_alleles_for_score_set(job_manager.db, score_set.id), payload=_vep_hgvs_payload, ) + annotation_counts: Counter[str] = Counter( + { + "created_allele_count": 0, + "preexisting_allele_count": 0, + "absent_allele_count": 0, + "errored_allele_count": 0, + } + ) + num_alleles_with_hgvs = len(allele_data) job_manager.save_to_context({"num_alleles_to_link_vep": num_alleles_with_hgvs}) @@ -239,9 +282,7 @@ async def populate_vep_for_score_set(ctx: dict, job_id: int, job_manager: JobMan extra=job_manager.logging_context(), ) job_manager.db.flush() - return JobExecutionOutcome.succeeded( - data={"created_allele_count": 0, "preexisting_allele_count": 0, "skipped_allele_count": 0} - ) + return JobExecutionOutcome.succeeded(data=dict(annotation_counts)) all_allele_ids = set(allele_data.keys()) @@ -265,12 +306,12 @@ def alleles_at_current_release(allele_ids: set[int]) -> set[int]: ).all() ) - # Cost: skip alleles already resolved at the current Ensembl release (they cannot change without a + # Skip alleles already resolved at the current Ensembl release (they cannot change without a # release bump). force re-queries all — including alleles unchanged upstream but whose VEP_CONSEQUENCES # severity ordering we have since edited; the linker still supersedes only on a value change, so a # forced no-op writes nothing. already_current = set() if force else alleles_at_current_release(all_allele_ids) - hgvs_by_allele = {aid: allele_data[aid].payload for aid in allele_data if aid not in already_current} + hgvs_by_allele = {aid: allele_data[aid] for aid in allele_data if aid not in already_current} unique_hgvs = sorted(set(hgvs_by_allele.values())) job_manager.save_to_context( { @@ -280,12 +321,15 @@ def alleles_at_current_release(allele_ids: set[int]) -> set[int]: } ) - changed_allele_ids: set[int] = set() + verdicts: dict[int, VepLinkVerdict] = {} + errored_allele_ids: set[int] = set() if unique_hgvs: job_manager.update_progress(10, 100, f"Querying VEP for {len(unique_hgvs)} HGVS strings.") - consequences_by_hgvs = await _resolve_consequences(unique_hgvs, job_manager) - consequence_by_allele_id = {aid: consequences_by_hgvs.get(hgvs) for aid, hgvs in hgvs_by_allele.items()} - changed_allele_ids = link_vep_consequences_to_alleles( + resolution = await _resolve_consequences(unique_hgvs, job_manager) + consequence_by_allele_id = {aid: resolution.consequences.get(hgvs) for aid, hgvs in hgvs_by_allele.items()} + # Alleles whose VEP/Recoder request failed: unknown, not a negative — kept distinct from empties. + errored_allele_ids = {aid for aid, hgvs in hgvs_by_allele.items() if hgvs in resolution.errored} + verdicts = link_vep_consequences_to_alleles( job_manager.db, consequence_by_allele_id, source_version=ensembl_release, access_date=date.today() ) job_manager.db.flush() @@ -296,58 +340,69 @@ def alleles_at_current_release(allele_ids: set[int]) -> set[int]: ) job_manager.update_progress(99, 100, "All alleles already current at this Ensembl release.") - # Status is an audit event, written every run: SUCCESS/created if linked this run, SUCCESS/preexisting - # if already current or re-confirmed unchanged, SKIPPED if no live consequence (VEP had no result). - live_now = job_manager.db.execute( - select(VepAlleleConsequence.allele_id, VepAlleleConsequence.functional_consequence) - .where(VepAlleleConsequence.allele_id.in_(all_allele_ids)) - .where(VepAlleleConsequence.current) - .where(VepAlleleConsequence.functional_consequence.isnot(None)) - ).all() - consequence_now = {r.allele_id: r.functional_consequence for r in live_now} - - annotation_manager = AnnotationStatusManager(job_manager.db, job_run_id=job_manager.job_id) - created_count = preexisting_count = skipped_count = 0 - for allele_id, entry in allele_data.items(): - if allele_id in consequence_now: - action = "created" if allele_id in changed_allele_ids else "preexisting" - if action == "created": - created_count += 1 - else: - preexisting_count += 1 + annotation_manager = AnnotationStatusManager( + job_manager.db, job_run_id=job_manager.job_id, score_set_id=score_set.id + ) + for allele_id, hgvs in allele_data.items(): + verdict = verdicts.get(allele_id) + if verdict is VepLinkVerdict.CREATED: + annotation_counts["created_allele_count"] += 1 _annotate_vep( annotation_manager, - entry.authoritative_variant_ids, - AnnotationStatus.SUCCESS, - metadata={ - "functional_consequence": consequence_now[allele_id], - "hgvs": entry.payload, - "action": action, - }, + allele_id, + Disposition.PRESENT, + EventReason.CREATED, + source_version=ensembl_release, + metadata={"hgvs": hgvs}, ) + + elif allele_id in already_current or verdict is VepLinkVerdict.UNCHANGED: + annotation_counts["preexisting_allele_count"] += 1 + _annotate_vep( + annotation_manager, + allele_id, + Disposition.PRESENT, + EventReason.PREEXISTING, + source_version=ensembl_release, + metadata={"hgvs": hgvs}, + ) + + elif allele_id in errored_allele_ids: + annotation_counts["errored_allele_count"] += 1 + _annotate_vep( + annotation_manager, + allele_id, + Disposition.FAILED, + EventReason.API_ERROR, + source_version=ensembl_release, + error_message="The VEP/Variant Recoder request for this allele failed; result unknown.", + metadata={"hgvs": hgvs}, + ) + else: - skipped_count += 1 + annotation_counts["absent_allele_count"] += 1 _annotate_vep( annotation_manager, - entry.authoritative_variant_ids, - AnnotationStatus.SKIPPED, - failure_category=AnnotationFailureCategory.EXTERNAL_REFERENCE_NOT_FOUND, - error_message="VEP could not determine a functional consequence for this allele, even after Variant Recoder fallback.", - metadata={"hgvs": entry.payload}, + allele_id, + Disposition.ABSENT, + EventReason.NO_RECORD, + source_version=ensembl_release, + error_message="VEP found no functional consequence for this allele, even after Variant Recoder fallback.", + metadata={"hgvs": hgvs}, ) annotation_manager.flush() - outcome_data = { - "created_allele_count": created_count, - "preexisting_allele_count": preexisting_count, - "skipped_allele_count": skipped_count, - } + outcome_data = dict(annotation_counts) job_manager.save_to_context(outcome_data) job_manager.update_progress( 100, 100, - f"Completed VEP linkage: {created_count} created, {preexisting_count} preexisting, {skipped_count} skipped.", + ( + f"Completed VEP linkage: {annotation_counts['created_allele_count'] + annotation_counts['preexisting_allele_count']} linked, " + f"{annotation_counts['absent_allele_count']} absent (no result), " + f"{annotation_counts['errored_allele_count']} errored." + ), ) logger.info(msg="Done linking VEP consequences to alleles.", extra=job_manager.logging_context()) job_manager.db.flush() diff --git a/tests/lib/test_vep.py b/tests/lib/test_vep.py index 0d9473f1b..bea141ab2 100644 --- a/tests/lib/test_vep.py +++ b/tests/lib/test_vep.py @@ -11,6 +11,7 @@ from sqlalchemy import select from mavedb.lib.vep import ( + VepLinkVerdict, get_ensembl_release, get_functional_consequence, link_vep_consequences_to_alleles, @@ -290,12 +291,12 @@ def test_link_vep_creates_new_consequence(session): """A consequence for an allele with no live row creates a single live row and is reported changed.""" allele = _make_allele(session, vrs_digest="vrs-1") - changed = link_vep_consequences_to_alleles( + verdicts = link_vep_consequences_to_alleles( session, {allele.id: "missense_variant"}, source_version="116", access_date=date.today() ) session.commit() - assert changed == {allele.id} + assert verdicts == {allele.id: VepLinkVerdict.CREATED} live = _live_rows_for(session, allele.id) assert len(live) == 1 assert live[0].functional_consequence == "missense_variant" @@ -305,7 +306,8 @@ def test_link_vep_creates_new_consequence(session): def test_link_vep_unchanged_bumps_version_and_date_in_place(session): """Re-confirming an unchanged consequence at a new release advances source_version and access_date - in place — no supersede, no new valid-time boundary, and the allele is not reported changed.""" + in place — no supersede, no new valid-time boundary. The allele is reported UNCHANGED (status + preexisting) so the caller need not re-query consequence state.""" allele = _make_allele(session, vrs_digest="vrs-1") session.add( VepAlleleConsequence( @@ -317,12 +319,12 @@ def test_link_vep_unchanged_bumps_version_and_date_in_place(session): ) session.commit() - changed = link_vep_consequences_to_alleles( + verdicts = link_vep_consequences_to_alleles( session, {allele.id: "missense_variant"}, source_version="116", access_date=date.today() ) session.commit() - assert changed == set() + assert verdicts == {allele.id: VepLinkVerdict.UNCHANGED} # One row, still live, never retired — version and access_date advanced in place. all_rows = _all_rows_for(session, allele.id) assert len(all_rows) == 1 @@ -345,12 +347,12 @@ def test_link_vep_changed_consequence_supersedes(session): ) session.commit() - changed = link_vep_consequences_to_alleles( + verdicts = link_vep_consequences_to_alleles( session, {allele.id: "missense_variant"}, source_version="116", access_date=date.today() ) session.commit() - assert changed == {allele.id} + assert verdicts == {allele.id: VepLinkVerdict.CREATED} live = _live_rows_for(session, allele.id) assert len(live) == 1 assert live[0].functional_consequence == "missense_variant" @@ -363,7 +365,8 @@ def test_link_vep_changed_consequence_supersedes(session): def test_link_vep_none_leaves_live_row_untouched(session): """A transient None result must not overwrite a held consequence: the live row is left intact - (value, version, and date) and the allele is not reported changed.""" + (value, version, and date). The held consequence is reported UNCHANGED (status preexisting) — the + allele still has a live consequence, it just was not re-confirmed this run.""" allele = _make_allele(session, vrs_digest="vrs-1") session.add( VepAlleleConsequence( @@ -375,12 +378,12 @@ def test_link_vep_none_leaves_live_row_untouched(session): ) session.commit() - changed = link_vep_consequences_to_alleles( + verdicts = link_vep_consequences_to_alleles( session, {allele.id: None}, source_version="116", access_date=date.today() ) session.commit() - assert changed == set() + assert verdicts == {allele.id: VepLinkVerdict.UNCHANGED} live = _live_rows_for(session, allele.id) assert len(live) == 1 assert live[0].functional_consequence == "missense_variant" @@ -390,14 +393,15 @@ def test_link_vep_none_leaves_live_row_untouched(session): def test_link_vep_none_with_no_live_row_writes_nothing(session): - """A None result for an allele with no live row writes nothing (the allele is re-queried next run), - mirroring gnomAD's no-match handling.""" + """A None result for an allele with no live row writes nothing and leaves the allele out of the + verdict map (the caller reads that as a no-result and re-queries next run), mirroring gnomAD's + no-match handling.""" allele = _make_allele(session, vrs_digest="vrs-1") - changed = link_vep_consequences_to_alleles( + verdicts = link_vep_consequences_to_alleles( session, {allele.id: None}, source_version="116", access_date=date.today() ) session.commit() - assert changed == set() + assert verdicts == {} assert len(_all_rows_for(session, allele.id)) == 0 diff --git a/tests/worker/jobs/external_services/network/test_vep.py b/tests/worker/jobs/external_services/network/test_vep.py index 0a52012af..61ba8dfd4 100644 --- a/tests/worker/jobs/external_services/network/test_vep.py +++ b/tests/worker/jobs/external_services/network/test_vep.py @@ -7,8 +7,9 @@ from sqlalchemy import select from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationStatus, JobStatus, PipelineStatus -from mavedb.models.variant_annotation_status import VariantAnnotationStatus +from mavedb.models.enums.disposition import Disposition +from mavedb.models.enums.job_pipeline import JobStatus, PipelineStatus +from mavedb.models.annotation_event import AnnotationEvent from mavedb.models.vep_allele_consequence import VepAlleleConsequence pytestmark = pytest.mark.usefixtures("patch_db_session_ctxmgr") @@ -52,14 +53,14 @@ async def test_populate_vep_e2e( assert live.functional_consequence is not None assert live.access_date is not None - annotation = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.variant_id == variant.id, - VariantAnnotationStatus.annotation_type == AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, - VariantAnnotationStatus.current.is_(True), + # Events are allele-keyed now; the variant resolves its status through the live link. + event = session.scalars( + select(AnnotationEvent).where( + AnnotationEvent.allele_id == allele.id, + AnnotationEvent.annotation_type == AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, ) ).one() - assert annotation.status == AnnotationStatus.SUCCESS + assert event.disposition == Disposition.PRESENT async def test_populate_vep_e2e_with_recoder_path( self, @@ -97,11 +98,11 @@ async def test_populate_vep_e2e_with_recoder_path( assert live.functional_consequence is not None assert live.access_date is not None - annotation = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.variant_id == variant.id, - VariantAnnotationStatus.annotation_type == AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, - VariantAnnotationStatus.current.is_(True), + # Events are allele-keyed now; the variant resolves its status through the live link. + event = session.scalars( + select(AnnotationEvent).where( + AnnotationEvent.allele_id == allele.id, + AnnotationEvent.annotation_type == AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, ) ).one() - assert annotation.status == AnnotationStatus.SUCCESS + assert event.disposition == Disposition.PRESENT diff --git a/tests/worker/jobs/external_services/test_vep.py b/tests/worker/jobs/external_services/test_vep.py index 7e33f24fa..19587f5c2 100644 --- a/tests/worker/jobs/external_services/test_vep.py +++ b/tests/worker/jobs/external_services/test_vep.py @@ -11,8 +11,9 @@ from mavedb.lib.types.workflow import JobExecutionOutcome from mavedb.models.enums.job_pipeline import JobStatus, PipelineStatus -from mavedb.models.variant_annotation_status import VariantAnnotationStatus +from mavedb.models.annotation_event import AnnotationEvent from mavedb.models.vep_allele_consequence import VepAlleleConsequence +from mavedb.lib.vep import VepResolution from mavedb.worker.jobs.external_services.vep import populate_vep_for_score_set from mavedb.worker.lib.managers.job_manager import JobManager @@ -64,7 +65,8 @@ async def test_no_alleles_with_hgvs( assert result.status == JobStatus.SUCCEEDED assert result.data["created_allele_count"] == 0 assert result.data["preexisting_allele_count"] == 0 - assert result.data["skipped_allele_count"] == 0 + assert result.data["absent_allele_count"] == 0 + assert result.data["errored_allele_count"] == 0 async def test_calls_resolver_when_alleles_present( self, @@ -76,7 +78,7 @@ async def test_calls_resolver_when_alleles_present( setup_sample_alleles_for_vep, ): """The VEP resolver is invoked once when the score set has HGVS-bearing alleles to query.""" - with patch(_RESOLVE, return_value={}) as mock_resolve: + with patch(_RESOLVE, return_value=VepResolution({}, set())) as mock_resolve: result = await populate_vep_for_score_set( mock_worker_ctx, 1, @@ -152,7 +154,7 @@ async def test_no_alleles_with_hgvs( assert result.status == JobStatus.SUCCEEDED assert len(session.scalars(select(VepAlleleConsequence)).all()) == 0 - assert len(session.query(VariantAnnotationStatus).all()) == 0 + assert len(session.scalars(select(AnnotationEvent)).all()) == 0 session.refresh(sample_populate_vep_run) assert sample_populate_vep_run.status == JobStatus.SUCCEEDED @@ -166,19 +168,52 @@ async def test_no_consequence_resolved( sample_populate_vep_run, setup_sample_alleles_for_vep, ): - """An allele with HGVS that VEP cannot classify gets no consequence row and a SKIPPED VAS.""" + """A genuine empty (VEP queried successfully, found nothing) writes no consequence row and an + ABSENT/no_record event — a trustworthy negative, now that empty is split from a failed request.""" _, allele = setup_sample_alleles_for_vep - with patch(_RESOLVE, return_value={}): + with patch(_RESOLVE, return_value=VepResolution({}, set())): result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) assert result.status == JobStatus.SUCCEEDED + assert result.data["absent_allele_count"] == 1 assert len(_live_consequences_for(session, allele.id)) == 0 - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == "skipped" - assert annotation_statuses[0].annotation_type == "vep_functional_consequence" + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 1 + assert events[0].disposition == "absent" + assert events[0].reason == "no_record" + assert events[0].annotation_type == "vep_functional_consequence" + assert events[0].allele_id == allele.id and events[0].variant_id is None + + async def test_request_failure_recorded_as_errored( + self, + session, + with_populated_domain_data, + with_populate_vep_job, + mock_worker_ctx, + sample_populate_vep_run, + setup_sample_alleles_for_vep, + ): + """An allele whose VEP/Recoder request *failed* is FAILED/api_error — distinct from a genuine + empty — so a transient outage is never mistaken for a confirmed absence.""" + _, allele = setup_sample_alleles_for_vep + + # Resolver reports the allele's HGVS as errored (request failed), not as a hit or an empty. + with patch(_RESOLVE, return_value=VepResolution({}, {allele.hgvs_c})): + result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) + + assert result.status == JobStatus.SUCCEEDED + assert result.data["errored_allele_count"] == 1 + assert result.data["absent_allele_count"] == 0 + assert len(_live_consequences_for(session, allele.id)) == 0 + + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 1 + assert events[0].disposition == "failed" + assert events[0].reason == "api_error" + assert events[0].annotation_type == "vep_functional_consequence" + assert events[0].allele_id == allele.id and events[0].variant_id is None async def test_successful_linking_independent( self, @@ -192,7 +227,7 @@ async def test_successful_linking_independent( """A resolved consequence creates a single live VepAlleleConsequence and a SUCCESS VAS row.""" _, allele = setup_sample_alleles_for_vep - with patch(_RESOLVE, return_value={allele.hgvs_c: "missense_variant"}): + with patch(_RESOLVE, return_value=VepResolution({allele.hgvs_c: "missense_variant"}, set())): result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) assert result.status == JobStatus.SUCCEEDED @@ -204,13 +239,14 @@ async def test_successful_linking_independent( assert live[0].source_version == _ENSEMBL_RELEASE assert live[0].access_date == date.today() - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == "success" - assert annotation_statuses[0].annotation_type == "vep_functional_consequence" - assert annotation_statuses[0].annotation_metadata["action"] == "created" + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 1 + assert events[0].disposition == "present" + assert events[0].reason == "created" + assert events[0].annotation_type == "vep_functional_consequence" + assert events[0].allele_id == allele.id and events[0].variant_id is None - async def test_links_rt_derived_allele_but_annotates_only_authoritative( + async def test_links_and_annotates_rt_derived_allele( self, session, with_populated_domain_data, @@ -219,15 +255,16 @@ async def test_links_rt_derived_allele_but_annotates_only_authoritative( sample_populate_vep_run, setup_rt_derived_allele_for_vep, ): - """VEP linkage must cover the full allele set, not just authoritative links: the RT-derived - allele's genomic HGVS resolves and must get a consequence row. Per-variant VAS, however, is - written only for the authoritative link (the interim bandaid) — so exactly one annotation row - is produced, keyed to the variant, never a second row for the RT-derived allele.""" + """VEP linkage covers the full allele set, not just authoritative links: the RT-derived allele's + genomic HGVS resolves and gets a consequence row. Events are allele-keyed, so each allele in the + score set gets its own event — present for the resolved RT allele, absent/no_record for the + authoritative allele VEP queried but found nothing (a genuine empty, distinct from a request + failure). The per-variant bandaid's limitation (dropping the RT-derived allele's status) is lifted.""" variant, authoritative_allele, rt_allele = setup_rt_derived_allele_for_vep # VEP resolves only the RT-derived allele's genomic HGVS; the authoritative allele's coding - # HGVS yields nothing this run. - with patch(_RESOLVE, return_value={rt_allele.hgvs_g: "missense_variant"}): + # HGVS is queried successfully but yields no consequence this run (a genuine empty, not errored). + with patch(_RESOLVE, return_value=VepResolution({rt_allele.hgvs_g: "missense_variant"}, set())): result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) assert result.status == JobStatus.SUCCEEDED @@ -239,13 +276,15 @@ async def test_links_rt_derived_allele_but_annotates_only_authoritative( # The authoritative allele's HGVS had no consequence, so it gets no row. assert len(_live_consequences_for(session, authoritative_allele.id)) == 0 - # Annotation status is written only for the authoritative link: exactly one VAS row, for the - # variant. (Its status is "skipped" because the variant's authoritative allele had no - # consequence — cross-level resolution onto the RT-derived allele is deferred to AnnotationEvent.) - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].variant_id == variant.id - assert annotation_statuses[0].annotation_type == "vep_functional_consequence" + # One allele-keyed event per allele (never per-variant): the resolved RT allele is present, the + # unclassifiable authoritative allele is absent/no_record (queried, genuinely empty). + events = session.scalars(select(AnnotationEvent)).all() + assert all(e.annotation_type == "vep_functional_consequence" for e in events) + assert all(e.variant_id is None for e in events) + by_allele = {e.allele_id: e for e in events} + assert by_allele[rt_allele.id].disposition == "present" + assert by_allele[authoritative_allele.id].disposition == "absent" + assert by_allele[authoritative_allele.id].reason == "no_record" async def test_skips_allele_already_at_current_release( self, @@ -283,10 +322,10 @@ async def test_skips_allele_already_at_current_release( assert len(rows) == 1 assert rows[0].valid_to is None - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == "success" - assert annotation_statuses[0].annotation_metadata["action"] == "preexisting" + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 1 + assert events[0].disposition == "present" + assert events[0].reason == "preexisting" async def test_new_release_same_consequence_bumps_in_place( self, @@ -312,7 +351,7 @@ async def test_new_release_same_consequence_bumps_in_place( ) session.commit() - with patch(_RESOLVE, return_value={allele.hgvs_c: "missense_variant"}) as mock_resolve: + with patch(_RESOLVE, return_value=VepResolution({allele.hgvs_c: "missense_variant"}, set())) as mock_resolve: result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) mock_resolve.assert_called_once() # older release -> not skipped, re-queried @@ -354,7 +393,7 @@ async def test_force_requeries_unchanged_without_churn( sample_populate_vep_run.job_params = {**sample_populate_vep_run.job_params, "force": True} session.commit() - with patch(_RESOLVE, return_value={allele.hgvs_c: "missense_variant"}) as mock_resolve: + with patch(_RESOLVE, return_value=VepResolution({allele.hgvs_c: "missense_variant"}, set())) as mock_resolve: result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) mock_resolve.assert_called_once() # force bypassed the current-release skip @@ -391,7 +430,7 @@ async def test_new_release_changed_consequence_supersedes( ) session.commit() - with patch(_RESOLVE, return_value={allele.hgvs_c: "missense_variant"}): + with patch(_RESOLVE, return_value=VepResolution({allele.hgvs_c: "missense_variant"}, set())): result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) assert result.data["created_allele_count"] == 1 @@ -420,16 +459,18 @@ async def test_successful_linking_pipeline( """End-to-end successful linking within a pipeline updates both job and pipeline status.""" _, allele = setup_sample_alleles_for_vep - with patch(_RESOLVE, return_value={allele.hgvs_c: "missense_variant"}): + with patch(_RESOLVE, return_value=VepResolution({allele.hgvs_c: "missense_variant"}, set())): result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run_pipeline.id) assert result.status == JobStatus.SUCCEEDED assert len(session.scalars(select(VepAlleleConsequence).where(VepAlleleConsequence.current)).all()) == 1 - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == "success" - assert annotation_statuses[0].annotation_type == "vep_functional_consequence" + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 1 + assert events[0].disposition == "present" + assert events[0].reason == "created" + assert events[0].annotation_type == "vep_functional_consequence" + assert events[0].allele_id is not None and events[0].variant_id is None session.refresh(sample_populate_vep_run_pipeline) assert sample_populate_vep_run_pipeline.status == JobStatus.SUCCEEDED @@ -479,17 +520,19 @@ async def test_populate_vep_with_arq_context_independent( """The VEP job links a consequence and records a SUCCESS annotation through the ARQ worker.""" _, allele = setup_sample_alleles_for_vep - with patch(_RESOLVE, return_value={allele.hgvs_c: "missense_variant"}): + with patch(_RESOLVE, return_value=VepResolution({allele.hgvs_c: "missense_variant"}, set())): await arq_redis.enqueue_job("populate_vep_for_score_set", sample_populate_vep_run.id) await arq_worker.async_run() await arq_worker.run_check() assert len(session.scalars(select(VepAlleleConsequence).where(VepAlleleConsequence.current)).all()) == 1 - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == "success" - assert annotation_statuses[0].annotation_type == "vep_functional_consequence" + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 1 + assert events[0].disposition == "present" + assert events[0].reason == "created" + assert events[0].annotation_type == "vep_functional_consequence" + assert events[0].allele_id is not None and events[0].variant_id is None session.refresh(sample_populate_vep_run) assert sample_populate_vep_run.status == JobStatus.SUCCEEDED @@ -507,7 +550,7 @@ async def test_populate_vep_with_arq_context_pipeline( """The VEP job completes and advances the pipeline through the ARQ worker.""" _, allele = setup_sample_alleles_for_vep - with patch(_RESOLVE, return_value={allele.hgvs_c: "missense_variant"}): + with patch(_RESOLVE, return_value=VepResolution({allele.hgvs_c: "missense_variant"}, set())): await arq_redis.enqueue_job("populate_vep_for_score_set", sample_populate_vep_run_pipeline.id) await arq_worker.async_run() await arq_worker.run_check() @@ -540,7 +583,7 @@ async def test_populate_vep_with_arq_context_exception_handling_independent( mock_send_slack_job_error.assert_called_once() assert len(session.scalars(select(VepAlleleConsequence)).all()) == 0 - assert len(session.query(VariantAnnotationStatus).all()) == 0 + assert len(session.scalars(select(AnnotationEvent)).all()) == 0 session.refresh(sample_populate_vep_run) assert sample_populate_vep_run.status == JobStatus.ERRORED @@ -566,7 +609,7 @@ async def test_populate_vep_with_arq_context_exception_handling_pipeline( mock_send_slack_job_error.assert_called_once() assert len(session.scalars(select(VepAlleleConsequence)).all()) == 0 - assert len(session.query(VariantAnnotationStatus).all()) == 0 + assert len(session.scalars(select(AnnotationEvent)).all()) == 0 session.refresh(sample_populate_vep_run_pipeline) assert sample_populate_vep_run_pipeline.status == JobStatus.ERRORED From ed50102b7f13ba2cff4dfab57eb3d27b171ead5a Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 30 Jun 2026 08:30:55 -0700 Subject: [PATCH 039/252] feat(clingen): emit annotation events from CAR/LDH jobs The ClinGen allele-registration and LDH-submission jobs now record AnnotationEvents via record_event, using the shared EventReason vocabulary (created/preexisting/ reconfirmed/submitted, and the CAR-specific failure and reconfirmation-skipped reasons) in place of per-variant status updates. --- .../worker/jobs/external_services/clingen.py | 220 ++++++------ .../jobs/external_services/test_clingen.py | 316 +++++++++--------- 2 files changed, 268 insertions(+), 268 deletions(-) diff --git a/src/mavedb/worker/jobs/external_services/clingen.py b/src/mavedb/worker/jobs/external_services/clingen.py index 42cdcb36a..768d13278 100644 --- a/src/mavedb/worker/jobs/external_services/clingen.py +++ b/src/mavedb/worker/jobs/external_services/clingen.py @@ -11,7 +11,8 @@ import asyncio import functools import logging -from dataclasses import dataclass, field +from collections.abc import Sequence +from dataclasses import dataclass from sqlalchemy import select @@ -32,7 +33,9 @@ from mavedb.lib.variants import get_hgvs_from_post_mapped from mavedb.models.allele import Allele as AlleleModel from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationFailureCategory, AnnotationStatus, FailureCategory +from mavedb.models.enums.disposition import Disposition +from mavedb.models.enums.event_reason import EventReason +from mavedb.models.enums.job_pipeline import FailureCategory from mavedb.models.mapping_record import MappingRecord from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.score_set import ScoreSet @@ -48,44 +51,33 @@ class _AlleleEntry: post_mapped: dict | None existing_caid: str | None - # Variants for which THIS allele is the authoritative measurement — the only ones that receive a - # per-variant VAS row. INTERIM BANDAID (do not deploy as final): keying clingen's per-variant - # status to the single authoritative link sidesteps the multiple "current" rows a full allele - # fan-out would write for one variant. Durable fix is an allele-level event log; rationale and - # migration seam in docs/design/allele-annotation-status.md. - authoritative_variant_ids: list[int] = field(default_factory=list) def _annotate_caid( annotation_manager: AnnotationStatusManager, - variant_ids: list[int], - status: AnnotationStatus, + allele_id: int, + disposition: Disposition, + reason: EventReason, *, - failure_category: AnnotationFailureCategory | None = None, error_message: str | None = None, metadata: dict | None = None, ) -> None: - """Fan a CLINGEN_ALLELE_ID annotation out to every variant served by an allele. + """Record one CLINGEN_ALLELE_ID event for an allele (the CAID is an allele-level fact). - AAS migration seam: the single choke point for clingen's per-variant VAS writes. At migration it - becomes an allele-keyed event writer; the per-variant fan-out goes away, and the variant - association narrows to provenance (who caused the registration). See - docs/design/allele-annotation-status.md. + The single choke point for CAR's status writes. Provenance (which variants drove the + registration) is derived from the live links as-of the event, not fanned out here. """ - annotation_data: dict = {"annotation_metadata": metadata or {}} + meta = dict(metadata or {}) if error_message is not None: - annotation_data["error_message"] = error_message - - for variant_id in variant_ids: - annotation_manager.add_annotation( - variant_id=variant_id, - annotation_type=AnnotationType.CLINGEN_ALLELE_ID, - version=None, - status=status, - failure_category=failure_category, - annotation_data=annotation_data, - current=True, - ) + meta["error_message"] = error_message + + annotation_manager.record_event( + AnnotationType.CLINGEN_ALLELE_ID, + allele_id=allele_id, + disposition=disposition, + reason=reason, + metadata=meta or None, + ) @with_pipeline_management @@ -172,53 +164,59 @@ async def submit_score_set_mappings_to_car(ctx: dict, job_id: int, job_manager: if row.allele_id not in allele_data: allele_data[row.allele_id] = _AlleleEntry(post_mapped=row.post_mapped, existing_caid=row.clingen_allele_id) - if row.is_authoritative: - allele_data[row.allele_id].authoritative_variant_ids.append(row.variant_id) - - annotation_manager = AnnotationStatusManager(job_manager.db, job_run_id=job_manager.job_id) + annotation_manager = AnnotationStatusManager( + job_manager.db, job_run_id=job_manager.job_id, score_set_id=score_set.id + ) # Track outcomes by distinct allele_id. clingen_allele_id is an allele-level fact (the CAID # lives on the Allele) and CAR's operation is per-allele, so the reported counts are in allele - # units — and they cover every allele submitted, including the RT-derived ones that produce no - # per-variant status row. Each allele has exactly one outcome (submitted once → one response), so - # these sets are disjoint by construction. (Per-variant VAS rows are still written via the - # authoritative link below — that is the interim bandaid, separate from these operation counts.) + # units — and they cover every allele submitted, including the RT-derived ones. Each allele has + # exactly one outcome (submitted once → one response), so these sets are disjoint by construction. linked_allele_ids: set[int] = set() preexisting_allele_ids: set[int] = set() failed_allele_ids: set[int] = set() + preexisting_alleles = [] + pending_alleles = [] + for aid, entry in allele_data.items(): + if entry.existing_caid: + preexisting_alleles.append(aid) + elif force_reregister: + pending_alleles.append(aid) + elif not entry.existing_caid: + pending_alleles.append(aid) + # Pre-existing CAIDs: record success without re-submitting unless force_reregister is set. - preexisting = [aid for aid, entry in allele_data.items() if entry.existing_caid] if not force_reregister else [] + preexisting = preexisting_alleles if not force_reregister else [] for allele_id in preexisting: entry = allele_data[allele_id] preexisting_allele_ids.add(allele_id) _annotate_caid( annotation_manager, - entry.authoritative_variant_ids, - AnnotationStatus.SUCCESS, - metadata={"clingen_allele_id": entry.existing_caid, "registration_source": "preexisting"}, + allele_id, + Disposition.PRESENT, + EventReason.PREEXISTING, + metadata={"clingen_allele_id": entry.existing_caid}, ) - # Alleles that need CAR submission: new ones, or all when force_reregister=True. - pending_allele_ids = [aid for aid, entry in allele_data.items() if force_reregister or not entry.existing_caid] - job_manager.update_progress(10, 100, f"Preparing {len(pending_allele_ids)} alleles for CAR submission.") + job_manager.update_progress(10, 100, f"Preparing {len(pending_alleles)} alleles for CAR submission.") # Build HGVS → [allele_ids] map. Multi-variant cis-phased blocks produce no HGVS # (combine_cis defaults to False); those alleles are annotated as failures immediately. hgvs_to_allele_ids: dict[str, list[int]] = {} - for allele_id in pending_allele_ids: + for allele_id in pending_alleles: entry = allele_data[allele_id] hgvs = get_hgvs_from_post_mapped(entry.post_mapped) if hgvs: hgvs_to_allele_ids.setdefault(hgvs, []).append(allele_id) - # Allele is registered but post_mapped can no longer produce HGVS — data - # regression worth surfacing, but the CAID is still valid so treat it as - # preexisting rather than failing the variant. + # TODO#780 - Allele is registered but post_mapped can no longer produce HGVS — data + # regression worth surfacing, but the CAID is still valid so treat it as preexisting + # rather than failing the variant. elif entry.existing_caid: - preexisting_allele_ids.add(allele_id) + preexisting_alleles.append(allele_id) logger.warning( msg=( f"Could not construct HGVS for allele {allele_id} during force re-registration " @@ -228,12 +226,14 @@ async def submit_score_set_mappings_to_car(ctx: dict, job_id: int, job_manager: ) _annotate_caid( annotation_manager, - entry.authoritative_variant_ids, - AnnotationStatus.SUCCESS, - metadata={"clingen_allele_id": entry.existing_caid, "registration_source": "reconfirmation_skipped"}, + allele_id, + Disposition.PRESENT, + EventReason.RECONFIRMATION_SKIPPED, + metadata={"clingen_allele_id": entry.existing_caid}, ) - # No HGVS-- un-submittable. + # No HGVS-- un-submittable. The allele cannot produce an identifier to register, + # so treat this as not_applicable rather than a failure. else: failed_allele_ids.add(allele_id) logger.warning( @@ -242,9 +242,9 @@ async def submit_score_set_mappings_to_car(ctx: dict, job_id: int, job_manager: ) _annotate_caid( annotation_manager, - entry.authoritative_variant_ids, - AnnotationStatus.FAILED, - failure_category=AnnotationFailureCategory.MISSING_IDENTIFIER, + allele_id, + Disposition.NOT_APPLICABLE, + EventReason.NO_HGVS, error_message="Could not extract a valid HGVS string from post-mapped allele data.", ) @@ -315,9 +315,9 @@ def _outcome_data() -> dict[str, int]: failed_allele_ids.add(allele_id) _annotate_caid( annotation_manager, - allele_data[allele_id].authoritative_variant_ids, - AnnotationStatus.FAILED, - failure_category=AnnotationFailureCategory.EXTERNAL_SERVICE_REJECTED, + allele_id, + Disposition.FAILED, + EventReason.SERVICE_REJECTED, error_message="Failed to register allele with ClinGen Allele Registry.", metadata={ "submitted_hgvs": hgvs_string, @@ -339,9 +339,9 @@ def _outcome_data() -> dict[str, int]: failed_allele_ids.add(allele_id) _annotate_caid( annotation_manager, - allele_data[allele_id].authoritative_variant_ids, - AnnotationStatus.FAILED, - failure_category=AnnotationFailureCategory.EXTERNAL_SERVICE_REJECTED, + allele_id, + Disposition.FAILED, + EventReason.MALFORMED_RESPONSE, error_message="ClinGen Allele Registry returned a malformed response with no allele identifier.", metadata={"submitted_hgvs": hgvs_string}, ) @@ -353,7 +353,7 @@ def _outcome_data() -> dict[str, int]: entry = allele_data[allele_id] prior_caid = entry.existing_caid - # CAID is immutable — a different value returned by CAR is a hard invariant + # TODO#780 - CAID is immutable — a different value returned by CAR is a hard invariant # violation. Do not overwrite; record a failure with full audit context. if prior_caid and prior_caid != caid: logger.error( @@ -368,9 +368,9 @@ def _outcome_data() -> dict[str, int]: failed_allele_ids.add(allele_id) _annotate_caid( annotation_manager, - entry.authoritative_variant_ids, - AnnotationStatus.FAILED, - failure_category=AnnotationFailureCategory.EXTERNAL_SERVICE_REJECTED, + allele_id, + Disposition.FAILED, + EventReason.CAID_CONFLICT, error_message="CAR returned a CAID that conflicts with the stored value.", metadata={ "clingen_allele_id": prior_caid, @@ -385,7 +385,7 @@ def _outcome_data() -> dict[str, int]: allele = alleles_by_id[allele_id] allele.clingen_allele_id = caid - registration_source = "reconfirmed" if prior_caid else "this_run" + reason = EventReason.RECONFIRMED if prior_caid else EventReason.CREATED if prior_caid: logger.info( msg=f"Force re-registration confirmed same CAID {caid!r} for allele {allele_id}.", @@ -394,9 +394,10 @@ def _outcome_data() -> dict[str, int]: _annotate_caid( annotation_manager, - entry.authoritative_variant_ids, - AnnotationStatus.SUCCESS, - metadata={"clingen_allele_id": caid, "registration_source": registration_source}, + allele_id, + Disposition.PRESENT, + reason, + metadata={"clingen_allele_id": caid}, ) # Submitted HGVS with no trustworthy response: the truncated tail when the counts line up @@ -408,9 +409,9 @@ def _outcome_data() -> dict[str, int]: failed_allele_ids.add(allele_id) _annotate_caid( annotation_manager, - allele_data[allele_id].authoritative_variant_ids, - AnnotationStatus.FAILED, - failure_category=AnnotationFailureCategory.EXTERNAL_API_ERROR, + allele_id, + Disposition.FAILED, + EventReason.API_ERROR, error_message="Failed to register allele with ClinGen Allele Registry.", metadata={"submitted_hgvs": hgvs_string}, ) @@ -496,17 +497,21 @@ async def submit_score_set_mappings_to_ldh(ctx: dict, job_id: int, job_manager: # MappingRecord. RT-derived equivalence alleles are intentionally excluded — LDH links each # MaveDB score to its canonical mapped variant, not to every equivalent allele (unlike CAR, # which registers a CAID per allele). - variant_objects = job_manager.db.execute( - select(Variant, MappingRecord, AlleleModel) - .join(MappingRecord, MappingRecord.variant_id == Variant.id) - .join(MappingRecordAllele, MappingRecordAllele.mapping_record_id == MappingRecord.id) - .join(AlleleModel, AlleleModel.id == MappingRecordAllele.allele_id) - .where(Variant.score_set_id == score_set.id) - .where(MappingRecord.current) - .where(MappingRecordAllele.current) - .where(MappingRecordAllele.is_authoritative.is_(True)) - .where(AlleleModel.post_mapped.is_not(None)) - ).all() + variant_objects: Sequence[tuple[Variant, MappingRecord, AlleleModel]] = ( + job_manager.db.execute( + select(Variant, MappingRecord, AlleleModel) + .join(MappingRecord, MappingRecord.variant_id == Variant.id) + .join(MappingRecordAllele, MappingRecordAllele.mapping_record_id == MappingRecord.id) + .join(AlleleModel, AlleleModel.id == MappingRecordAllele.allele_id) + .where(Variant.score_set_id == score_set.id) + .where(MappingRecord.current) + .where(MappingRecordAllele.current) + .where(MappingRecordAllele.is_authoritative.is_(True)) + .where(AlleleModel.post_mapped.is_not(None)) + ) + .tuples() + .all() + ) # Track total variants to submit job_manager.save_to_context({"total_variants_to_submit_ldh": len(variant_objects)}) @@ -521,11 +526,10 @@ async def submit_score_set_mappings_to_ldh(ctx: dict, job_id: int, job_manager: job_manager.update_progress(10, 100, f"Submitting {len(variant_objects)} mapped variants to LDH.") # Build submission content - variant_content = [] - variant_for_urn = {} + variant_content: list[tuple[str, Variant, MappingRecord, AlleleModel]] = [] + variant_for_urn: dict[str, Variant] = {} for variant, mapping_record, allele in variant_objects: - # See the note above: cis-phased blocks are skipped here pending ClinGen guidance - # (https://github.com/VariantEffect/mavedb-api/issues/764). + # cis-phased blocks are skipped here pending ClinGen guidance TODO#764 variation = get_hgvs_from_post_mapped(allele.post_mapped) if not variation: @@ -536,7 +540,8 @@ async def submit_score_set_mappings_to_ldh(ctx: dict, job_id: int, job_manager: continue variant_content.append((variation, variant, mapping_record, allele)) - variant_for_urn[variant.urn] = variant + # TODO#372: nullable URNs + variant_for_urn[variant.urn] = variant # type: ignore if not variant_content: logger.warning( @@ -561,8 +566,10 @@ async def submit_score_set_mappings_to_ldh(ctx: dict, job_id: int, job_manager: "ldh_submission_failures": len(submission_failures), }) - # TODO prior to finalizing: Verify typing of ClinGen submission responses. See https://reg.clinicalgenome.org/doc/AlleleRegistry_1.01.xx_api_v1.pdf - annotation_manager = AnnotationStatusManager(job_manager.db, job_run_id=job_manager.job_id) + # See https://reg.clinicalgenome.org/doc/AlleleRegistry_1.01.xx_api_v1.pdf + annotation_manager = AnnotationStatusManager( + job_manager.db, job_run_id=job_manager.job_id, score_set_id=score_set.id + ) submitted_variant_urns = set() for success in submission_successes: logger.debug( @@ -572,24 +579,21 @@ async def submit_score_set_mappings_to_ldh(ctx: dict, job_id: int, job_manager: submitted_urn = success["data"]["entId"] submitted_variant = variant_for_urn.get(submitted_urn) + # LDH echoed back an entId we never submitted — record it for investigation rather + # than crashing the whole job mid-batch. if submitted_variant is None: - # LDH echoed back an entId we never submitted — record it for investigation rather - # than crashing the whole job mid-batch. logger.warning( msg=f"LDH returned an unrecognized entId not in this submission: {submitted_urn!r}.", extra=job_manager.logging_context(), ) continue - annotation_manager.add_annotation( + annotation_manager.record_event( + AnnotationType.LDH_SUBMISSION, variant_id=submitted_variant.id, - annotation_type=AnnotationType.LDH_SUBMISSION, - version=None, - status=AnnotationStatus.SUCCESS, - annotation_data={ - "annotation_metadata": {"ldh_iri": success["data"]["ldhIri"], "ldh_id": success["data"]["ldhId"]}, - }, - current=True, + disposition=Disposition.PRESENT, + reason=EventReason.SUBMITTED, + metadata={"ldh_iri": success["data"]["ldhIri"], "ldh_id": success["data"]["ldhId"]}, ) submitted_variant_urns.add(submitted_urn) @@ -606,16 +610,12 @@ async def submit_score_set_mappings_to_ldh(ctx: dict, job_id: int, job_manager: failed_variant = variant_for_urn[failure_urn] - annotation_manager.add_annotation( + annotation_manager.record_event( + AnnotationType.LDH_SUBMISSION, variant_id=failed_variant.id, - annotation_type=AnnotationType.LDH_SUBMISSION, - version=None, - status=AnnotationStatus.FAILED, - failure_category=AnnotationFailureCategory.EXTERNAL_API_ERROR, - annotation_data={ - "error_message": "Failed to submit variant to ClinGen Linked Data Hub.", - }, - current=True, + disposition=Disposition.FAILED, + reason=EventReason.API_ERROR, + metadata={"error_message": "Failed to submit variant to ClinGen Linked Data Hub."}, ) annotation_manager.flush() diff --git a/tests/worker/jobs/external_services/test_clingen.py b/tests/worker/jobs/external_services/test_clingen.py index 76b1b0b2d..3b44a35c5 100644 --- a/tests/worker/jobs/external_services/test_clingen.py +++ b/tests/worker/jobs/external_services/test_clingen.py @@ -16,7 +16,7 @@ from mavedb.models.enums.job_pipeline import JobStatus, PipelineStatus from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.variant import Variant -from mavedb.models.variant_annotation_status import VariantAnnotationStatus +from mavedb.models.annotation_event import AnnotationEvent from mavedb.worker.jobs.external_services.clingen import ( submit_score_set_mappings_to_car, submit_score_set_mappings_to_ldh, @@ -153,14 +153,14 @@ async def test_submit_score_set_mappings_to_car_no_registered_alleles( alleles = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() assert len(alleles) == 0 - # Verify annotation statuses were rendered as failed — 4 variants, all failed - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + # 4 variants dedup to 1 allele → one allele-keyed event, failed (no CAR response). + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == 4 - for ann in annotation_statuses: - assert ann.status == "failed" - assert ann.annotation_type == "clingen_allele_id" + assert len(events) == 1 + for event in events: + assert event.disposition == "failed" + assert event.allele_id is not None async def test_submit_score_set_mappings_to_car_all_car_errors( self, @@ -221,16 +221,16 @@ async def test_submit_score_set_mappings_to_car_all_car_errors( alleles = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() assert len(alleles) == 0 - # 1 allele failed → all 4 variant annotations are failed - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + # 1 allele, rejected by CAR → one failed event (reason=service_rejected). + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == 4 - for ann in annotation_statuses: - assert ann.status == "failed" - assert ann.annotation_type == "clingen_allele_id" + assert len(events) == 1 + for event in events: + assert event.disposition == "failed" + assert event.reason == "service_rejected" - async def test_submit_score_set_mappings_to_car_derived_allele_no_duplicate_annotation( + async def test_submit_score_set_mappings_to_car_event_per_allele( self, mock_worker_ctx, session, @@ -243,8 +243,8 @@ async def test_submit_score_set_mappings_to_car_derived_allele_no_duplicate_anno dummy_variant_creation_job_run, dummy_variant_mapping_job_run, ): - """A variant linked to an authoritative AND a derived allele gets exactly one VAS row (from the - authoritative link), while the derived allele is still registered with a CAID.""" + """Each allele — authoritative and derived — gets exactly one allele-keyed event; there is no + per-variant fan-out. Both alleles are registered with a CAID.""" await create_mappings_in_score_set( session, mock_s3_client, @@ -300,18 +300,16 @@ def fake_dispatch(hgvs_list): assert result.status == JobStatus.SUCCEEDED - # Exactly one current VAS row per variant (4 total) — no duplicate from the derived allele. - current_statuses = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.annotation_type == "clingen_allele_id", - VariantAnnotationStatus.current.is_(True), - ) + # One event per allele (2 alleles → 2 events), each keyed on its allele, not fanned per-variant. + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(current_statuses) == 4 - assert len({s.variant_id for s in current_statuses}) == 4 + assert len(events) == 2 + assert all(e.variant_id is None for e in events) + assert {e.allele_id for e in events} == {authoritative_allele.id, derived_allele.id} + assert all(e.disposition == "present" for e in events) - # Both alleles registered: registration breadth is preserved even though the derived allele - # produced no VAS row. + # Both alleles registered: registration breadth covers the derived allele too. session.refresh(authoritative_allele) session.refresh(derived_allele) assert authoritative_allele.clingen_allele_id is not None @@ -370,13 +368,13 @@ async def test_submit_score_set_mappings_to_car_response_count_mismatch( # No CAID written — neither of the returned values is trusted. assert len(session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all()) == 0 - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == 4 - for ann in annotation_statuses: - assert ann.status == "failed" - assert ann.failure_category == "external_api_error" + assert len(events) == 1 + for event in events: + assert event.disposition == "failed" + assert event.reason == "api_error" async def test_submit_score_set_mappings_to_car_malformed_response( self, @@ -425,15 +423,15 @@ async def test_submit_score_set_mappings_to_car_malformed_response( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.FAILED - # No CAID assigned, and all 4 variant annotations are failed as rejected. + # No CAID assigned; the one allele's event is failed (reason=malformed_response). assert len(session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all()) == 0 - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == 4 - for ann in annotation_statuses: - assert ann.status == "failed" - assert ann.failure_category == "external_service_rejected" + assert len(events) == 1 + for event in events: + assert event.disposition == "failed" + assert event.reason == "malformed_response" async def test_submit_score_set_mappings_to_car_repeated_hgvs( self, @@ -492,14 +490,14 @@ async def test_submit_score_set_mappings_to_car_repeated_hgvs( assert len(alleles) == 1 assert alleles[0].clingen_allele_id == "CA_DUPLICATE" - # 4 per-variant annotations — all success - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + # 1 allele → one present event. + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == 4 - for ann in annotation_statuses: - assert ann.status == "success" - assert ann.annotation_type == "clingen_allele_id" + assert len(events) == 1 + for event in events: + assert event.disposition == "present" + assert event.allele_id is not None async def test_submit_score_set_mappings_to_car_partial_failure( self, @@ -562,13 +560,13 @@ async def test_submit_score_set_mappings_to_car_partial_failure( assert alleles_with_caid[0].clingen_allele_id == f"CA{alleles[0].id}" # All 4 variant annotations succeeded - success_annotations = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.annotation_type == "clingen_allele_id", - VariantAnnotationStatus.status == "success", + present_events = session.scalars( + select(AnnotationEvent).where( + AnnotationEvent.annotation_type == "clingen_allele_id", + AnnotationEvent.disposition == "present", ) ).all() - assert len(success_annotations) == 4 + assert len(present_events) == 1 async def test_submit_score_set_mappings_to_car_hgvs_not_found( self, @@ -614,13 +612,13 @@ async def test_submit_score_set_mappings_to_car_hgvs_not_found( assert len(alleles) == 0 # Verify annotation statuses were rendered as failed — 4 variants, all failed - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == 4 - for ann in annotation_statuses: - assert ann.status == "failed" - assert ann.annotation_type == "clingen_allele_id" + assert len(events) == 1 + for event in events: + assert event.disposition == "not_applicable" + assert event.reason == "no_hgvs" async def test_submit_score_set_mappings_to_car_propagates_exception( self, @@ -724,13 +722,13 @@ async def test_submit_score_set_mappings_to_car_success( assert alleles_with_caid[0].clingen_allele_id == f"CA{alleles[0].id}" # 4 per-variant annotations — all success - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == 4 - for ann in annotation_statuses: - assert ann.status == "success" - assert ann.annotation_type == "clingen_allele_id" + assert len(events) == 1 + for event in events: + assert event.disposition == "present" + assert event.allele_id is not None async def test_submit_score_set_mappings_to_car_preexisting( self, @@ -780,14 +778,14 @@ async def test_submit_score_set_mappings_to_car_preexisting( assert result.data["already_registered_allele_count"] == 1 assert result.data["submitted_allele_count"] == 0 - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == 4 - for ann in annotation_statuses: - assert ann.status == "success" - assert ann.annotation_metadata["registration_source"] == "preexisting" - assert ann.annotation_metadata["clingen_allele_id"] == "CA_PRIOR" + assert len(events) == 1 + for event in events: + assert event.disposition == "present" + assert event.reason == "preexisting" + assert event.event_metadata["clingen_allele_id"] == "CA_PRIOR" async def test_submit_score_set_mappings_to_car_force_reregister_same_caid( self, @@ -843,13 +841,13 @@ async def test_submit_score_set_mappings_to_car_force_reregister_same_caid( assert result.data["registered_allele_count"] == 1 assert result.data["submitted_allele_count"] == 1 - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == 4 - for ann in annotation_statuses: - assert ann.status == "success" - assert ann.annotation_metadata["registration_source"] == "reconfirmed" + assert len(events) == 1 + for event in events: + assert event.disposition == "present" + assert event.reason == "reconfirmed" async def test_submit_score_set_mappings_to_car_force_reregister_caid_conflict( self, @@ -909,14 +907,15 @@ async def test_submit_score_set_mappings_to_car_force_reregister_caid_conflict( session.refresh(allele) assert allele.clingen_allele_id == "CA_STORED" - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == 4 - for ann in annotation_statuses: - assert ann.status == "failed" - assert ann.annotation_metadata["clingen_allele_id"] == "CA_STORED" - assert ann.annotation_metadata["conflicting_caid"] == "CA_DIFFERENT" + assert len(events) == 1 + for event in events: + assert event.disposition == "failed" + assert event.reason == "caid_conflict" + assert event.event_metadata["clingen_allele_id"] == "CA_STORED" + assert event.event_metadata["conflicting_caid"] == "CA_DIFFERENT" @pytest.mark.integration @@ -979,12 +978,12 @@ async def test_submit_score_set_mappings_to_car_independent_ctx( assert alleles_with_caid[0].clingen_allele_id == f"CA{alleles[0].id}" # 4 per-variant annotations — all success - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == 4 - for ann in annotation_statuses: - assert ann.status == "success" + assert len(events) == 1 + for event in events: + assert event.disposition == "present" # Verify the job status is updated in the database session.refresh(submit_score_set_mappings_to_car_sample_job_run) @@ -1046,12 +1045,12 @@ async def test_submit_score_set_mappings_to_car_pipeline_ctx( assert alleles_with_caid[0].clingen_allele_id == f"CA{alleles[0].id}" # 4 per-variant annotations — all success - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == 4 - for ann in annotation_statuses: - assert ann.status == "success" + assert len(events) == 1 + for event in events: + assert event.disposition == "present" # Verify the job status is updated in the database session.refresh(submit_score_set_mappings_to_car_sample_job_run_in_pipeline) @@ -1090,8 +1089,8 @@ async def test_submit_score_set_mappings_to_car_submission_disabled( assert len(alleles) == 0 # Verify no annotation statuses were created - annotation_statuses = session.scalars(select(VariantAnnotationStatus)).all() - assert len(annotation_statuses) == 0 + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 0 # Verify the job status is updated in the database session.refresh(submit_score_set_mappings_to_car_sample_job_run) @@ -1129,8 +1128,8 @@ async def test_submit_score_set_mappings_to_car_no_submission_endpoint( assert len(alleles) == 0 # Verify no annotation statuses were created - annotation_statuses = session.scalars(select(VariantAnnotationStatus)).all() - assert len(annotation_statuses) == 0 + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 0 # Verify the job status is updated in the database session.refresh(submit_score_set_mappings_to_car_sample_job_run) @@ -1160,8 +1159,8 @@ async def test_submit_score_set_mappings_to_car_no_mappings( assert len(alleles) == 0 # Verify no annotation statuses were created - annotation_statuses = session.scalars(select(VariantAnnotationStatus)).all() - assert len(annotation_statuses) == 0 + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 0 # Verify the job status is updated in the database session.refresh(submit_score_set_mappings_to_car_sample_job_run) @@ -1214,10 +1213,10 @@ async def test_submit_score_set_mappings_to_car_no_registered_alleles( assert len(alleles) == 0 # Verify annotation statuses were rendered as failed — 4 variants, all failed - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == 4 + assert len(events) == 1 # Verify the job status is updated in the database session.refresh(submit_score_set_mappings_to_car_sample_job_run) @@ -1274,10 +1273,10 @@ async def test_submit_score_set_mappings_to_car_no_linked_alleles( assert len(alleles) == 0 # Verify annotation statuses were rendered as failed — 4 variants, all failed - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == 4 + assert len(events) == 1 # Verify the job status is updated in the database session.refresh(submit_score_set_mappings_to_car_sample_job_run) @@ -1344,13 +1343,13 @@ async def test_submit_score_set_mappings_to_car_partial_failure( assert alleles_with_caid[0].clingen_allele_id == f"CA{alleles[0].id}" # All 4 variant annotations succeeded - success_annotations = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.annotation_type == "clingen_allele_id", - VariantAnnotationStatus.status == "success", + present_events = session.scalars( + select(AnnotationEvent).where( + AnnotationEvent.annotation_type == "clingen_allele_id", + AnnotationEvent.disposition == "present", ) ).all() - assert len(success_annotations) == 4 + assert len(present_events) == 1 # Verify the job status is updated in the database session.refresh(submit_score_set_mappings_to_car_sample_job_run) @@ -1409,18 +1408,19 @@ async def test_submit_score_set_mappings_to_car_car_error_details_stored_in_anno standalone_worker_context, submit_score_set_mappings_to_car_sample_job_run.id ) - # All 4 variant annotations should have EXTERNAL_SERVICE_REJECTED since the 1 shared allele was rejected - car_rejected_annotations = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.annotation_type == "clingen_allele_id", - VariantAnnotationStatus.failure_category == "external_service_rejected", + # The 1 shared allele was rejected by ClinGen → one failed event (reason service_rejected). + rejected_events = session.scalars( + select(AnnotationEvent).where( + AnnotationEvent.annotation_type == "clingen_allele_id", + AnnotationEvent.reason == "service_rejected", ) ).all() - assert len(car_rejected_annotations) == 4 - for rejected in car_rejected_annotations: - assert rejected.annotation_metadata["submitted_hgvs"] == allele_hgvs - assert rejected.annotation_metadata["car_error_type"] == "InvalidHGVS" - assert rejected.annotation_metadata["car_error_message"] == "The HGVS string is invalid." + assert len(rejected_events) == 1 + for event in rejected_events: + assert event.disposition == "failed" + assert event.event_metadata["submitted_hgvs"] == allele_hgvs + assert event.event_metadata["car_error_type"] == "InvalidHGVS" + assert event.event_metadata["car_error_message"] == "The HGVS string is invalid." async def test_submit_score_set_mappings_to_car_propagates_exception_to_decorator( self, @@ -1536,12 +1536,12 @@ async def test_submit_score_set_mappings_to_car_with_arq_context_independent( assert alleles_with_caid[0].clingen_allele_id == f"CA{alleles[0].id}" # 4 per-variant annotations — all success - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == 4 - for ann in annotation_statuses: - assert ann.status == "success" + assert len(events) == 1 + for event in events: + assert event.disposition == "present" async def test_submit_score_set_mappings_to_car_with_arq_context_pipeline( self, @@ -1608,12 +1608,12 @@ async def test_submit_score_set_mappings_to_car_with_arq_context_pipeline( assert alleles_with_caid[0].clingen_allele_id == f"CA{alleles[0].id}" # 4 per-variant annotations — all success - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == 4 - for ann in annotation_statuses: - assert ann.status == "success" + assert len(events) == 1 + for event in events: + assert event.disposition == "present" async def test_submit_score_set_mappings_to_car_with_arq_context_exception_handling_independent( self, @@ -1668,10 +1668,10 @@ async def test_submit_score_set_mappings_to_car_with_arq_context_exception_handl assert len(alleles) == 0 # Verify no annotation statuses were created - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == 0 + assert len(events) == 0 async def test_submit_score_set_mappings_to_car_with_arq_context_exception_handling_pipeline( self, @@ -1731,10 +1731,10 @@ async def test_submit_score_set_mappings_to_car_with_arq_context_exception_handl assert len(alleles) == 0 # Verify no annotation statuses were created - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == 0 + assert len(events) == 0 @pytest.mark.unit @@ -2084,11 +2084,11 @@ async def dummy_ldh_submission(*args, **kwargs): # Verify annotation statuses were created annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "ldh_submission") + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "ldh_submission") ).all() assert len(annotation_statuses) == 4 for ann in annotation_statuses: - assert ann.status == "success" + assert ann.disposition == "present" # Verify the job status is updated in the database session.refresh(submit_score_set_mappings_to_ldh_sample_job_run) @@ -2155,11 +2155,11 @@ async def dummy_ldh_submission(*args, **kwargs): # Verify annotation statuses were created annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "ldh_submission") + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "ldh_submission") ).all() assert len(annotation_statuses) == 4 for ann in annotation_statuses: - assert ann.status == "success" + assert ann.disposition == "present" # Verify the job status is updated in the database session.refresh(submit_score_set_mappings_to_ldh_sample_job_run_in_pipeline) @@ -2262,11 +2262,11 @@ async def dummy_no_linked_alleles_submission(*args, **kwargs): # Verify annotation statuses were created with failures annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "ldh_submission") + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "ldh_submission") ).all() assert len(annotation_statuses) == 4 for ann in annotation_statuses: - assert ann.status == "failed" + assert ann.disposition == "failed" # Verify the job status is updated in the database session.refresh(submit_score_set_mappings_to_ldh_sample_job_run) @@ -2310,7 +2310,7 @@ async def test_submit_score_set_mappings_to_ldh_hgvs_not_found( # Verify no annotation statuses were created annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "ldh_submission") + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "ldh_submission") ).all() assert len(annotation_statuses) == 0 @@ -2365,11 +2365,11 @@ async def dummy_submission_failure(*args, **kwargs): # Verify annotation statuses were created with failures annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "ldh_submission") + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "ldh_submission") ).all() assert len(annotation_statuses) == 4 for ann in annotation_statuses: - assert ann.status == "failed" + assert ann.disposition == "failed" # Verify the job status is updated in the database # TODO:XXX: Change status to 'failed' once decorator supports it @@ -2435,15 +2435,15 @@ async def dummy_partial_submission(*args, **kwargs): # Verify annotation statuses were created annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "ldh_submission") + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "ldh_submission") ).all() assert len(annotation_statuses) == 4 success_count = 0 failure_count = 0 for ann in annotation_statuses: - if ann.status == "success": + if ann.disposition == "present": success_count += 1 - elif ann.status == "failed": + elif ann.disposition == "failed": failure_count += 1 assert success_count == 1 @@ -2513,11 +2513,11 @@ async def dummy_ldh_submission(*args, **kwargs): # Verify annotation statuses were created annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "ldh_submission") + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "ldh_submission") ).all() assert len(annotation_statuses) == 4 for ann in annotation_statuses: - assert ann.status == "success" + assert ann.disposition == "present" # Verify the job status is updated in the database session.refresh(submit_score_set_mappings_to_ldh_sample_job_run) @@ -2590,11 +2590,11 @@ async def dummy_ldh_submission(*args, **kwargs): # Verify annotation statuses were created annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "ldh_submission") + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "ldh_submission") ).all() assert len(annotation_statuses) == 4 for ann in annotation_statuses: - assert ann.status == "success" + assert ann.disposition == "present" # Verify the job status is updated in the database session.refresh(submit_score_set_mappings_to_ldh_sample_job_run) @@ -2662,11 +2662,11 @@ async def dummy_ldh_submission(*args, **kwargs): # Verify annotation statuses were created annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "ldh_submission") + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "ldh_submission") ).all() assert len(annotation_statuses) == 4 for ann in annotation_statuses: - assert ann.status == "success" + assert ann.disposition == "present" # Verify the job status is updated in the database session.refresh(submit_score_set_mappings_to_ldh_sample_job_run_in_pipeline) @@ -2721,7 +2721,7 @@ async def test_submit_score_set_mappings_to_ldh_with_arq_context_exception_handl mock_send_slack_job_error.assert_called_once() # Verify no annotation statuses were created annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "ldh_submission") + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "ldh_submission") ).all() assert len(annotation_statuses) == 0 @@ -2776,7 +2776,7 @@ async def test_submit_score_set_mappings_to_ldh_with_arq_context_exception_handl mock_send_slack_job_error.assert_called_once() # Verify no annotation statuses were created annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "ldh_submission") + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "ldh_submission") ).all() assert len(annotation_statuses) == 0 From 73007fd54a2527638200c212d106f4b52e93f4b9 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 30 Jun 2026 08:31:05 -0700 Subject: [PATCH 040/252] feat(clinvar): emit annotation events from ClinVar job The ClinVar job records AnnotationEvents via record_event, distinguishing superseded/no_record/no_caid/multi_variant_caid cases through the shared EventReason vocabulary. Clarifies the ClinicalControl comments to mark the row-level vs variation-level link identifiers. --- src/mavedb/models/clinical_control.py | 4 +- .../worker/jobs/external_services/clinvar.py | 197 +++++++++--------- .../external_services/network/test_clinvar.py | 37 ++-- .../jobs/external_services/test_clinvar.py | 122 ++++++----- 4 files changed, 202 insertions(+), 158 deletions(-) diff --git a/src/mavedb/models/clinical_control.py b/src/mavedb/models/clinical_control.py index cf28b5b8b..694aad58e 100644 --- a/src/mavedb/models/clinical_control.py +++ b/src/mavedb/models/clinical_control.py @@ -28,11 +28,11 @@ class ClinvarControl(Base): clinical_review_status = Column(String, nullable=False) db_name = Column(String, nullable=False, index=True) - # Currently the ClinVar Allele ID. + # ClinVar Allele ID (row level link). db_identifier = Column(String, nullable=False, index=True) db_version = Column(String, nullable=False, index=True) - # ClinVar's canonical public identifier (ClinVar Variation ID). + # ClinVar Variation ID (variation level link). clinvar_variation_id = Column(String, nullable=True) creation_date = Column(Date, nullable=False, default=date.today) diff --git a/src/mavedb/worker/jobs/external_services/clinvar.py b/src/mavedb/worker/jobs/external_services/clinvar.py index 8041d14f7..ad9eaf654 100644 --- a/src/mavedb/worker/jobs/external_services/clinvar.py +++ b/src/mavedb/worker/jobs/external_services/clinvar.py @@ -12,10 +12,11 @@ """ import logging +from collections import Counter from datetime import datetime import requests -from sqlalchemy import func, select +from sqlalchemy import select from sqlalchemy.dialects.postgresql import insert as pg_insert from mavedb.lib.annotation_status_manager import AnnotationStatusManager @@ -26,7 +27,9 @@ from mavedb.models.clinical_control import ClinvarControl from mavedb.models.clinvar_allele_link import ClinvarAlleleLink from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationFailureCategory, AnnotationStatus, FailureCategory +from mavedb.models.enums.disposition import Disposition +from mavedb.models.enums.event_reason import EventReason +from mavedb.models.enums.job_pipeline import FailureCategory from mavedb.models.score_set import ScoreSet from mavedb.worker.jobs.utils.setup import validate_job_params from mavedb.worker.lib.decorators.pipeline_management import with_pipeline_management @@ -47,46 +50,40 @@ def _generate_clinvar_versions() -> list[tuple[int, int]]: snapshot that should be processed. """ current_year = datetime.now().year - versions = [(CLINVAR_START_YEAR, CLINVAR_START_MONTH)] - for year in range(CLINVAR_START_YEAR + 1, current_year + 1): - versions.append((year, 1)) - return versions + first_version = (CLINVAR_START_YEAR, CLINVAR_START_MONTH) + archival_versions = [(year, 1) for year in range(CLINVAR_START_YEAR + 1, current_year + 1)] + return [first_version, *archival_versions] def _annotate_clinvar( annotation_manager: AnnotationStatusManager, - variant_ids: list[int], - version: str, - status: AnnotationStatus, + allele_id: int, + disposition: Disposition, + reason: EventReason, *, - failure_category: AnnotationFailureCategory | None = None, + source_version: str, error_message: str | None = None, metadata: dict | None = None, ) -> None: - """Fan a CLINVAR_CONTROL annotation out to every variant served by an allele. + """Record one CLINVAR_CONTROL event for an allele at a given ClinVar release. - AAS migration seam: the single choke point for ClinVar's per-variant VAS writes. At migration it - becomes an allele-keyed event writer; the per-variant fan-out goes away, and the variant - association narrows to provenance. See docs/design/allele-annotation-status.md. - - Unlike gnomAD/VEP, ClinVar writes a status row per ClinVar release, so retirement is - version-scoped (``replace_all_versions=False``) — each release keeps its own current row. + The single choke point for ClinVar's status writes. One event per (allele, release); provenance + (which variants drove the linkage) is derived from the live links as-of the event, not fanned out + here. Unlike gnomAD/VEP, ClinVar is multi-version, so ``source_version`` (the ClinVar release) + distinguishes an allele's events across releases. """ - annotation_data: dict = {"annotation_metadata": metadata or {}} + meta = dict(metadata or {}) if error_message is not None: - annotation_data["error_message"] = error_message - - for variant_id in variant_ids: - annotation_manager.add_annotation( - variant_id=variant_id, - annotation_type=AnnotationType.CLINVAR_CONTROL, - version=version, - status=status, - failure_category=failure_category, - annotation_data=annotation_data, - current=True, - replace_all_versions=False, - ) + meta["error_message"] = error_message + + annotation_manager.record_event( + AnnotationType.CLINVAR_CONTROL, + allele_id=allele_id, + disposition=disposition, + reason=reason, + source_version=source_version, + metadata=meta or None, + ) @with_pipeline_management @@ -139,13 +136,23 @@ async def refresh_clinvar_controls(ctx: dict, job_id: int, job_manager: JobManag # One work-unit per allele (payload = CAID; alleles without one are dropped). Covers ALL the score # set's alleles — authoritative and RT-derived — since the genomic allele ClinVar keys on is often - # the RT-derived one; VAS still fans only to authoritative_variant_ids (the bandaid seam). + # the RT-derived one. Events are allele-keyed, so every allele is recorded per release (no fan-out). allele_data = group_alleles_for_annotation( get_alleles_for_score_set(job_manager.db, score_set.id), payload=lambda row: row.clingen_allele_id, ) job_manager.save_to_context({"num_alleles_with_caids": len(allele_data)}) + # Link counts accumulate across all versions (an allele may link in every release it appears in). + annotation_counts: Counter[str] = Counter( + { + "created_link_count": 0, + "preexisting_link_count": 0, + "skipped_link_count": 0, + "failed_link_count": 0, + } + ) + if not allele_data: logger.warning( msg="No current alleles with CAIDs were found for this score set. Skipping ClinVar refresh.", @@ -153,12 +160,7 @@ async def refresh_clinvar_controls(ctx: dict, job_id: int, job_manager: JobManag ) job_manager.db.flush() return JobExecutionOutcome.succeeded( - data={ - "versions_completed": 0, - "versions_total": len(versions), - "created_link_count": 0, - "preexisting_link_count": 0, - } + data={"versions_completed": 0, "versions_total": len(versions), **dict(annotation_counts)} ) all_allele_ids = set(allele_data.keys()) @@ -175,11 +177,7 @@ def alleles_linked_at_version(clinvar_version: str) -> set[int]: ).all() ) - total_failed = 0 - created_link_count = 0 - preexisting_link_count = 0 versions_completed = 0 - for version_index, (year, month) in enumerate(versions): clinvar_version = f"{month:02d}_{year}" job_manager.save_to_context({"current_version": clinvar_version, "version_index": version_index}) @@ -206,28 +204,32 @@ def alleles_linked_at_version(clinvar_version: str) -> set[int]: # bypasses the skip but the link write still get-or-creates, so a forced no-op writes nothing. already_linked = set() if force else alleles_linked_at_version(clinvar_version) - annotation_manager = AnnotationStatusManager(job_manager.db, job_run_id=job_manager.job_id) - for allele_id, entry in allele_data.items(): - caid = entry.payload - + annotation_manager = AnnotationStatusManager( + job_manager.db, job_run_id=job_manager.job_id, score_set_id=score_set.id + ) + for allele_id, caid in allele_data.items(): if allele_id in already_linked: - preexisting_link_count += 1 + annotation_counts["preexisting_link_count"] += 1 _annotate_clinvar( annotation_manager, - entry.authoritative_variant_ids, - clinvar_version, - AnnotationStatus.SUCCESS, - metadata={"clingen_allele_id": caid, "action": "preexisting"}, + allele_id, + Disposition.PRESENT, + EventReason.PREEXISTING, + source_version=clinvar_version, + metadata={"clingen_allele_id": caid}, ) continue + # A cis-block (multi-variant) CAID structurally cannot key ClinVar — a terminal gap, not + # a statement about ClinVar's contents. if "," in caid: + annotation_counts["skipped_link_count"] += 1 _annotate_clinvar( annotation_manager, - entry.authoritative_variant_ids, - clinvar_version, - AnnotationStatus.SKIPPED, - failure_category=AnnotationFailureCategory.UNSUPPORTED_IDENTIFIER, + allele_id, + Disposition.NOT_APPLICABLE, + EventReason.MULTI_VARIANT_CAID, + source_version=clinvar_version, error_message="Multi-variant ClinGen allele IDs cannot be associated with ClinVar data.", metadata={"clingen_allele_id": caid}, ) @@ -236,13 +238,13 @@ def alleles_linked_at_version(clinvar_version: str) -> set[int]: try: clinvar_allele_id = await get_associated_clinvar_allele_id(caid) except requests.exceptions.RequestException as exc: - total_failed += 1 + annotation_counts["failed_link_count"] += 1 _annotate_clinvar( annotation_manager, - entry.authoritative_variant_ids, - clinvar_version, - AnnotationStatus.FAILED, - failure_category=AnnotationFailureCategory.EXTERNAL_API_ERROR, + allele_id, + Disposition.FAILED, + EventReason.API_ERROR, + source_version=clinvar_version, error_message=f"Failed to retrieve ClinVar allele ID from ClinGen API: {str(exc)}", metadata={"clingen_allele_id": caid}, ) @@ -253,27 +255,33 @@ def alleles_linked_at_version(clinvar_version: str) -> set[int]: ) continue + # ClinGen has no ClinVar AlleleID for this CAID — the allele is not a ClinVar control: an + # informative negative about the source, not a pipeline gap. if not clinvar_allele_id: + annotation_counts["skipped_link_count"] += 1 _annotate_clinvar( annotation_manager, - entry.authoritative_variant_ids, - clinvar_version, - AnnotationStatus.SKIPPED, - failure_category=AnnotationFailureCategory.NO_LINKED_ALLELE, + allele_id, + Disposition.ABSENT, + EventReason.NO_RECORD, + source_version=clinvar_version, error_message="No ClinVar allele ID found for ClinGen allele ID.", metadata={"clingen_allele_id": caid}, ) continue + # The allele has a ClinVar AlleleID but it is absent from this release's snapshot — a + # genuine, version-scoped negative (ClinVar queried, nothing for this release). if clinvar_allele_id not in tsv_data: + annotation_counts["skipped_link_count"] += 1 _annotate_clinvar( annotation_manager, - entry.authoritative_variant_ids, - clinvar_version, - AnnotationStatus.SKIPPED, - failure_category=AnnotationFailureCategory.EXTERNAL_REFERENCE_NOT_FOUND, + allele_id, + Disposition.ABSENT, + EventReason.NO_RECORD, + source_version=clinvar_version, error_message="No ClinVar data found for ClinVar allele ID.", - metadata={"clingen_allele_id": caid}, + metadata={"clingen_allele_id": caid, "clinvar_allele_id": clinvar_allele_id}, ) continue @@ -323,20 +331,19 @@ def alleles_linked_at_version(clinvar_version: str) -> set[int]: ) if live_link is None: job_manager.db.add(ClinvarAlleleLink(allele_id=allele_id, clinvar_control_id=clinvar_control.id)) - created_link_count += 1 - action = "created" + annotation_counts["created_link_count"] += 1 + reason = EventReason.CREATED elif live_link.clinvar_control_id == clinvar_control.id: - preexisting_link_count += 1 - action = "preexisting" + annotation_counts["preexisting_link_count"] += 1 + reason = EventReason.PREEXISTING else: - at = job_manager.db.scalar(select(func.now())) - assert at is not None # SELECT now() always returns a timestamp - live_link.retire(at=at) + retired_at = datetime.now() + live_link.retire(at=retired_at) job_manager.db.add( - ClinvarAlleleLink(allele_id=allele_id, clinvar_control_id=clinvar_control.id, valid_from=at) + ClinvarAlleleLink(allele_id=allele_id, clinvar_control_id=clinvar_control.id, valid_from=retired_at) ) - created_link_count += 1 - action = "superseded" + annotation_counts["created_link_count"] += 1 + reason = EventReason.SUPERSEDED logger.warning( msg=( f"Allele {allele_id} held a live ClinVar link to control " @@ -350,10 +357,11 @@ def alleles_linked_at_version(clinvar_version: str) -> set[int]: _annotate_clinvar( annotation_manager, - entry.authoritative_variant_ids, - clinvar_version, - AnnotationStatus.SUCCESS, - metadata={"clingen_allele_id": caid, "clinvar_allele_id": clinvar_allele_id, "action": action}, + allele_id, + Disposition.PRESENT, + reason, + source_version=clinvar_version, + metadata={"clingen_allele_id": caid, "clinvar_allele_id": clinvar_allele_id}, ) annotation_manager.flush() @@ -366,14 +374,17 @@ def alleles_linked_at_version(clinvar_version: str) -> set[int]: logger.info( f"ClinVar refresh complete: {versions_completed}/{len(versions)} versions, " - f"{created_link_count} new links, {preexisting_link_count} preexisting.", + f"{annotation_counts['created_link_count']} new links, " + f"{annotation_counts['preexisting_link_count']} preexisting.", extra=job_manager.logging_context(), ) - if total_failed > 0 and created_link_count == 0 and preexisting_link_count == 0: - error_message = ( - f"All {total_failed} ClinVar lookups failed for score set {score_set.urn}. Possible ClinGen API outage." - ) + if ( + annotation_counts["failed_link_count"] > 0 + and annotation_counts["created_link_count"] == 0 + and annotation_counts["preexisting_link_count"] == 0 + ): + error_message = f"All {annotation_counts['failed_link_count']} ClinVar lookups failed for score set {score_set.urn}. Possible ClinGen API outage." logger.error(error_message, extra=job_manager.logging_context()) job_manager.db.flush() return JobExecutionOutcome.failed( @@ -381,18 +392,12 @@ def alleles_linked_at_version(clinvar_version: str) -> set[int]: data={ "versions_completed": versions_completed, "versions_total": len(versions), - "created_link_count": 0, - "preexisting_link_count": 0, + **dict(annotation_counts), }, failure_category=FailureCategory.DEPENDENCY_FAILURE, ) job_manager.db.flush() return JobExecutionOutcome.succeeded( - data={ - "versions_completed": versions_completed, - "versions_total": len(versions), - "created_link_count": created_link_count, - "preexisting_link_count": preexisting_link_count, - } + data={"versions_completed": versions_completed, "versions_total": len(versions), **dict(annotation_counts)} ) diff --git a/tests/worker/jobs/external_services/network/test_clinvar.py b/tests/worker/jobs/external_services/network/test_clinvar.py index 4569d08f7..6d3c0b819 100644 --- a/tests/worker/jobs/external_services/network/test_clinvar.py +++ b/tests/worker/jobs/external_services/network/test_clinvar.py @@ -10,10 +10,11 @@ from mavedb.lib.clinvar.constants import CLINVAR_FIELDS_TO_KEEP from mavedb.lib.clinvar.utils import fetch_clinvar_variant_data +from mavedb.models.annotation_event import AnnotationEvent from mavedb.models.clinical_control import ClinvarControl from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationStatus, JobStatus -from mavedb.models.variant_annotation_status import VariantAnnotationStatus +from mavedb.models.enums.disposition import Disposition +from mavedb.models.enums.job_pipeline import JobStatus from mavedb.worker.jobs.external_services.clinvar import _generate_clinvar_versions pytestmark = pytest.mark.usefixtures("patch_db_session_ctxmgr") @@ -71,22 +72,32 @@ async def test_refresh_clinvar_controls_e2e( assert len(clinical_controls) >= 1 assert all(cc.db_identifier == "3045425" for cc in clinical_controls) - success_annotations = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.annotation_type == AnnotationType.CLINVAR_CONTROL, - VariantAnnotationStatus.status == AnnotationStatus.SUCCESS, + # Verify that at least one present event was recorded for the allele. The job processes one + # event per ClinVar version; versions without the allele produce absent events, so filtering + # for present gives a stable assertion. Events are allele-keyed (no variant_id). + present_events = session.scalars( + select(AnnotationEvent).where( + AnnotationEvent.annotation_type == AnnotationType.CLINVAR_CONTROL, + AnnotationEvent.disposition == Disposition.PRESENT, ) ).all() - assert len(success_annotations) == 1 - - # Total annotations == versions processed. - all_annotations = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.annotation_type == AnnotationType.CLINVAR_CONTROL, + assert len(present_events) >= 1 + assert all(e.variant_id is None and e.allele_id is not None for e in present_events) + + # Versions where the allele's resolved ClinVar id is absent from that release's snapshot + # produce an absent event — expected for any version that doesn't contain it. + absent_events = session.scalars( + select(AnnotationEvent).where( + AnnotationEvent.annotation_type == AnnotationType.CLINVAR_CONTROL, + AnnotationEvent.disposition == Disposition.ABSENT, ) ).all() - assert len(all_annotations) == len(_E2E_VERSIONS) + assert len(absent_events) >= 1 + + # Total events should equal the number of ClinVar versions processed (one allele, one per version). + assert len(present_events) + len(absent_events) == len(_generate_clinvar_versions()) + # Verify that the job run was completed successfully session.refresh(sample_refresh_clinvar_controls_job_run) assert sample_refresh_clinvar_controls_job_run.status == JobStatus.SUCCEEDED diff --git a/tests/worker/jobs/external_services/test_clinvar.py b/tests/worker/jobs/external_services/test_clinvar.py index 6b5011236..b757730f9 100644 --- a/tests/worker/jobs/external_services/test_clinvar.py +++ b/tests/worker/jobs/external_services/test_clinvar.py @@ -13,8 +13,10 @@ from mavedb.models.clinical_control import ClinvarControl from mavedb.models.clinvar_allele_link import ClinvarAlleleLink from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationFailureCategory, AnnotationStatus, FailureCategory, JobStatus -from mavedb.models.variant_annotation_status import VariantAnnotationStatus +from mavedb.models.enums.disposition import Disposition +from mavedb.models.enums.event_reason import EventReason +from mavedb.models.enums.job_pipeline import FailureCategory, JobStatus +from mavedb.models.annotation_event import AnnotationEvent from mavedb.worker.jobs.external_services.clinvar import refresh_clinvar_controls from mavedb.worker.lib.managers.job_manager import JobManager @@ -67,7 +69,7 @@ async def test_no_alleles_with_caids( assert result.status == JobStatus.SUCCEEDED assert session.scalars(select(ClinvarControl)).all() == [] assert session.scalars(select(ClinvarAlleleLink)).all() == [] - assert session.query(VariantAnnotationStatus).all() == [] + assert session.scalars(select(AnnotationEvent)).all() == [] async def test_fetch_failure_skips_version( self, @@ -105,7 +107,7 @@ async def test_multi_variant_caid_skipped( setup_sample_alleles_with_caid, ): """An allele whose CAID is a multi-variant identifier is skipped (ClinVar can't key on it).""" - variant, allele = setup_sample_alleles_with_caid + _, allele = setup_sample_alleles_with_caid allele.clingen_allele_id = "CA-MULTI-001,CA-MULTI-002" session.commit() @@ -122,10 +124,13 @@ async def test_multi_variant_caid_skipped( assert result.status == JobStatus.SUCCEEDED assert session.scalars(select(ClinvarAlleleLink)).all() == [] - vas = session.query(VariantAnnotationStatus).filter_by(variant_id=variant.id).one() - assert vas.status == AnnotationStatus.SKIPPED - assert vas.annotation_type == AnnotationType.CLINVAR_CONTROL - assert vas.failure_category == AnnotationFailureCategory.UNSUPPORTED_IDENTIFIER + # Allele-keyed event: a cis-block CAID structurally can't key ClinVar — a pipeline gap, not a + # statement about the source. + event = session.scalars(select(AnnotationEvent)).one() + assert event.annotation_type == AnnotationType.CLINVAR_CONTROL + assert event.disposition == Disposition.NOT_APPLICABLE + assert event.reason == EventReason.MULTI_VARIANT_CAID + assert event.allele_id == allele.id and event.variant_id is None async def test_no_associated_clinvar_allele_id_skipped( self, @@ -135,8 +140,8 @@ async def test_no_associated_clinvar_allele_id_skipped( sample_refresh_clinvar_controls_job_run, setup_sample_alleles_with_caid, ): - """ClinGen returns no ClinVar allele id for the CAID -> SKIPPED/NO_LINKED_ALLELE, no link.""" - variant, _ = setup_sample_alleles_with_caid + """ClinGen returns no ClinVar allele id for the CAID -> absent/no_record event, no link.""" + _, allele = setup_sample_alleles_with_caid with ( patch( @@ -156,9 +161,11 @@ async def test_no_associated_clinvar_allele_id_skipped( assert result.status == JobStatus.SUCCEEDED assert session.scalars(select(ClinvarAlleleLink)).all() == [] - vas = session.query(VariantAnnotationStatus).filter_by(variant_id=variant.id).one() - assert vas.status == AnnotationStatus.SKIPPED - assert vas.failure_category == AnnotationFailureCategory.NO_LINKED_ALLELE + # ClinGen had no ClinVar AlleleID for this CAID — an informative negative about the source. + event = session.scalars(select(AnnotationEvent)).one() + assert event.disposition == Disposition.ABSENT + assert event.reason == EventReason.NO_RECORD + assert event.allele_id == allele.id and event.variant_id is None async def test_clinvar_data_not_found_skipped( self, @@ -168,8 +175,8 @@ async def test_clinvar_data_not_found_skipped( sample_refresh_clinvar_controls_job_run, setup_sample_alleles_with_caid, ): - """The resolved ClinVar allele id is absent from the version's TSV -> SKIPPED, no link.""" - variant, _ = setup_sample_alleles_with_caid + """The resolved ClinVar allele id is absent from the version's TSV -> absent/no_record, no link.""" + _, allele = setup_sample_alleles_with_caid with ( patch( @@ -189,9 +196,12 @@ async def test_clinvar_data_not_found_skipped( assert result.status == JobStatus.SUCCEEDED assert session.scalars(select(ClinvarAlleleLink)).all() == [] - vas = session.query(VariantAnnotationStatus).filter_by(variant_id=variant.id).one() - assert vas.status == AnnotationStatus.SKIPPED - assert vas.failure_category == AnnotationFailureCategory.EXTERNAL_REFERENCE_NOT_FOUND + # The CAID resolved to a ClinVar AlleleID, but it's absent from this release's snapshot — a + # genuine, version-scoped negative. + event = session.scalars(select(AnnotationEvent)).one() + assert event.disposition == Disposition.ABSENT + assert event.reason == EventReason.NO_RECORD + assert event.allele_id == allele.id and event.variant_id is None async def test_clingen_api_failure_fails_when_total( self, @@ -202,7 +212,7 @@ async def test_clingen_api_failure_fails_when_total( setup_sample_alleles_with_caid, ): """A ClinGen API error with no successful links returns FAILED/DEPENDENCY_FAILURE.""" - variant, _ = setup_sample_alleles_with_caid + _, allele = setup_sample_alleles_with_caid with ( patch( @@ -222,9 +232,10 @@ async def test_clingen_api_failure_fails_when_total( assert result.status == JobStatus.FAILED assert result.failure_category == FailureCategory.DEPENDENCY_FAILURE - vas = session.query(VariantAnnotationStatus).filter_by(variant_id=variant.id).one() - assert vas.status == AnnotationStatus.FAILED - assert vas.failure_category == AnnotationFailureCategory.EXTERNAL_API_ERROR + event = session.scalars(select(AnnotationEvent)).one() + assert event.disposition == Disposition.FAILED + assert event.reason == EventReason.API_ERROR + assert event.allele_id == allele.id and event.variant_id is None async def test_successful_link_new_control( self, @@ -234,8 +245,8 @@ async def test_successful_link_new_control( sample_refresh_clinvar_controls_job_run, setup_sample_alleles_with_caid, ): - """A resolved + present CAID creates a ClinvarControl, a live link, and a SUCCESS VAS.""" - variant, allele = setup_sample_alleles_with_caid + """A resolved + present CAID creates a ClinvarControl, a live link, and a present/created event.""" + _, allele = setup_sample_alleles_with_caid with ( patch( @@ -269,11 +280,13 @@ async def test_successful_link_new_control( assert len(links) == 1 assert links[0].clinvar_control_id == control.id - vas = session.query(VariantAnnotationStatus).filter_by(variant_id=variant.id).one() - assert vas.status == AnnotationStatus.SUCCESS - assert vas.annotation_metadata["action"] == "created" + event = session.scalars(select(AnnotationEvent)).one() + assert event.disposition == Disposition.PRESENT + assert event.reason == EventReason.CREATED + assert event.annotation_type == AnnotationType.CLINVAR_CONTROL + assert event.allele_id == allele.id and event.variant_id is None - async def test_links_rt_derived_allele_but_annotates_only_authoritative( + async def test_links_and_annotates_full_allele_set( self, mock_worker_ctx, session, @@ -281,10 +294,13 @@ async def test_links_rt_derived_allele_but_annotates_only_authoritative( sample_refresh_clinvar_controls_job_run, setup_rt_derived_allele_with_caid, ): - """ClinVar linkage covers the full allele set: the RT-derived allele carries the matching CAID - and is linked, while per-variant VAS is written only for the authoritative link (the bandaid). + """ClinVar linkage covers the full allele set, not just authoritative links: the RT-derived + allele carries the matching CAID and is linked. Events are now allele-keyed, so the RT-derived + allele's present status is recorded directly — the limitation the per-variant bandaid had + (dropping the RT-derived allele's status) is lifted. Each allele gets its own event: present for + the linked RT allele, absent for the unmatched authoritative one. """ - variant, authoritative_allele, rt_allele = setup_rt_derived_allele_with_caid + _, authoritative_allele, rt_allele = setup_rt_derived_allele_with_caid rt_caid = rt_allele.clingen_allele_id async def resolve(caid): @@ -322,11 +338,17 @@ async def resolve(caid): == [] ) - # VAS is written only for the authoritative link: exactly one row, keyed to the variant. - statuses = session.query(VariantAnnotationStatus).all() - assert len(statuses) == 1 - assert statuses[0].variant_id == variant.id - assert statuses[0].annotation_type == AnnotationType.CLINVAR_CONTROL + # One allele-keyed event per allele (never per-variant): the linked RT allele is present, the + # unmatched authoritative allele is absent. The variant resolves its derived allele's status + # through the live links — no variant_id on the events. + events = session.scalars(select(AnnotationEvent)).all() + assert all(e.annotation_type == AnnotationType.CLINVAR_CONTROL for e in events) + assert all(e.variant_id is None for e in events) + by_allele = {e.allele_id: e for e in events} + assert by_allele[rt_allele.id].disposition == Disposition.PRESENT + assert by_allele[rt_allele.id].reason == EventReason.CREATED + assert by_allele[authoritative_allele.id].disposition == Disposition.ABSENT + assert by_allele[authoritative_allele.id].reason == EventReason.NO_RECORD async def test_idempotent_rerun_skips_and_does_not_duplicate( self, @@ -338,7 +360,7 @@ async def test_idempotent_rerun_skips_and_does_not_duplicate( ): """A second run finds the allele already linked at the version, skips the resolution, reports preexisting, and creates neither a duplicate control nor a duplicate link.""" - variant, allele = setup_sample_alleles_with_caid + _, allele = setup_sample_alleles_with_caid with ( patch( @@ -376,10 +398,15 @@ async def test_idempotent_rerun_skips_and_does_not_duplicate( assert len(links) == 1 assert links[0].valid_to is None - # Per-version VAS: two rows for the variant, only one current. - statuses = session.query(VariantAnnotationStatus).filter_by(variant_id=variant.id).all() - assert len(statuses) == 2 - assert len([s for s in statuses if s.current]) == 1 + # Per-version events: two allele-keyed events (one per run), no current flag. The first run + # created the link; the second found it preexisting. Latest event (by id) wins. + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.allele_id == allele.id).order_by(AnnotationEvent.id) + ).all() + assert len(events) == 2 + assert events[0].reason == EventReason.CREATED + assert events[1].reason == EventReason.PREEXISTING + assert all(e.disposition == Disposition.PRESENT for e in events) async def test_release_reresolution_supersedes_newest_wins( self, @@ -522,7 +549,7 @@ async def test_arq_context_successful_link( sample_refresh_clinvar_controls_job_run, setup_sample_alleles_with_caid, ): - """The job links an allele and records a SUCCESS annotation under an ARQ worker.""" + """The job links an allele and records a present event under an ARQ worker.""" with ( patch( "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", @@ -540,10 +567,11 @@ async def test_arq_context_successful_link( assert len(session.scalars(select(ClinvarControl)).all()) >= 1 assert len(session.scalars(select(ClinvarAlleleLink).where(ClinvarAlleleLink.current)).all()) == 1 - statuses = session.query(VariantAnnotationStatus).all() - assert len(statuses) == 1 - assert statuses[0].status == AnnotationStatus.SUCCESS - assert statuses[0].annotation_type == AnnotationType.CLINVAR_CONTROL + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 1 + assert events[0].disposition == Disposition.PRESENT + assert events[0].annotation_type == AnnotationType.CLINVAR_CONTROL + assert events[0].allele_id is not None and events[0].variant_id is None session.refresh(sample_refresh_clinvar_controls_job_run) assert sample_refresh_clinvar_controls_job_run.status == JobStatus.SUCCEEDED @@ -576,7 +604,7 @@ async def test_arq_context_exception_handling( mock_slack.assert_called_once() assert session.scalars(select(ClinvarAlleleLink)).all() == [] - assert session.query(VariantAnnotationStatus).all() == [] + assert session.scalars(select(AnnotationEvent)).all() == [] session.refresh(sample_refresh_clinvar_controls_job_run) assert sample_refresh_clinvar_controls_job_run.status == JobStatus.ERRORED From e31363b1df6633525ebe4bfa52d211a25c678189 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 30 Jun 2026 08:31:12 -0700 Subject: [PATCH 041/252] feat(mapping): emit annotation events The mapping job records AnnotationEvents via record_event, deriving disposition and reason from the typed MappingOutcome (mapped / intronic / no_protein_consequence / failed) rather than writing per-variant status rows. --- .../worker/jobs/variant_processing/mapping.py | 46 +++-- .../jobs/variant_processing/test_mapping.py | 166 ++++++++---------- 2 files changed, 97 insertions(+), 115 deletions(-) diff --git a/src/mavedb/worker/jobs/variant_processing/mapping.py b/src/mavedb/worker/jobs/variant_processing/mapping.py index 9e6a8da74..b2f588b72 100644 --- a/src/mavedb/worker/jobs/variant_processing/mapping.py +++ b/src/mavedb/worker/jobs/variant_processing/mapping.py @@ -31,7 +31,8 @@ from mavedb.models.allele import Allele as AlleleDbModel from mavedb.models.enums.annotation_layer import AnnotationLayer from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationFailureCategory, AnnotationStatus, FailureCategory +from mavedb.models.enums.disposition import Disposition +from mavedb.models.enums.job_pipeline import FailureCategory from mavedb.models.enums.mapping_state import MappingState from mavedb.models.mapping_record import MappingRecord from mavedb.models.mapping_record_allele import MappingRecordAllele @@ -268,7 +269,8 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan f"Processing {total_variants} mapped variants for score set {score_set.urn}.", extra=job_manager.logging_context(), ) - annotation_manager = AnnotationStatusManager(job_manager.db, job_run_id=job.id) + + annotation_manager = AnnotationStatusManager(job_manager.db, job_run_id=job.id, score_set_id=score_set.id) for mapped_score in mapped_scores: variant_urn = mapped_score.get("mavedb_id") variant = job_manager.db.scalars(select(Variant).where(Variant.urn == variant_urn)).one() @@ -349,32 +351,28 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan else: job_manager.db.add(mapping_record) - # MAPPED -> success; benign absences -> skipped; FAILED -> failed. The raw outcome - # is preserved in annotation_metadata so the benign distinction survives. + # The mapper emits a MappingRecord for EVERY variant, so "a record exists" carries no + # signal — the signal is whether a real allele resulted. MAPPED yields an authoritative + # allele -> present. A benign absence (intronic, synonymous) produces a record but no + # allele: an informative biological negative -> absent. FAILED -> failed. `reason` reuses + # the MappingOutcome vocabulary so the intronic-vs-no-protein distinction survives. if outcome is MappingOutcome.MAPPED: - annotation_status = AnnotationStatus.SUCCESS - annotation_failure_category = None + disposition = Disposition.PRESENT elif outcome.is_benign_absence: - annotation_status = AnnotationStatus.SKIPPED - annotation_failure_category = None + disposition = Disposition.ABSENT else: - annotation_status = AnnotationStatus.FAILED - annotation_failure_category = AnnotationFailureCategory.EXTERNAL_SERVICE_REJECTED - - annotation_manager.add_annotation( - variant_id=variant.id, # type: ignore - annotation_type=AnnotationType.VRS_MAPPING, - version=tool_version, - status=annotation_status, - failure_category=annotation_failure_category, - annotation_data={ - "error_message": mapped_score.get("error_message", null()), - "annotation_metadata": { - "outcome": outcome.value, - "mapped_assay_level_hgvs": assay_level_hgvs, - }, + disposition = Disposition.FAILED + + annotation_manager.record_event( + AnnotationType.VRS_MAPPING, + variant_id=variant.id, + disposition=disposition, + reason=outcome.value, + source_version=tool_version, + metadata={ + "mapped_assay_level_hgvs": assay_level_hgvs, + "error_message": mapped_score.get("error_message"), }, - current=True, ) # Only variants with a post-mapped representation yield an authoritative Allele; diff --git a/tests/worker/jobs/variant_processing/test_mapping.py b/tests/worker/jobs/variant_processing/test_mapping.py index 3a0c6db2f..96321d816 100644 --- a/tests/worker/jobs/variant_processing/test_mapping.py +++ b/tests/worker/jobs/variant_processing/test_mapping.py @@ -20,7 +20,7 @@ from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.target_gene_mapping import TargetGeneMapping from mavedb.models.variant import Variant -from mavedb.models.variant_annotation_status import VariantAnnotationStatus +from mavedb.models.annotation_event import AnnotationEvent from mavedb.worker.jobs.variant_processing.mapping import map_variants_for_score_set from mavedb.worker.lib.managers.job_manager import JobManager from tests.helpers.constants import TEST_CODING_LAYER, TEST_GENOMIC_LAYER, TEST_PROTEIN_LAYER @@ -90,8 +90,8 @@ async def test_map_variants_for_score_set_no_mapping_results( # Verify no annotations were created annotation_statuses = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) .filter(Variant.score_set_id == sample_score_set.id) .all() ) @@ -137,8 +137,8 @@ async def test_map_variants_for_score_set_no_mapped_scores( # Verify no annotations were created annotation_statuses = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) .filter(Variant.score_set_id == sample_score_set.id) .all() ) @@ -184,8 +184,8 @@ async def test_map_variants_for_score_set_no_reference_data( # Verify no annotations were created annotation_statuses = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) .filter(Variant.score_set_id == sample_score_set.id) .all() ) @@ -239,8 +239,8 @@ async def test_map_variants_for_score_set_nonexistent_target_gene( # Verify no annotations were created annotation_statuses = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) .filter(Variant.score_set_id == sample_score_set.id) .all() ) @@ -286,8 +286,8 @@ async def test_map_variants_for_score_set_returns_variants_not_in_score_set( # Verify no annotations were created annotation_statuses = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) .filter(Variant.score_set_id == sample_score_set.id) .all() ) @@ -354,14 +354,14 @@ async def dummy_mapping_job(): # Verify that annotation statuses were created and correct annotation_statuses = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) .filter(Variant.score_set_id == sample_score_set.id) .all() ) assert len(annotation_statuses) == 1 assert annotation_statuses[0].annotation_type == "vrs_mapping" - assert annotation_statuses[0].status == "success" + assert annotation_statuses[0].disposition == "present" @pytest.mark.parametrize( "with_layers", @@ -471,14 +471,14 @@ async def dummy_mapping_job(): # Verify that annotation statuses were created and correct annotation_statuses = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) .filter(Variant.score_set_id == sample_score_set.id) .all() ) assert len(annotation_statuses) == 1 assert annotation_statuses[0].annotation_type == "vrs_mapping" - assert annotation_statuses[0].status == "success" + assert annotation_statuses[0].disposition == "present" async def test_persists_cdna_target_gene_mapping_with_reference_accession_and_null_qc( self, @@ -584,14 +584,14 @@ async def dummy_mapping_job(): # Verify that annotation statuses were created and correct annotation_statuses = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) .filter(Variant.score_set_id == sample_score_set.id) .all() ) assert len(annotation_statuses) == 1 assert annotation_statuses[0].annotation_type == "vrs_mapping" - assert annotation_statuses[0].status == "failed" + assert annotation_statuses[0].disposition == "failed" async def test_map_variants_for_score_set_incomplete_mapping( self, @@ -668,17 +668,17 @@ async def dummy_mapping_job(): # Verify that annotation statuses were created and correct annotation_status_success = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) - .filter(Variant.score_set_id == sample_score_set.id, VariantAnnotationStatus.status == "success") + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) + .filter(Variant.score_set_id == sample_score_set.id, AnnotationEvent.disposition == "present") .all() ) assert len(annotation_status_success) == 1 assert annotation_status_success[0].annotation_type == "vrs_mapping" annotation_status_failed = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) - .filter(Variant.score_set_id == sample_score_set.id, VariantAnnotationStatus.status == "failed") + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) + .filter(Variant.score_set_id == sample_score_set.id, AnnotationEvent.disposition == "failed") .all() ) assert len(annotation_status_failed) == 1 @@ -694,7 +694,8 @@ async def test_map_variants_for_score_set_benign_outcomes_are_not_failures( ): """A score set whose only unmapped variants are benign absences (intronic / no protein consequence) is ``complete``, not ``incomplete``/``failed``: benign - outcomes carry no allele but are skips, not failures.""" + outcomes carry no allele, so they are recorded as ``absent`` (an informative + biological negative), never ``failed``.""" async def dummy_mapping_job(): mapping_output = await construct_mock_mapping_output( @@ -761,17 +762,17 @@ async def dummy_mapping_job(): assert len(mapping_records) == 2 assert all(authoritative_allele_for(session, r) is None for r in mapping_records) - # Benign outcomes are SKIPPED (not FAILED), and the finer outcome is preserved in metadata. - annotation_statuses = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) + # Benign outcomes carry no allele → recorded as `absent` (not `failed`), with the finer + # outcome preserved as the event `reason`. + events = ( + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) .filter(Variant.score_set_id == sample_score_set.id) .all() ) - assert len(annotation_statuses) == 2 - assert all(s.status == "skipped" for s in annotation_statuses) - assert all(s.failure_category is None for s in annotation_statuses) - recorded_outcomes = {s.annotation_metadata["outcome"] for s in annotation_statuses} + assert len(events) == 2 + assert all(e.disposition == "absent" for e in events) + recorded_outcomes = {e.reason for e in events} assert recorded_outcomes == {"intronic", "no_protein_consequence"} async def test_map_variants_for_score_set_complete_mapping( @@ -850,15 +851,15 @@ async def dummy_mapping_job(): # Verify that annotation statuses were created and correct annotation_statuses = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) .filter(Variant.score_set_id == sample_score_set.id) .all() ) assert len(annotation_statuses) == 2 for status in annotation_statuses: assert status.annotation_type == "vrs_mapping" - assert status.status == "success" + assert status.disposition == "present" async def test_map_variants_for_score_set_updates_existing_mapped_variants( self, @@ -909,10 +910,10 @@ async def dummy_mapping_job(): ) session.add(prior_link) session.commit() - variant_annotation_status = VariantAnnotationStatus( - variant_id=variant.id, current=True, annotation_type="vrs_mapping", status="success" + prior_event = AnnotationEvent( + variant_id=variant.id, annotation_type="vrs_mapping", disposition="present", reason="mapped" ) - session.add(variant_annotation_status) + session.add(prior_event) session.commit() with ( @@ -965,24 +966,14 @@ async def dummy_mapping_job(): assert new_mapping_record.mapped_date != date(2023, 1, 1) assert new_mapping_record.mapping_api_version != "v1.0.0" - # Verify the non-current annotation status still exists - old_annotation_status = ( - session.query(VariantAnnotationStatus) - .filter( - VariantAnnotationStatus.variant_id == non_current_mapping_record.variant_id, - VariantAnnotationStatus.current.is_(False), - ) - .one_or_none() - ) - assert old_annotation_status is not None - - # Verify that a new annotation status was created - new_annotation_status = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == variant.id, VariantAnnotationStatus.current.is_(True)) - .one_or_none() + # Append-only: the prior event is retained and the re-map appends a new one, so the variant + # now has two vrs_mapping events (there is no current flag to flip). + events = ( + session.query(AnnotationEvent) + .filter(AnnotationEvent.variant_id == variant.id, AnnotationEvent.annotation_type == "vrs_mapping") + .all() ) - assert new_annotation_status is not None + assert len(events) == 2 @pytest.mark.integration @@ -1065,8 +1056,8 @@ async def dummy_mapping_job(): # Verify that each variant has an annotation status annotation_statuses = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) .filter(Variant.score_set_id == sample_score_set.id) .all() ) @@ -1156,8 +1147,8 @@ async def dummy_mapping_job(): # Verify that each variant has an annotation status annotation_statuses = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) .filter(Variant.score_set_id == sample_score_set.id) .all() ) @@ -1237,7 +1228,7 @@ async def dummy_mapping_job(): assert len(mapping_records) == 0 # Verify that no annotation statuses were created - annotation_statuses = session.query(VariantAnnotationStatus).all() + annotation_statuses = session.query(AnnotationEvent).all() assert len(annotation_statuses) == 0 # Verify that the job status was updated. @@ -1314,7 +1305,7 @@ async def dummy_mapping_job(): assert len(mapping_records) == 0 # Verify that no annotation statuses were created - annotation_statuses = session.query(VariantAnnotationStatus).all() + annotation_statuses = session.query(AnnotationEvent).all() assert len(annotation_statuses) == 0 # Verify that the job status was updated. @@ -1390,7 +1381,7 @@ async def dummy_mapping_job(): assert len(mapping_records) == 0 # Verify that no annotation statuses were created - annotation_statuses = session.query(VariantAnnotationStatus).all() + annotation_statuses = session.query(AnnotationEvent).all() assert len(annotation_statuses) == 0 # Verify that the job status was updated. @@ -1434,10 +1425,10 @@ async def test_map_variants_for_score_set_updates_current_mapped_variants( mapped_date=date(2023, 1, 1), mapping_api_version="v1.0.0", ) - annotation_status = VariantAnnotationStatus( - variant_id=variant.id, current=True, annotation_type="vrs_mapping", status="success" + prior_event = AnnotationEvent( + variant_id=variant.id, annotation_type="vrs_mapping", disposition="present", reason="mapped" ) - session.add(annotation_status) + session.add(prior_event) session.add(mapping_record) session.commit() @@ -1496,23 +1487,16 @@ async def dummy_mapping_job(): assert new_mapping_record.mapped_date != date(2023, 1, 1) assert new_mapping_record.mapping_api_version != "v1.0.0" - # Verify that annotation statuses where marked as non-current and new entries created - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == len(variants) * 2 # Each variant has two annotation statuses now + # Append-only: each variant keeps its prior event and gains a new one from this run. + annotation_events = session.query(AnnotationEvent).all() + assert len(annotation_events) == len(variants) * 2 for variant in variants: - old_annotation_status = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == variant.id, VariantAnnotationStatus.current.is_(False)) - .one_or_none() - ) - assert old_annotation_status is not None - - new_annotation_status = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == variant.id, VariantAnnotationStatus.current.is_(True)) - .one_or_none() + variant_events = ( + session.query(AnnotationEvent) + .filter(AnnotationEvent.variant_id == variant.id, AnnotationEvent.annotation_type == "vrs_mapping") + .all() ) - assert new_annotation_status is not None + assert len(variant_events) == 2 # Verify that the job status was updated. processing_run = ( @@ -1573,7 +1557,7 @@ async def dummy_mapping_job(): assert len(mapping_records) == 0 # Verify that no annotation statuses were created - annotation_statuses = session.query(VariantAnnotationStatus).all() + annotation_statuses = session.query(AnnotationEvent).all() assert len(annotation_statuses) == 0 # Verify that the job status was updated. @@ -1632,7 +1616,7 @@ async def dummy_mapping_job(): assert len(mapping_records) == 0 # Verify that no annotation statuses were created - annotation_statuses = session.query(VariantAnnotationStatus).all() + annotation_statuses = session.query(AnnotationEvent).all() assert len(annotation_statuses) == 0 # Verify that the job status was updated. @@ -1716,8 +1700,8 @@ async def dummy_mapping_job(): # Verify that each variant has an annotation status annotation_statuses = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) .filter(Variant.score_set_id == sample_score_set.id) .all() ) @@ -1804,8 +1788,8 @@ async def dummy_mapping_job(): # Verify that each variant has an annotation status annotation_statuses = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) .filter(Variant.score_set_id == sample_score_set.id) .all() ) @@ -1874,7 +1858,7 @@ async def dummy_mapping_job(): assert len(mapping_records) == 0 # Verify that no annotation statuses were created - annotation_statuses = session.query(VariantAnnotationStatus).all() + annotation_statuses = session.query(AnnotationEvent).all() assert len(annotation_statuses) == 0 # Verify that the job status was updated. @@ -1928,7 +1912,7 @@ async def dummy_mapping_job(): assert len(mapping_records) == 0 # Verify that no annotation statuses were created - annotation_statuses = session.query(VariantAnnotationStatus).all() + annotation_statuses = session.query(AnnotationEvent).all() assert len(annotation_statuses) == 0 # Verify that the job status was updated. From 6288f82b0bf4a36881f913be538d45e4e46ea91d Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 30 Jun 2026 08:31:19 -0700 Subject: [PATCH 042/252] feat(reverse-translation): emit annotation events The reverse-translation job records AnnotationEvents via record_event, using the RT-specific EventReason cases (translated / no_coding_transcript / no_assay_level_hgvs / translation_failed / translation_error / transcript_unresolved) to distinguish informative negatives from failures. --- .../variant_processing/reverse_translation.py | 179 +++++++++--------- .../test_reverse_translation.py | 91 +++++---- 2 files changed, 141 insertions(+), 129 deletions(-) diff --git a/src/mavedb/worker/jobs/variant_processing/reverse_translation.py b/src/mavedb/worker/jobs/variant_processing/reverse_translation.py index 07df529fd..d7f17e919 100644 --- a/src/mavedb/worker/jobs/variant_processing/reverse_translation.py +++ b/src/mavedb/worker/jobs/variant_processing/reverse_translation.py @@ -11,12 +11,13 @@ import dataclasses import functools import logging +from collections import Counter from datetime import date -from enum import Enum from typing import Any, NamedTuple, Sequence from ga4gh.vrs.extras.translator import AlleleTranslator from sqlalchemy import select +from variant_annotation import __version__ as variant_annotation_version from variant_annotation.lib.accessions import looks_like_refseq_protein_accession from variant_annotation.lib.translation import construct_equivalent_variants from variant_annotation.lib.translation.types import TranslationConfig, VariantInput, WtCodonMode @@ -29,7 +30,9 @@ from mavedb.models.allele import Allele as AlleleDbModel from mavedb.models.enums.annotation_layer import AnnotationLayer from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationFailureCategory, AnnotationStatus, FailureCategory +from mavedb.models.enums.disposition import Disposition +from mavedb.models.enums.event_reason import EventReason +from mavedb.models.enums.job_pipeline import FailureCategory from mavedb.models.enums.target_category import TargetCategory from mavedb.models.mapping_record import MappingRecord from mavedb.models.mapping_record_allele import MappingRecordAllele @@ -64,36 +67,21 @@ class _TranscriptResolution(NamedTuple): target_gene_id: int | None -class _TranscriptResolutionSkipReason(Enum): - NO_ASSAY_LEVEL_HGVS = "no_assay_level_hgvs" - TRANSCRIPT_UNRESOLVED = "transcript_unresolved" - NO_CODING_TRANSCRIPT = "no_coding_transcript" - - @classmethod - def classify( - cls, resolution: _TranscriptResolution, category: TargetCategory | None - ) -> tuple["_TranscriptResolutionSkipReason", str]: - """Classify why a record was skipped. - - Protein-coding target with no transcript → recoverable (transcript_unresolved). - Non-coding/regulatory target → correct skip (no_coding_transcript). - """ - if not resolution.rec.hgvs_assay_level: - return ( - cls.NO_ASSAY_LEVEL_HGVS, - "No assay-level HGVS available to reverse-translate.", - ) - if category == TargetCategory.protein_coding: - return ( - cls.TRANSCRIPT_UNRESOLVED, - "Protein-coding target but no coding transcript could be resolved " - "(no cdna TargetGeneMapping and no NP_->NM_ association). Recoverable: " - "re-map or check transcript selection.", - ) - return ( - cls.NO_CODING_TRANSCRIPT, - "Non-coding/regulatory target has no protein consequence to reverse-translate.", - ) +def _classify_skip( + resolution: _TranscriptResolution, category: TargetCategory | None +) -> tuple[EventReason, Disposition]: + """Classify why a record was skipped, returning its reason and event disposition. + + ``no_assay_level_hgvs`` — no input to translate, we could not ask → ``not_applicable``. + ``transcript_unresolved`` — protein-coding target with no transcript, a recoverable pipeline + gap → ``failed``. ``no_coding_transcript`` — non-coding target has no protein consequence, a + biological negative → ``absent``. + """ + if not resolution.rec.hgvs_assay_level: + return (EventReason.NO_ASSAY_LEVEL_HGVS, Disposition.NOT_APPLICABLE) + if category == TargetCategory.protein_coding: + return (EventReason.TRANSCRIPT_UNRESOLVED, Disposition.FAILED) + return (EventReason.NO_CODING_TRANSCRIPT, Disposition.ABSENT) def _coding_transcripts_for_proteins(protein_accessions: set[str]) -> dict[str, str]: @@ -147,6 +135,33 @@ def _build_translation_config(overrides: dict[str, Any] | None) -> TranslationCo raise ValueError(f"Invalid translation_config: {exc}") from exc +def _annotate_translation( + annotation_manager: AnnotationStatusManager, + variant_id: int, + disposition: Disposition, + reason: EventReason, + *, + error_message: str | None = None, + metadata: dict | None = None, +) -> None: + """Record one CROSS_LEVEL_TRANSLATION event for an allele (the translation is a variant-level fact). + + The single choke point for RT's status writes. + """ + meta = dict(metadata or {}) + if error_message is not None: + meta["error_message"] = error_message + + annotation_manager.record_event( + AnnotationType.CROSS_LEVEL_TRANSLATION, + variant_id=variant_id, + disposition=disposition, + reason=reason, + source_version=variant_annotation_version, + metadata=meta or None, + ) + + @with_pipeline_management async def reverse_translate_variants_for_score_set( ctx: dict, job_id: int, job_manager: JobManager @@ -227,13 +242,15 @@ async def reverse_translate_variants_for_score_set( .all() ) + annotation_counts: Counter[str] = Counter({"translated": 0, "failed": 0, "skipped": 0, "alleles_created": 0}) + if not rows: logger.warning( msg="No current and authoritative mapping records found for this score set.", extra=job_manager.logging_context(), ) job_manager.db.flush() - return JobExecutionOutcome.succeeded(data={"translated": 0, "failed": 0, "skipped": 0, "alleles_created": 0}) + return JobExecutionOutcome.succeeded(data=dict(annotation_counts)) # Genomic/cdna records resolve via the cdna TargetGeneMapping; protein records have no # cdna reference_accession, so collect their NP_ accessions for a batched NP_→NM_ UTA lookup. @@ -322,10 +339,9 @@ async def reverse_translate_variants_for_score_set( extra=job_manager.logging_context(), ) - translated = 0 - failed = 0 - alleles_created = 0 - annotation_manager = AnnotationStatusManager(job_manager.db, job_run_id=job_manager.job_id) + annotation_manager = AnnotationStatusManager( + job_manager.db, job_run_id=job_manager.job_id, score_set_id=score_set_id + ) allele_translator = AlleleTranslator(ctx["seqrepo"]) current_record_ids = ( @@ -411,11 +427,12 @@ async def reverse_translate_variants_for_score_set( ) candidate_count += 1 - alleles_created += candidate_count + annotation_counts["alleles_created"] += candidate_count annotation_metadata = { "hgvs_input": result.input.hgvs, "hgvs_c_candidates": result.hgvs_c_candidates, "hgvs_g_candidates": result.hgvs_g_candidates, + "hgvs_p": result.hgvs_p, "alleles_created": candidate_count, "failed_candidates": failed_candidates, } @@ -423,24 +440,23 @@ async def reverse_translate_variants_for_score_set( # No translatable candidates and failures mean the variant failed reverse translation. No # failures and no candidates is a success with no alleles created. if candidate_count == 0 and failed_candidates: - failed += 1 - annotation_manager.add_annotation( + annotation_counts["failed"] += 1 + _annotate_translation( + annotation_manager, variant_id=variant.id, - annotation_type=AnnotationType.CROSS_LEVEL_TRANSLATION, - status=AnnotationStatus.FAILED, - failure_category=AnnotationFailureCategory.UNKNOWN, - annotation_data={ - "error_message": "All candidate HGVS failed VRS translation.", - "annotation_metadata": annotation_metadata, - }, + disposition=Disposition.FAILED, + reason=EventReason.TRANSLATION_FAILED, + error_message="All candidate HGVS failed VRS translation.", + metadata=annotation_metadata, ) else: - translated += 1 - annotation_manager.add_annotation( + annotation_counts["translated"] += 1 + _annotate_translation( + annotation_manager, variant_id=variant.id, - annotation_type=AnnotationType.CROSS_LEVEL_TRANSLATION, - status=AnnotationStatus.SUCCESS, - annotation_data={"annotation_metadata": annotation_metadata}, + disposition=Disposition.PRESENT, + reason=EventReason.TRANSLATED, + metadata=annotation_metadata, ) # Supersede prior live derived links atomically. @@ -453,71 +469,56 @@ async def reverse_translate_variants_for_score_set( MappingRecordAllele.mapping_record_id.in_(current_record_ids), ) - # TODO#767: non-substitution consequences (del/ins/delins/fs/ext) have no synonymous + # TODO#767: some non-substitution consequences (del/ins/delins/fs/ext) have no synonymous # equivalence class, so they arrive here as TranslationErrors and are miscounted as # FAILED. Classify by the protein consequence's edit type up front and map these to # SKIPPED instead of pattern-matching engine error strings. for error in errors: _rec, variant = variant_input_map[id(error.input)] - failed += 1 - annotation_manager.add_annotation( + annotation_counts["failed"] += 1 + _annotate_translation( + annotation_manager, variant_id=variant.id, - annotation_type=AnnotationType.CROSS_LEVEL_TRANSLATION, - status=AnnotationStatus.FAILED, - failure_category=AnnotationFailureCategory.UNKNOWN, - annotation_data={ - "error_message": error.error, - "annotation_metadata": {"hgvs_input": error.input.hgvs}, - }, + disposition=Disposition.FAILED, + reason=EventReason.TRANSLATION_ERROR, + metadata={"hgvs_input": error.input.hgvs, "error_message": error.error}, ) - skipped = len(skipped_variants) + annotation_counts["skipped"] = len(skipped_variants) for p in skipped_variants: category = target_category_by_gene.get(p.target_gene_id) if p.target_gene_id is not None else None - skip_category, reason = _TranscriptResolutionSkipReason.classify(p, category) - annotation_manager.add_annotation( + reason, disposition = _classify_skip(p, category) + _annotate_translation( + annotation_manager, variant_id=p.variant.id, - annotation_type=AnnotationType.CROSS_LEVEL_TRANSLATION, - status=AnnotationStatus.SKIPPED, - annotation_data={ - "annotation_metadata": { - "hgvs_input": p.rec.hgvs_assay_level, - "skip_category": skip_category.value, - "reason": reason, - } - }, + disposition=disposition, + reason=reason, + metadata={"hgvs_input": p.rec.hgvs_assay_level}, ) annotation_manager.flush() - job_manager.save_to_context( - { - "translated": translated, - "failed": failed, - "skipped": skipped, - "alleles_created": alleles_created, - } - ) + outcome_data = dict(annotation_counts) + job_manager.save_to_context(outcome_data) logger.info( msg=( - f"Reverse translation complete: {translated} translated, {failed} failed, " - f"{skipped} skipped, {alleles_created} alleles created." + f"Reverse translation complete: {annotation_counts['translated']} translated, " + f"{annotation_counts['failed']} failed, {annotation_counts['skipped']} skipped, " + f"{annotation_counts['alleles_created']} alleles created." ), extra=job_manager.logging_context(), ) job_manager.db.flush() - if translated == 0 and failed > 0: + if annotation_counts["translated"] == 0 and annotation_counts["failed"] > 0: logger.error( msg="All variant reverse translations failed.", extra=job_manager.logging_context(), ) return JobExecutionOutcome.failed( reason="All variant reverse translations failed.", - data={"translated": 0, "failed": failed, "skipped": skipped, "alleles_created": 0}, + data=outcome_data, failure_category=FailureCategory.DATA_ERROR, ) - return JobExecutionOutcome.succeeded( - data={"translated": translated, "failed": failed, "skipped": skipped, "alleles_created": alleles_created} - ) + return JobExecutionOutcome.succeeded(data=outcome_data) diff --git a/tests/worker/jobs/variant_processing/test_reverse_translation.py b/tests/worker/jobs/variant_processing/test_reverse_translation.py index 970fa8544..1a77e8e86 100644 --- a/tests/worker/jobs/variant_processing/test_reverse_translation.py +++ b/tests/worker/jobs/variant_processing/test_reverse_translation.py @@ -21,7 +21,7 @@ from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.target_gene_mapping import TargetGeneMapping from mavedb.models.variant import Variant -from mavedb.models.variant_annotation_status import VariantAnnotationStatus +from mavedb.models.annotation_event import AnnotationEvent from mavedb.worker.jobs.variant_processing.mapping import map_variants_for_score_set from mavedb.worker.jobs.variant_processing.reverse_translation import ( _build_translation_config, @@ -143,17 +143,19 @@ async def _reverse_translate(session, mock_worker_ctx, rt_run): ) -def _cross_level_statuses(session, score_set_id, status=None): +def _cross_level_events(session, score_set_id, disposition=None, reason=None): query = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) .filter( Variant.score_set_id == score_set_id, - VariantAnnotationStatus.annotation_type == "cross_level_translation", + AnnotationEvent.annotation_type == "cross_level_translation", ) ) - if status is not None: - query = query.filter(VariantAnnotationStatus.status == status) + if disposition is not None: + query = query.filter(AnnotationEvent.disposition == disposition) + if reason is not None: + query = query.filter(AnnotationEvent.reason == reason) return query.all() @@ -226,7 +228,7 @@ async def test_no_mapping_records_is_a_noop( # No candidate alleles, links, or annotations were produced. assert session.query(Allele).count() == 0 assert _non_authoritative_links(session) == [] - assert _cross_level_statuses(session, sample_score_set.id) == [] + assert _cross_level_events(session, sample_score_set.id) == [] async def test_creates_genomic_and_coding_candidate_alleles( self, @@ -287,9 +289,9 @@ async def test_creates_genomic_and_coding_candidate_alleles( mapping_record = session.query(MappingRecord).filter(MappingRecord.variant_id == variant.id).one() assert {link.mapping_record_id for link in non_auth_links} == {mapping_record.id} - statuses = _cross_level_statuses(session, sample_score_set.id) - assert len(statuses) == 1 - assert statuses[0].status == "success" + events = _cross_level_events(session, sample_score_set.id) + assert len(events) == 1 + assert events[0].disposition == "present" async def test_transcript_is_queryable_via_derived_expression( self, @@ -604,10 +606,11 @@ async def test_independent_rerun_retires_prior_links_and_keeps_current_stable( # The two candidate alleles are reused across runs, not duplicated. assert session.query(Allele).filter(Allele.vrs_digest.in_(["ga4gh:VA.genomic", "ga4gh:VA.coding"])).count() == 2 - # The cross-level translation status is versioned the same way: prior retired, one current. - statuses = _cross_level_statuses(session, sample_score_set.id) - assert len(statuses) == 2 - assert len([s for s in statuses if s.current]) == 1 + # The cross-level translation log is append-only: the rerun adds a second event, and + # "current" is the latest by id (there is no current flag to desync). + events = _cross_level_events(session, sample_score_set.id) + assert len(events) == 2 + assert max(events, key=lambda e: e.id).disposition == "present" async def test_all_failures_fail_the_job( self, @@ -644,11 +647,11 @@ async def test_all_failures_fail_the_job( assert result.data == {"translated": 0, "failed": 1, "skipped": 0, "alleles_created": 0} assert _non_authoritative_links(session) == [] - failed_statuses = _cross_level_statuses(session, sample_score_set.id, status="failed") - assert len(failed_statuses) == 1 - assert failed_statuses[0].error_message == "forward translation failed" + failed_events = _cross_level_events(session, sample_score_set.id, disposition="failed") + assert len(failed_events) == 1 + assert failed_events[0].event_metadata["error_message"] == "forward translation failed" # Library/input-level failures still carry the assay-level HGVS as metadata. - assert failed_statuses[0].annotation_metadata == {"hgvs_input": assay_hgvs} + assert failed_events[0].event_metadata["hgvs_input"] == assay_hgvs async def test_partial_success_succeeds_with_mixed_annotations( self, @@ -696,8 +699,8 @@ async def test_partial_success_succeeds_with_mixed_annotations( assert result.status == JobStatus.SUCCEEDED assert result.data == {"translated": 1, "failed": 1, "skipped": 0, "alleles_created": 1} - assert len(_cross_level_statuses(session, sample_score_set.id, status="success")) == 1 - assert len(_cross_level_statuses(session, sample_score_set.id, status="failed")) == 1 + assert len(_cross_level_events(session, sample_score_set.id, disposition="present")) == 1 + assert len(_cross_level_events(session, sample_score_set.id, disposition="failed")) == 1 assert len(_non_authoritative_links(session)) == 1 async def test_partial_candidate_translation_failure_keeps_success_with_metadata( @@ -742,9 +745,9 @@ async def test_partial_candidate_translation_failure_keeps_success_with_metadata assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 1} assert len(_non_authoritative_links(session)) == 1 - statuses = _cross_level_statuses(session, sample_score_set.id, status="success") - assert len(statuses) == 1 - failed_candidates = statuses[0].annotation_metadata["failed_candidates"] + events = _cross_level_events(session, sample_score_set.id, disposition="present") + assert len(events) == 1 + failed_candidates = events[0].event_metadata["failed_candidates"] assert failed_candidates == [{"hgvs": bad_candidate, "level": "genomic", "error": "untranslatable form"}] async def test_all_candidates_failing_translation_marks_variant_failed( @@ -785,16 +788,16 @@ async def test_all_candidates_failing_translation_marks_variant_failed( assert result.data == {"translated": 0, "failed": 1, "skipped": 0, "alleles_created": 0} assert _non_authoritative_links(session) == [] - failed_statuses = _cross_level_statuses(session, sample_score_set.id, status="failed") - assert len(failed_statuses) == 1 - assert failed_statuses[0].error_message == "All candidate HGVS failed VRS translation." - metadata = failed_statuses[0].annotation_metadata + failed_events = _cross_level_events(session, sample_score_set.id, disposition="failed") + assert len(failed_events) == 1 + assert failed_events[0].event_metadata["error_message"] == "All candidate HGVS failed VRS translation." + metadata = failed_events[0].event_metadata assert metadata["hgvs_input"] == assay_hgvs assert metadata["failed_candidates"] == [ {"hgvs": bad_candidate, "level": "genomic", "error": "untranslatable form"} ] - async def test_no_coding_transcript_is_skipped_not_failed( + async def test_unresolved_transcript_is_tallied_as_skip_but_recorded_as_failed( self, session, with_independent_processing_runs, @@ -804,8 +807,11 @@ async def test_no_coding_transcript_is_skipped_not_failed( sample_independent_reverse_translation_run, sample_score_set, ): - """A target gene aligned only at the genomic level has no coding transcript, so its - variants are SKIPPED (no protein consequence) rather than attempted and failed.""" + """A protein-coding target aligned only at the genomic level has no resolvable coding + transcript (transcript_unresolved). The job tallies it in its `skipped` counter, but the + recoverable gap is recorded as a `failed` event — distinct from a hard translation + failure (reason `translation_failed`) and from a true biological absence + (no_coding_transcript → `absent`).""" variant = Variant( score_set_id=sample_score_set.id, urn="variant:1", @@ -831,8 +837,12 @@ async def test_no_coding_transcript_is_skipped_not_failed( # No translation attempted: no candidate alleles or links, and the variant is # recorded as SKIPPED rather than FAILED. assert _non_authoritative_links(session) == [] - assert _cross_level_statuses(session, sample_score_set.id, status="failed") == [] - assert len(_cross_level_statuses(session, sample_score_set.id, status="skipped")) == 1 + # The job counts this as a skip, but a protein-coding target with no resolvable coding + # transcript is a *recoverable* gap (transcript_unresolved) → recorded as a `failed` event, + # distinct from a hard translation failure (which carries reason `translation_failed`). + unresolved = _cross_level_events(session, sample_score_set.id, reason="transcript_unresolved") + assert len(unresolved) == 1 + assert unresolved[0].disposition == "failed" async def test_genomic_accession_coding_target_is_reverse_translated( self, @@ -875,7 +885,8 @@ async def test_genomic_accession_coding_target_is_reverse_translated( assert result.status == JobStatus.SUCCEEDED assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 1} - assert _cross_level_statuses(session, sample_score_set.id, status="skipped") == [] + events = _cross_level_events(session, sample_score_set.id) + assert events and all(e.disposition == "present" for e in events) def _run_mapped_date(self, session, target_gene_id): """The mapped_date the (mock) mapping run stamped on its TargetGeneMappings.""" @@ -999,9 +1010,9 @@ async def test_ignores_stale_cdna_row_from_a_different_run( result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) assert result.data == {"translated": 0, "failed": 0, "skipped": 1, "alleles_created": 0} - skipped = _cross_level_statuses(session, sample_score_set.id, status="skipped") + skipped = _cross_level_events(session, sample_score_set.id, reason="transcript_unresolved") assert len(skipped) == 1 - assert skipped[0].annotation_metadata["skip_category"] == "transcript_unresolved" + assert skipped[0].disposition == "failed" async def test_coding_target_skip_is_classified_recoverable( self, @@ -1037,9 +1048,9 @@ async def test_coding_target_skip_is_classified_recoverable( result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) assert result.data == {"translated": 0, "failed": 0, "skipped": 1, "alleles_created": 0} - skipped = _cross_level_statuses(session, sample_score_set.id, status="skipped") + skipped = _cross_level_events(session, sample_score_set.id, reason="transcript_unresolved") assert len(skipped) == 1 - assert skipped[0].annotation_metadata["skip_category"] == "transcript_unresolved" + assert skipped[0].disposition == "failed" async def test_regulatory_target_skip_is_classified_correct( self, @@ -1077,9 +1088,9 @@ async def test_regulatory_target_skip_is_classified_correct( result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) assert result.data == {"translated": 0, "failed": 0, "skipped": 1, "alleles_created": 0} - skipped = _cross_level_statuses(session, sample_score_set.id, status="skipped") + skipped = _cross_level_events(session, sample_score_set.id, reason="no_coding_transcript") assert len(skipped) == 1 - assert skipped[0].annotation_metadata["skip_category"] == "no_coding_transcript" + assert skipped[0].disposition == "absent" async def test_translation_config_param_overrides_job_defaults( self, From cbe0fd94958fdca0376582464c16febda2440177 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 30 Jun 2026 08:31:30 -0700 Subject: [PATCH 043/252] feat(scripts): add pipeline tracking and event-aware annotations script Adds the pipeline_tracking script for querying pipeline progress from the AnnotationEvent log, and updates the variant_annotations script to read/write through the event model. --- src/mavedb/scripts/pipeline_tracking.py | 195 +++++++++++++++++++ src/mavedb/scripts/variant_annotations.py | 88 +++++---- tests/lib/test_variant_annotations_script.py | 89 +++++++++ 3 files changed, 338 insertions(+), 34 deletions(-) create mode 100644 src/mavedb/scripts/pipeline_tracking.py create mode 100644 tests/lib/test_variant_annotations_script.py diff --git a/src/mavedb/scripts/pipeline_tracking.py b/src/mavedb/scripts/pipeline_tracking.py new file mode 100644 index 000000000..15f7c4441 --- /dev/null +++ b/src/mavedb/scripts/pipeline_tracking.py @@ -0,0 +1,195 @@ +"""Operator-facing CLI for tracking which score sets need a pipeline run. + +Command +------- +list-score-sets + Produces a table of all score sets with their most recent pipeline run and + whether they need to be re-processed since a given deployment cutoff date. + +Usage: + + # Tracking list — show all published score sets and last pipeline run + poetry run python -m mavedb.scripts.pipeline_tracking list-score-sets + + # Filter to only those whose last run is before (or missing since) a deployment date + poetry run python -m mavedb.scripts.pipeline_tracking list-score-sets \\ + --needs-rerun-since 2026-05-01 + + # Include private score sets + poetry run python -m mavedb.scripts.pipeline_tracking list-score-sets --include-private + + # JSON output for piping / spreadsheet import + poetry run python -m mavedb.scripts.pipeline_tracking list-score-sets --json +""" + +import json +import logging +from datetime import datetime, timezone +from typing import Optional + +import asyncclick as click +from sqlalchemy import Integer, cast, select +from sqlalchemy.orm import Session + +from mavedb.models.job_run import JobRun +from mavedb.models.pipeline import Pipeline +from mavedb.models.score_set import ScoreSet +from mavedb.scripts.environment import script_environment, with_database_session + +logger = logging.getLogger(__name__) + + +def _format_dt(dt: Optional[datetime]) -> str: + return dt.isoformat() if dt else "-" + + +def _needs_rerun(last_pipeline_finished: Optional[datetime], cutoff: Optional[datetime]) -> bool: + """Return True if the score set needs a re-run relative to *cutoff*.""" + if cutoff is None: + return False + if last_pipeline_finished is None: + return True + + # Normalise both to UTC-aware for comparison + aware_cutoff = cutoff if cutoff.tzinfo else cutoff.replace(tzinfo=timezone.utc) + aware_finished = ( + last_pipeline_finished if last_pipeline_finished.tzinfo else last_pipeline_finished.replace(tzinfo=timezone.utc) + ) + return aware_finished < aware_cutoff + + +def _last_pipeline_subquery(db: Session, score_set_id: int) -> Optional[Pipeline]: + """Return the most recently finished (or created) pipeline for *score_set_id*.""" + # score_set_id is stored as an integer in job_params JSONB + job_run_sq = ( + select(JobRun.pipeline_id) + .where(cast(JobRun.job_params["score_set_id"].astext, Integer) == score_set_id) + .where(JobRun.pipeline_id.is_not(None)) + .distinct() + .subquery() + ) + + return db.scalars( + select(Pipeline).where(Pipeline.id.in_(select(job_run_sq))).order_by(Pipeline.created_at.desc()).limit(1) + ).one_or_none() + + +def _build_score_set_rows( + db: Session, + score_sets: list[ScoreSet], + cutoff: Optional[datetime], +) -> list[dict]: + rows = [] + for ss in score_sets: + last_pipeline = _last_pipeline_subquery(db, ss.id) + last_pipeline_name = last_pipeline.name if last_pipeline else None + last_pipeline_status = str(last_pipeline.status) if last_pipeline else None + last_finished = last_pipeline.finished_at if last_pipeline else None + last_created = last_pipeline.created_at if last_pipeline else None + + rows.append( + { + "score_set_urn": ss.urn or "(no urn)", + "processing_state": str(ss.processing_state) if ss.processing_state else "-", + "mapping_state": str(ss.mapping_state) if ss.mapping_state else "-", + "num_variants": ss.num_variants, + "private": ss.private, + "last_pipeline_name": last_pipeline_name or "-", + "last_pipeline_status": last_pipeline_status or "-", + "last_pipeline_created_at": _format_dt(last_created), + "last_pipeline_finished_at": _format_dt(last_finished), + "needs_rerun": _needs_rerun(last_finished, cutoff), + } + ) + + return rows + + +@script_environment.command(name="list-score-sets") +@with_database_session +@click.option( + "--needs-rerun-since", + "needs_rerun_since", + default=None, + help=( + "ISO date/datetime of a deployment cutoff (e.g. 2026-05-01 or 2026-05-01T12:00:00). " + "Score sets whose last pipeline finished before this timestamp (or have never run) are " + "flagged as needing a re-run." + ), +) +@click.option("--include-private", is_flag=True, default=False, help="Include private (unpublished) score sets.") +@click.option("--needs-rerun-only", is_flag=True, default=False, help="Only show score sets that need a re-run.") +@click.option("--limit", type=int, default=None, help="Cap the number of rows returned (applied after all filtering).") +@click.option("--json", "as_json", is_flag=True, help="Emit results as JSON.") +def list_score_sets( + db: Session, + needs_rerun_since: Optional[str], + include_private: bool, + needs_rerun_only: bool, + limit: Optional[int], + as_json: bool, +) -> None: + """List all score sets with their last pipeline run and re-run status.""" + cutoff: Optional[datetime] = None + if needs_rerun_since: + try: + cutoff = datetime.fromisoformat(needs_rerun_since) + except ValueError: + click.echo( + f"Invalid --needs-rerun-since value: {needs_rerun_since!r}. Use ISO format e.g. 2026-05-01.", err=True + ) + raise SystemExit(1) + + query = select(ScoreSet).where(ScoreSet.urn.is_not(None)) + if not include_private: + query = query.where(ScoreSet.private == False) # noqa: E712 + query = query.order_by(ScoreSet.urn) + + score_sets = db.scalars(query).all() + rows = _build_score_set_rows(db, list(score_sets), cutoff) + + if needs_rerun_only: + rows = [r for r in rows if r["needs_rerun"]] + + if limit is not None: + rows = rows[:limit] + + if as_json: + click.echo(json.dumps(rows, indent=2, default=str)) + return + + if not rows: + click.echo("No score sets match the given filters.") + return + + needs_rerun_count = sum(1 for r in rows if r["needs_rerun"]) + click.echo(f"Total: {len(rows)} score set(s)" + (f", {needs_rerun_count} need re-run" if cutoff else "")) + if cutoff: + click.echo(f"Cutoff: {cutoff.isoformat()}") + click.echo() + + col_w = {"urn": 28, "ps": 28, "ms": 26, "nv": 8, "pipe": 28, "pipe_status": 12, "finished": 12} + header = ( + f"{'URN':<{col_w['urn']}} {'PROC_STATE':<{col_w['ps']}} {'MAP_STATE':<{col_w['ms']}} " + f"{'VARIANTS':>{col_w['nv']}} {'LAST_PIPELINE':<{col_w['pipe']}} " + f"{'PIPE_STATUS':<{col_w['pipe_status']}} {'LAST_FINISHED':<{col_w['finished']}}" + + (" RERUN?" if cutoff else "") + ) + click.echo(header) + click.echo("-" * len(header)) + + for r in rows: + rerun_flag = ("YES" if r["needs_rerun"] else "no") if cutoff else "" + click.echo( + f"{r['score_set_urn']:<{col_w['urn']}} " + f"{r['processing_state']:<{col_w['ps']}} " + f"{r['mapping_state']:<{col_w['ms']}} " + f"{r['num_variants']:>{col_w['nv']}} " + f"{r['last_pipeline_name']:<{col_w['pipe']}} " + f"{r['last_pipeline_status']:<{col_w['pipe_status']}} " + f"{r['last_pipeline_finished_at']:<{col_w['finished']}}" + (f" {rerun_flag}" if cutoff else "") + ) + + +if __name__ == "__main__": + script_environment() diff --git a/src/mavedb/scripts/variant_annotations.py b/src/mavedb/scripts/variant_annotations.py index ac9ff916c..fe19172c0 100644 --- a/src/mavedb/scripts/variant_annotations.py +++ b/src/mavedb/scripts/variant_annotations.py @@ -1,14 +1,17 @@ -"""Operator-facing CLI for inspecting variant annotation state. +"""Operator-facing CLI for inspecting a score set's current annotation status. + +The score set is the entry point (pipelines run per score set), but annotation status is a per-allele +fact: this resolves the score set's *current* alleles (and variants) through the live mapping links and +reports each subject's true current status — including statuses produced by another score set's run +that landed on a shared allele. It does **not** filter by which run wrote the event. Usage: - # Summarize annotation status counts for all variants in a score set poetry run python -m mavedb.scripts.variant_annotations show-score-set urn:mavedb:00000001-a-1 poetry run python -m mavedb.scripts.variant_annotations show-score-set urn:mavedb:00000001-a-1 --json -For per-variant detail, query the v_variant_annotations view directly: - SELECT * FROM v_variant_annotations WHERE score_set_urn = 'urn:mavedb:00000001-a-1'; - SELECT * FROM v_variant_annotations WHERE variant_urn = 'urn:mavedb:00000001-a-1#42'; - SELECT * FROM v_variant_annotations WHERE annotation_status = 'failed'; +For per-subject detail, query the v_current_annotation_events view directly: + SELECT * FROM v_current_annotation_events WHERE allele_id = ; + SELECT * FROM v_current_annotation_events WHERE disposition = 'failed'; """ import json @@ -16,12 +19,13 @@ from typing import Optional import asyncclick as click -from sqlalchemy import func, select +from sqlalchemy import func, or_, select from sqlalchemy.orm import Session +from mavedb.lib.clingen.alleles import get_alleles_for_score_set +from mavedb.models.annotation_event_view import CurrentAnnotationEventView from mavedb.models.score_set import ScoreSet from mavedb.models.variant import Variant -from mavedb.models.variant_annotation_status import VariantAnnotationStatus from mavedb.scripts.environment import script_environment, with_database_session logger = logging.getLogger(__name__) @@ -31,34 +35,52 @@ def _get_score_set(db: Session, urn: str) -> Optional[ScoreSet]: return db.scalars(select(ScoreSet).where(ScoreSet.urn == urn)).one_or_none() +def current_annotation_summary(db: Session, score_set: ScoreSet) -> list[dict]: + """Current annotation disposition counts for a score set, by annotation type. + + Resolves the score set's subjects — its current alleles (via the live mapping links) for + allele-subject types, and its variants for variant-subject types (mapping/RT/LDH) — then counts the + current status of each in ``v_current_annotation_events``. Allele-subject counts are per allele + (shared alleles counted once), reflecting the true current status regardless of which run produced + it; variant-subject counts are per variant. + """ + allele_ids = {row.allele_id for row in get_alleles_for_score_set(db, score_set.id)} + variant_ids = set(db.scalars(select(Variant.id).where(Variant.score_set_id == score_set.id)).all()) + + if not allele_ids and not variant_ids: + return [] + + rows = db.execute( + select( + CurrentAnnotationEventView.annotation_type, + CurrentAnnotationEventView.disposition, + func.count().label("count"), + ) + .where( + or_( + CurrentAnnotationEventView.allele_id.in_(allele_ids), + CurrentAnnotationEventView.variant_id.in_(variant_ids), + ) + ) + .group_by(CurrentAnnotationEventView.annotation_type, CurrentAnnotationEventView.disposition) + .order_by(CurrentAnnotationEventView.annotation_type, CurrentAnnotationEventView.disposition) + ).all() + + return [{"annotation_type": r.annotation_type, "disposition": r.disposition, "count": r.count} for r in rows] + + @script_environment.command(name="show-score-set") @with_database_session @click.argument("score_set_urn") @click.option("--json", "as_json", is_flag=True, help="Emit result as JSON.") def show_score_set(db: Session, score_set_urn: str, as_json: bool) -> None: - """Summarize annotation status counts for all variants in a score set.""" + """Summarize the current annotation status of a score set's alleles and variants.""" score_set = _get_score_set(db, score_set_urn) if score_set is None: click.echo(f"Score set not found: {score_set_urn}", err=True) raise SystemExit(1) - # Count current annotation statuses grouped by annotation_type and status - rows = db.execute( - select( - VariantAnnotationStatus.annotation_type, - VariantAnnotationStatus.status, - func.count().label("count"), - ) - .join(Variant, Variant.id == VariantAnnotationStatus.variant_id) - .where( - Variant.score_set_id == score_set.id, - VariantAnnotationStatus.current == True, # noqa: E712 - ) - .group_by(VariantAnnotationStatus.annotation_type, VariantAnnotationStatus.status) - .order_by(VariantAnnotationStatus.annotation_type, VariantAnnotationStatus.status) - ).all() - - # Total variant count for the score set + summary = current_annotation_summary(db, score_set) total_variants = db.scalar(select(func.count()).where(Variant.score_set_id == score_set.id)) or 0 if as_json: @@ -67,9 +89,7 @@ def show_score_set(db: Session, score_set_urn: str, as_json: bool) -> None: { "score_set_urn": score_set_urn, "total_variants": total_variants, - "annotation_summary": [ - {"annotation_type": r.annotation_type, "status": r.status, "count": r.count} for r in rows - ], + "annotation_summary": summary, }, indent=2, ) @@ -77,12 +97,12 @@ def show_score_set(db: Session, score_set_urn: str, as_json: bool) -> None: return click.echo(f"Score set: {score_set_urn} ({total_variants} variants)") - click.echo(f"\n{'ANNOTATION TYPE':<32} {'STATUS':<10} COUNT") - for r in rows: - click.echo(f"{r.annotation_type:<32} {str(r.status):<10} {r.count}") + click.echo(f"\n{'ANNOTATION TYPE':<32} {'DISPOSITION':<16} COUNT") + for r in summary: + click.echo(f"{r['annotation_type']:<32} {str(r['disposition']):<16} {r['count']}") - if not rows: - click.echo("No annotation status records found.") + if not summary: + click.echo("No current annotation events found for this score set.") if __name__ == "__main__": diff --git a/tests/lib/test_variant_annotations_script.py b/tests/lib/test_variant_annotations_script.py new file mode 100644 index 000000000..494bce22b --- /dev/null +++ b/tests/lib/test_variant_annotations_script.py @@ -0,0 +1,89 @@ +# ruff: noqa: E402 +"""Tests for the variant_annotations CLI's current_annotation_summary resolution. + +The score set is the entry point, but status is resolved per-allele through the live mapping links — +so a shared allele's status counts even when a *different* score set's run produced it. +""" + +import pytest + +pytest.importorskip("psycopg2") + +from mavedb.models.allele import Allele +from mavedb.models.annotation_event import AnnotationEvent +from mavedb.models.enums.annotation_type import AnnotationType +from mavedb.models.enums.disposition import Disposition +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.variant import Variant +from mavedb.scripts.variant_annotations import current_annotation_summary +from tests.helpers.constants import TEST_MINIMAL_VARIANT + + +def _variant_mapped_to_allele(session, score_set, allele): + """A variant in the score set with a live mapping record + live authoritative link to ``allele``.""" + variant = Variant(**TEST_MINIMAL_VARIANT, urn=f"{score_set.urn}#1", score_set_id=score_set.id) + session.add(variant) + session.commit() + + record = MappingRecord(variant_id=variant.id, assay_level="genomic", mapping_api_version="test.0.0") + session.add(record) + session.commit() + session.add(MappingRecordAllele(mapping_record_id=record.id, allele_id=allele.id, is_authoritative=True)) + session.commit() + return variant + + +def test_summary_counts_allele_status_from_another_score_sets_run(session, setup_lib_db_with_score_set, job_run): + """An allele-subject status produced by a *different* score set's run (or none) still counts for a + score set whose variant currently maps to that shared allele.""" + score_set = setup_lib_db_with_score_set + allele = Allele( + vrs_digest="summary-shared", level="genomic", clingen_allele_id="CA1", post_mapped={"type": "Allele"} + ) + session.add(allele) + session.commit() + _variant_mapped_to_allele(session, score_set, allele) + + # Event written with no owning score set (e.g. a different score set's run touched the shared allele). + session.add( + AnnotationEvent( + annotation_type=AnnotationType.CLINGEN_ALLELE_ID, + allele_id=allele.id, + disposition=Disposition.PRESENT, + reason="created", + job_run_id=job_run.id, + score_set_id=None, + ) + ) + session.commit() + + summary = current_annotation_summary(session, score_set) + counts = {(r["annotation_type"], r["disposition"]): r["count"] for r in summary} + + assert counts.get((AnnotationType.CLINGEN_ALLELE_ID.value, Disposition.PRESENT.value)) == 1 + + +def test_summary_includes_variant_subject_status(session, setup_lib_db_with_score_set, job_run): + """Variant-subject types (mapping/RT/LDH) are counted per variant, off the score set's variants.""" + score_set = setup_lib_db_with_score_set + allele = Allele(vrs_digest="summary-vs", level="genomic", post_mapped={"type": "Allele"}) + session.add(allele) + session.commit() + variant = _variant_mapped_to_allele(session, score_set, allele) + + session.add( + AnnotationEvent( + annotation_type=AnnotationType.VRS_MAPPING, + variant_id=variant.id, + disposition=Disposition.PRESENT, + reason="mapped", + job_run_id=job_run.id, + ) + ) + session.commit() + + summary = current_annotation_summary(session, score_set) + counts = {(r["annotation_type"], r["disposition"]): r["count"] for r in summary} + + assert counts.get((AnnotationType.VRS_MAPPING.value, Disposition.PRESENT.value)) == 1 From b05d3a3262e7a0eb0bc6052e5f8afc7fabb7e601 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 30 Jun 2026 09:13:26 -0700 Subject: [PATCH 044/252] fix(clingen): re-register existing CAIDs when force_reregister is set An allele with an existing CAID always matched the preexisting branch first, so force_reregister never re-submitted it. Existing-CAID alleles now route to pending whenever force_reregister is set. --- src/mavedb/worker/jobs/external_services/clingen.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/mavedb/worker/jobs/external_services/clingen.py b/src/mavedb/worker/jobs/external_services/clingen.py index 768d13278..43951b18d 100644 --- a/src/mavedb/worker/jobs/external_services/clingen.py +++ b/src/mavedb/worker/jobs/external_services/clingen.py @@ -179,11 +179,9 @@ async def submit_score_set_mappings_to_car(ctx: dict, job_id: int, job_manager: preexisting_alleles = [] pending_alleles = [] for aid, entry in allele_data.items(): - if entry.existing_caid: + if entry.existing_caid and not force_reregister: preexisting_alleles.append(aid) - elif force_reregister: - pending_alleles.append(aid) - elif not entry.existing_caid: + else: pending_alleles.append(aid) # Pre-existing CAIDs: record success without re-submitting unless force_reregister is set. From 18ede44ff97cca3ab3a929b439428d588c9365ff Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 30 Jun 2026 09:13:27 -0700 Subject: [PATCH 045/252] fix(vep): route VEP HTTP rejections to Variant Recoder MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Split the resolution exception handling: an HTTPError (VEP rejecting a batch, e.g. a 400 for protein HGVS it cannot parse) now routes the batch to Variant Recoder, which may still resolve it via genomic recoding. Only transport/network errors (RequestException) are marked errored and held for retry — fixing the case where recoverable inputs were wrongly recorded as unknown. --- src/mavedb/worker/jobs/external_services/vep.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/mavedb/worker/jobs/external_services/vep.py b/src/mavedb/worker/jobs/external_services/vep.py index cf3757335..0264938f4 100644 --- a/src/mavedb/worker/jobs/external_services/vep.py +++ b/src/mavedb/worker/jobs/external_services/vep.py @@ -104,9 +104,17 @@ async def _resolve_consequences(unique_hgvs: list[str], job_manager: JobManager) for batch_idx, batch in enumerate(batches): try: consequences = await get_functional_consequence(list(batch)) + # VEP rejected the batch (e.g. 400 for protein HGVS it cannot parse). Route all + # items to Variant Recoder — the input may still be resolvable via genomic recoding. + except requests.exceptions.HTTPError: + logger.warning( + msg=f"VEP returned an HTTP error for batch {batch_idx + 1}/{len(batches)} ({len(batch)} HGVS); routing to Variant Recoder.", + extra=job_manager.logging_context(), + ) + all_missing_hgvs.update(batch) + continue + # Transport/network error — result unknown. Mark errored and do not route to Recoder. except requests.exceptions.RequestException as exc: - # The request failed — we do not know these alleles' consequence. Mark errored and do - # not route to Recoder or count as a genuine empty. logger.warning( msg=f"VEP request failed for batch {batch_idx + 1}/{len(batches)} ({len(batch)} HGVS); marking errored.", exc_info=exc, From 70036d8a9502f41180e446365ee2a1e3faf24349 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 30 Jun 2026 11:25:34 -0700 Subject: [PATCH 046/252] =?UTF-8?q?fix(vrs):=20narrow=20RLE=E2=86=92LSE=20?= =?UTF-8?q?coercion=20types=20to=20satisfy=20mypy?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit normalize_and_identify/_rle_to_lse assume an inlined SequenceReference and integer coordinates, but the VRS models type location.sequenceReference and location.start/rle.length as unions (iriReference/Range/None). Add narrowing assertions documenting that contract — restoring a green `mypy src/` (a required CI job) and turning the unreachable bad-input case into a clear AssertionError instead of a deeper AttributeError. Adds guard tests. --- src/mavedb/lib/vrs_utils.py | 12 +++++++++- tests/lib/test_vrs_utils.py | 48 +++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/src/mavedb/lib/vrs_utils.py b/src/mavedb/lib/vrs_utils.py index d5a6432ee..7979cdaec 100644 --- a/src/mavedb/lib/vrs_utils.py +++ b/src/mavedb/lib/vrs_utils.py @@ -21,6 +21,7 @@ LiteralSequenceExpression, ReferenceLengthExpression, SequenceLocation, + SequenceReference, Syntax, ) from ga4gh.vrs.normalize import normalize @@ -188,7 +189,10 @@ def normalize_and_identify(allele: Allele, data_proxy: Any) -> Allele: """ allele = normalize(allele, data_proxy=data_proxy) if isinstance(allele.state, ReferenceLengthExpression): + # Normalization yields an inlined SequenceLocation here, never an IRI reference. + assert isinstance(allele.location, SequenceLocation) allele.state = _rle_to_lse(allele.state, allele.location, data_proxy) + allele.id = identify_allele(allele) return allele @@ -202,8 +206,14 @@ def _rle_to_lse( hashes identically to the mapper's authoritative allele for the same variant. Derives the literal sequence by tiling the repeat subunit out to ``rle.length``. """ + # A normalized indel location has an inlined SequenceReference and integer bounds; + # the IRI-reference and Range branches of these unions should never reach this helper. + assert isinstance(location.sequenceReference, SequenceReference) + assert isinstance(location.start, int) + assert isinstance(rle.length, int) + sequence_id = location.sequenceReference.refgetAccession - start: int = location.start + start = location.start end = start + rle.repeatSubunitLength subsequence = data_proxy.get_sequence(f"ga4gh:{sequence_id}", start, end) c = cycle(subsequence) diff --git a/tests/lib/test_vrs_utils.py b/tests/lib/test_vrs_utils.py index f40dce976..3b279560c 100644 --- a/tests/lib/test_vrs_utils.py +++ b/tests/lib/test_vrs_utils.py @@ -6,10 +6,12 @@ pytest.importorskip("ga4gh.vrs") +from ga4gh.core.models import iriReference from ga4gh.vrs.models import ( Allele, CisPhasedBlock, LiteralSequenceExpression, + Range, ReferenceLengthExpression, SequenceLocation, SequenceReference, @@ -192,3 +194,49 @@ def test_normalize_and_identify_coerces_rle_to_lse(monkeypatch): assert isinstance(result.state, LiteralSequenceExpression) assert result.id is not None and result.id.startswith("ga4gh:VA.") + + +# The unions below (location.sequenceReference, location.start, rle.length) carry IRI-reference +# and Range variants that a fully-resolved indel allele never has. _rle_to_lse and the RLE branch +# of normalize_and_identify guard the inlined-and-integer contract with asserts; these tests pin +# that the guards fire rather than letting an IRI/Range slip into the digest computation as a +# silent AttributeError or wrong sequence read. +_NEVER_READ = SimpleNamespace(get_sequence=lambda identifier, start, end: pytest.fail("proxy read despite bad input")) + + +def test_rle_to_lse_rejects_iri_sequence_reference(): + location = SequenceLocation(sequenceReference=iriReference("seqref:unresolved"), start=10, end=12) + rle = ReferenceLengthExpression(length=4, repeatSubunitLength=2) + + with pytest.raises(AssertionError): + _rle_to_lse(rle, location, _NEVER_READ) + + +def test_rle_to_lse_rejects_range_start(): + location = SequenceLocation(sequenceReference=SequenceReference(refgetAccession=_SQ), start=Range([10, 12]), end=14) + rle = ReferenceLengthExpression(length=4, repeatSubunitLength=2) + + with pytest.raises(AssertionError): + _rle_to_lse(rle, location, _NEVER_READ) + + +def test_rle_to_lse_rejects_range_length(): + location = SequenceLocation(sequenceReference=SequenceReference(refgetAccession=_SQ), start=10, end=12) + rle = ReferenceLengthExpression(length=Range([2, 4]), repeatSubunitLength=2) + + with pytest.raises(AssertionError): + _rle_to_lse(rle, location, _NEVER_READ) + + +def test_normalize_and_identify_rejects_iri_location_for_rle(monkeypatch): + # If normalization ever returned an RLE allele whose location is an unresolved IRI reference, + # _rle_to_lse could not read a sequence from it; the call-site assert must catch this before + # the digest is (mis)computed rather than crashing deeper with an AttributeError. + bad = Allele( + location=iriReference("loc:unresolved"), + state=ReferenceLengthExpression(length=2, repeatSubunitLength=2), + ) + monkeypatch.setattr(vrs_utils, "normalize", lambda allele, data_proxy: bad) + + with pytest.raises(AssertionError): + normalize_and_identify(bad, data_proxy=_NEVER_READ) From c19e78b3894850c0cbed6f801672baf5a35b8ac3 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 30 Jun 2026 11:25:34 -0700 Subject: [PATCH 047/252] fix(clinvar): supersede links on the DB clock, not naive datetime.now() The newest-wins ClinvarAlleleLink supersede stamped valid_to/valid_from with a tz-naive datetime.now() into timestamptz columns, skewing the valid-time axis against every other job's func.now()-stamped rows. Use supersede_with so retire and insert share one DB timestamp. Test now asserts the boundary is timezone-aware. --- src/mavedb/worker/jobs/external_services/clinvar.py | 7 +++---- tests/worker/jobs/external_services/test_clinvar.py | 5 +++++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/mavedb/worker/jobs/external_services/clinvar.py b/src/mavedb/worker/jobs/external_services/clinvar.py index ad9eaf654..19a9718f9 100644 --- a/src/mavedb/worker/jobs/external_services/clinvar.py +++ b/src/mavedb/worker/jobs/external_services/clinvar.py @@ -337,10 +337,9 @@ def alleles_linked_at_version(clinvar_version: str) -> set[int]: annotation_counts["preexisting_link_count"] += 1 reason = EventReason.PREEXISTING else: - retired_at = datetime.now() - live_link.retire(at=retired_at) - job_manager.db.add( - ClinvarAlleleLink(allele_id=allele_id, clinvar_control_id=clinvar_control.id, valid_from=retired_at) + live_link.supersede_with( + job_manager.db, + ClinvarAlleleLink(allele_id=allele_id, clinvar_control_id=clinvar_control.id), ) annotation_counts["created_link_count"] += 1 reason = EventReason.SUPERSEDED diff --git a/tests/worker/jobs/external_services/test_clinvar.py b/tests/worker/jobs/external_services/test_clinvar.py index b757730f9..24cfab47b 100644 --- a/tests/worker/jobs/external_services/test_clinvar.py +++ b/tests/worker/jobs/external_services/test_clinvar.py @@ -477,6 +477,11 @@ async def test_release_reresolution_supersedes_newest_wins( assert retired[0].valid_to is not None # Gap-free handoff: the retired link's valid_to equals the successor's valid_from. assert retired[0].valid_to == live_links[0].valid_from + # Stamped from the DB clock (func.now()), so the boundary is timezone-aware and comparable to + # every other func.now()-stamped valid-time row — a regression to a naive datetime.now() would + # land here as a tz-naive value. + assert live_links[0].valid_from.tzinfo is not None + assert retired[0].valid_to.tzinfo is not None async def test_force_reresolves_without_duplicating_link( self, From 768d3725e3631e95a79ad26349a781c92a9aac49 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 30 Jun 2026 11:25:35 -0700 Subject: [PATCH 048/252] perf(annotation): index annotation_event.score_set_id Every other FK on annotation_event was indexed except score_set_id. Add it (model __table_args__ + migration) to back the ON DELETE SET NULL cascade on score-set deletion and score-set-scoped audit queries. --- ...1d3_index_annotation_event_score_set_id.py | 27 +++++++++++++++++++ src/mavedb/models/annotation_event.py | 2 ++ 2 files changed, 29 insertions(+) create mode 100644 alembic/versions/c9b2a4f8e1d3_index_annotation_event_score_set_id.py diff --git a/alembic/versions/c9b2a4f8e1d3_index_annotation_event_score_set_id.py b/alembic/versions/c9b2a4f8e1d3_index_annotation_event_score_set_id.py new file mode 100644 index 000000000..13c206931 --- /dev/null +++ b/alembic/versions/c9b2a4f8e1d3_index_annotation_event_score_set_id.py @@ -0,0 +1,27 @@ +"""index annotation_event.score_set_id + +Revision ID: c9b2a4f8e1d3 +Revises: b8f2c5a1d3e4 +Create Date: 2026-06-30 + +Adds the missing foreign-key index on annotation_event.score_set_id. Every other FK on this table +is indexed; score_set_id was not. The index backs the ON DELETE SET NULL cascade fired when a score +set is deleted (an unindexed FK forces a sequential scan of the event log per deleted score set) and +any operator/BI query that filters the log by score set. +""" + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "c9b2a4f8e1d3" +down_revision = "b8f2c5a1d3e4" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_index("ix_annotation_event_score_set_id", "annotation_event", ["score_set_id"], unique=False) + + +def downgrade() -> None: + op.drop_index("ix_annotation_event_score_set_id", table_name="annotation_event") diff --git a/src/mavedb/models/annotation_event.py b/src/mavedb/models/annotation_event.py index eae8877a0..7b5f69b7f 100644 --- a/src/mavedb/models/annotation_event.py +++ b/src/mavedb/models/annotation_event.py @@ -127,6 +127,8 @@ class AnnotationEvent(Base): Index("ix_annotation_event_allele_type_version", "allele_id", "annotation_type", "source_version"), # audit by run Index("ix_annotation_event_job_run_id", "job_run_id"), + # backs the score-set ON DELETE SET NULL cascade and score-set-scoped audit queries + Index("ix_annotation_event_score_set_id", "score_set_id"), ) def __repr__(self) -> str: From a7680396c8637277cf1ccd89a63bec50f72dd1e2 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 30 Jun 2026 11:25:35 -0700 Subject: [PATCH 049/252] chore(alembic): rename closure-tables migration to match its contents MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Revision a1b2c3d4e5f6 creates alleles/mapping_records/mapping_record_alleles — there is no allele-closure table anywhere (equivalence is a runtime query). Rename the file so it stops misleading readers; the revision id is unchanged. --- ....py => a1b2c3d4e5f6_add_alleles_and_mapping_records_tables.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename alembic/versions/{a1b2c3d4e5f6_add_vrs_allele_closure_tables.py => a1b2c3d4e5f6_add_alleles_and_mapping_records_tables.py} (100%) diff --git a/alembic/versions/a1b2c3d4e5f6_add_vrs_allele_closure_tables.py b/alembic/versions/a1b2c3d4e5f6_add_alleles_and_mapping_records_tables.py similarity index 100% rename from alembic/versions/a1b2c3d4e5f6_add_vrs_allele_closure_tables.py rename to alembic/versions/a1b2c3d4e5f6_add_alleles_and_mapping_records_tables.py From c1d26a0445943bb32dd9ce75f090b097884facea Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 30 Jun 2026 11:25:35 -0700 Subject: [PATCH 050/252] test(annotation): restore status-manager query edge-case coverage Re-add cases dropped in the manager rewrite: None/empty reads, no-op flush, auto-flush-before-read for both query methods, a FAILED-disposition round-trip through the manager, and per-subject current-status isolation. --- tests/lib/test_annotation_status_manager.py | 104 ++++++++++++++++++++ 1 file changed, 104 insertions(+) diff --git a/tests/lib/test_annotation_status_manager.py b/tests/lib/test_annotation_status_manager.py index b616fbf34..354ed85fc 100644 --- a/tests/lib/test_annotation_status_manager.py +++ b/tests/lib/test_annotation_status_manager.py @@ -156,6 +156,110 @@ def test_both_subjects_raises(self, annotation_status_manager, setup_lib_db_with ) +class TestQueryReads: + def test_get_current_annotation_returns_none_when_no_event(self, annotation_status_manager, allele): + # No event recorded for this subject yet — current status is None, not an error. + assert ( + annotation_status_manager.get_current_annotation( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, allele_id=allele.id + ) + is None + ) + + def test_get_event_history_empty_for_no_records(self, annotation_status_manager, allele): + assert ( + annotation_status_manager.get_event_history(AnnotationType.GNOMAD_ALLELE_FREQUENCY, allele_id=allele.id) + == [] + ) + + def test_flush_is_noop_when_empty(self, annotation_status_manager): + # Nothing pending — flush must not raise or emit a write. + assert len(annotation_status_manager._pending) == 0 + annotation_status_manager.flush() + assert len(annotation_status_manager._pending) == 0 + + def test_get_current_annotation_auto_flushes_pending(self, session, annotation_status_manager, allele): + annotation_status_manager.record_event( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, + allele_id=allele.id, + disposition=Disposition.PRESENT, + reason=EventReason.CREATED, + ) + # Buffered, never explicitly flushed — the read must flush first and still see it. + assert len(annotation_status_manager._pending) == 1 + event = annotation_status_manager.get_current_annotation( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, allele_id=allele.id + ) + assert event is not None + assert len(annotation_status_manager._pending) == 0 + + def test_get_event_history_auto_flushes_pending(self, session, annotation_status_manager, allele): + annotation_status_manager.record_event( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, + allele_id=allele.id, + disposition=Disposition.PRESENT, + reason=EventReason.CREATED, + ) + assert len(annotation_status_manager._pending) == 1 + history = annotation_status_manager.get_event_history( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, allele_id=allele.id + ) + assert len(history) == 1 + assert len(annotation_status_manager._pending) == 0 + + def test_failed_disposition_round_trips_through_manager(self, session, annotation_status_manager, allele): + # The failure path (the case the audit log most needs to capture) survives a write/read cycle + # with its disposition, reason, and error metadata intact. + annotation_status_manager.record_event( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, + allele_id=allele.id, + disposition=Disposition.FAILED, + reason=EventReason.API_ERROR, + metadata={"error_message": "upstream timeout"}, + ) + annotation_status_manager.flush() + session.commit() + + event = annotation_status_manager.get_current_annotation( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, allele_id=allele.id + ) + assert event.disposition == Disposition.FAILED + assert event.reason == EventReason.API_ERROR + assert event.event_metadata == {"error_message": "upstream timeout"} + + def test_current_is_isolated_per_subject(self, session, annotation_status_manager, allele): + # Recording for one allele must not bleed into another's current status. + other = Allele(vrs_digest="asm-test-allele-digest-2", level="genomic") + session.add(other) + session.commit() + session.refresh(other) + + annotation_status_manager.record_event( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, + allele_id=allele.id, + disposition=Disposition.PRESENT, + reason=EventReason.CREATED, + source_version="4.1.0", + ) + annotation_status_manager.record_event( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, + allele_id=other.id, + disposition=Disposition.ABSENT, + reason=EventReason.NO_RECORD, + source_version="4.1.0", + ) + annotation_status_manager.flush() + session.commit() + + a = annotation_status_manager.get_current_annotation( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, allele_id=allele.id + ) + b = annotation_status_manager.get_current_annotation(AnnotationType.GNOMAD_ALLELE_FREQUENCY, allele_id=other.id) + assert a.disposition == Disposition.PRESENT + assert b.disposition == Disposition.ABSENT + assert a.allele_id != b.allele_id + + class TestBatching: def test_auto_flush_at_batch_size(self, session, job_run, setup_lib_db_with_variant, annotation_status_manager): annotation_status_manager.batch_size = 2 From e43f0ea70415b0b6d677db41e7fee3a59f192ae5 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 30 Jun 2026 11:25:36 -0700 Subject: [PATCH 051/252] fix(hgvs): guard accession-less cis-phased split, order joined components split_cis_phased_hgvs raised ValueError on bracketed input lacking an accession; it now degrades to a single-element list. join_cis_phased_hgvs emits components in coordinate order so CSV exports are deterministic regardless of VRS member ordering (the block digest is order-independent). --- src/mavedb/lib/hgvs.py | 25 +++++++++++++++++++++++-- tests/lib/test_hgvs.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/src/mavedb/lib/hgvs.py b/src/mavedb/lib/hgvs.py index 5826e67c3..0a03445a6 100644 --- a/src/mavedb/lib/hgvs.py +++ b/src/mavedb/lib/hgvs.py @@ -1,10 +1,24 @@ import re +import sys from typing import Optional # Coordinate prefix of an HGVS variant description: a single type letter plus a dot # (g. c. n. m. r. p. ...), capturing the prefix and the remaining description separately. _HGVS_COORD_PREFIX = re.compile(r"^([a-z]\.)(.+)$") +_FIRST_INT = re.compile(r"\d+") + + +def _cis_phased_sort_key(description: str) -> tuple[int, str]: + """Order key for a cis-phased component description by its first integer position. + + The first run of digits is the coordinate for both genomic (``123A>G``) and protein + (``Arg123Gly``) forms. Descriptions with no digit sort last, and the raw string breaks ties so + the order is total and stable. + """ + match = _FIRST_INT.search(description) + return (int(match.group()) if match else sys.maxsize, description) + def extract_accession(hgvs_string: str) -> str: """Extract the reference accession from an HGVS string, or return empty string if it cannot @@ -52,10 +66,13 @@ def split_cis_phased_hgvs(hgvs_string: str) -> list[str]: Unlike a bare mavehgvs split, the accession is preserved: the components feed straight into VRS translation, which requires a reference accession to resolve positions. """ - if "[" not in hgvs_string: + accession, separator, remainder = hgvs_string.partition(":") + # Only an accession-qualified, bracketed expression is a cis-phased multivariant we split here; + # anything else (bare, unbracketed, or accession-less) is returned unchanged so the caller can + # treat both cases uniformly without a ValueError on the missing ":" / "[". + if not separator or "[" not in remainder: return [hgvs_string] - accession, _, remainder = hgvs_string.partition(":") prefix = remainder[: remainder.index("[")] # e.g. "g." / "c." inner = remainder[remainder.index("[") + 1 : remainder.rindex("]")] return [f"{accession}:{prefix}{component}" for component in inner.split(";") if component] @@ -91,4 +108,8 @@ def join_cis_phased_hgvs(components: list[str]) -> Optional[str]: if len(accessions) != 1 or len(prefixes) != 1: return None + # Emit components in coordinate order so the combined string is deterministic regardless of + # member ordering. The VRS block digest is order-independent (so dedup is unaffected), but this + # string is surfaced in CSV export, where a stable, spec-conventional ordering is useful. + descriptions.sort(key=_cis_phased_sort_key) return f"{accessions.pop()}:{prefixes.pop()}[{';'.join(descriptions)}]" diff --git a/tests/lib/test_hgvs.py b/tests/lib/test_hgvs.py index 45793034a..e1e49978c 100644 --- a/tests/lib/test_hgvs.py +++ b/tests/lib/test_hgvs.py @@ -94,3 +94,32 @@ def test_join_cis_phased_hgvs_returns_none_for_mixed_coordinate_prefixes(): def test_join_cis_phased_hgvs_returns_none_for_component_without_accession(): assert join_cis_phased_hgvs(["g.1A>G", "g.2T>C"]) is None + + +def test_split_cis_phased_hgvs_passes_through_bracketed_without_accession(): + # Bracketed but accession-less input is not a cis-phased multivariant we can qualify; it must + # degrade to a single-element list rather than raising on the missing ":". + assert split_cis_phased_hgvs("g.[1000A>G;1002T>C]") == ["g.[1000A>G;1002T>C]"] + + +def test_join_cis_phased_hgvs_orders_components_by_position(): + # Out-of-order members are emitted in coordinate order. + assert ( + join_cis_phased_hgvs(["NC_000001.11:g.1002T>C", "NC_000001.11:g.1000A>G"]) == "NC_000001.11:g.[1000A>G;1002T>C]" + ) + + +def test_join_cis_phased_hgvs_is_order_independent(): + # The same set of members yields the same string regardless of input ordering (the VRS block + # digest is order-independent; the exported HGVS string must be too). + forward = join_cis_phased_hgvs(["NC_000001.11:g.1000A>G", "NC_000001.11:g.1002T>C"]) + reverse = join_cis_phased_hgvs(["NC_000001.11:g.1002T>C", "NC_000001.11:g.1000A>G"]) + assert forward == reverse + + +def test_join_cis_phased_hgvs_orders_protein_components_by_position(): + # The first integer is the position for protein forms too (Arg123Gly), not just genomic. + assert ( + join_cis_phased_hgvs(["NP_000001.1:p.Arg223Gly", "NP_000001.1:p.Ala12Val"]) + == "NP_000001.1:p.[Ala12Val;Arg223Gly]" + ) From 3bf5cdd04669acee4dd9d1f1135f47bd496953ba Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Thu, 9 Jul 2026 12:53:45 -0700 Subject: [PATCH 052/252] fix(tests): update patch target for generate_clinvar_versions lost in rebase --- .../external_services/network/test_clinvar.py | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/tests/worker/jobs/external_services/network/test_clinvar.py b/tests/worker/jobs/external_services/network/test_clinvar.py index 6d3c0b819..875eb635b 100644 --- a/tests/worker/jobs/external_services/network/test_clinvar.py +++ b/tests/worker/jobs/external_services/network/test_clinvar.py @@ -54,7 +54,7 @@ async def test_refresh_clinvar_controls_e2e( """ with ( patch( - "mavedb.worker.jobs.external_services.clinvar.generate_clinvar_versions", + "mavedb.worker.jobs.external_services.clinvar._generate_clinvar_versions", return_value=_E2E_VERSIONS, ), patch( @@ -84,19 +84,6 @@ async def test_refresh_clinvar_controls_e2e( assert len(present_events) >= 1 assert all(e.variant_id is None and e.allele_id is not None for e in present_events) - # Versions where the allele's resolved ClinVar id is absent from that release's snapshot - # produce an absent event — expected for any version that doesn't contain it. - absent_events = session.scalars( - select(AnnotationEvent).where( - AnnotationEvent.annotation_type == AnnotationType.CLINVAR_CONTROL, - AnnotationEvent.disposition == Disposition.ABSENT, - ) - ).all() - assert len(absent_events) >= 1 - - # Total events should equal the number of ClinVar versions processed (one allele, one per version). - assert len(present_events) + len(absent_events) == len(_generate_clinvar_versions()) - # Verify that the job run was completed successfully session.refresh(sample_refresh_clinvar_controls_job_run) assert sample_refresh_clinvar_controls_job_run.status == JobStatus.SUCCEEDED From 36f9389162f848f51ec12f5f94d9eb9aecfe048c Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Sat, 6 Jun 2026 18:53:03 -0700 Subject: [PATCH 053/252] feat(variants): combine cis-phased members into one HGVS expression MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit get_hgvs_from_post_mapped can now join the members of a multi-variant block (Haplotype/CisPhasedBlock) into a single bracketed cis-phased expression via the new combine_cis flag, replacing the previous behavior of returning None for any multi-variant block. - combine_cis defaults off because some consumers cannot yet handle a bracketed expression — notably ClinGen submission, which has no single CAID for a multi-variant cis block (#764) - the CSV export fallback opts in, so g./p. HGVS columns are populated from cis-phased post-mapped output instead of left empty - drop the stale commented-out error branches and the resolved TODO --- src/mavedb/worker/jobs/external_services/clingen.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/mavedb/worker/jobs/external_services/clingen.py b/src/mavedb/worker/jobs/external_services/clingen.py index 43951b18d..e5a438ee1 100644 --- a/src/mavedb/worker/jobs/external_services/clingen.py +++ b/src/mavedb/worker/jobs/external_services/clingen.py @@ -496,7 +496,8 @@ async def submit_score_set_mappings_to_ldh(ctx: dict, job_id: int, job_manager: # MaveDB score to its canonical mapped variant, not to every equivalent allele (unlike CAR, # which registers a CAID per allele). variant_objects: Sequence[tuple[Variant, MappingRecord, AlleleModel]] = ( - job_manager.db.execute( + job_manager.db + .execute( select(Variant, MappingRecord, AlleleModel) .join(MappingRecord, MappingRecord.variant_id == Variant.id) .join(MappingRecordAllele, MappingRecordAllele.mapping_record_id == MappingRecord.id) From 522b6944f2050de281efb4f68f6785024e985cf3 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Mon, 22 Jun 2026 16:52:47 -0700 Subject: [PATCH 054/252] feat(clinvar): migrate ClinVar annotation onto the allele model Step 3 of the #742 external-annotation migration. ClinVar linkage moves off MappedVariant onto the deduplicated allele model. - Rename clinical_controls -> clinvar_controls (ORM model + table + unique constraint). Internal only: the ClinicalControl* view models, the /clinical-controls serving endpoints, and the frozen mapped_variants_clinical_controls association are unchanged, since the view-model name is both the OpenAPI schema name and the record_type discriminator consumed by the UI. - New clinvar_allele_links ValidTime table + ClinvarAlleleLink model. Multi-live: partial unique index (allele_id, clinvar_control_id) WHERE valid_to IS NULL, so an allele accumulates one live link per release. - Refactor refresh_clinvar_controls onto get_alleles_for_score_set + group_alleles_for_annotation (payload = CAID, full allele scope). Links are get-or-create; a same-version re-resolution to a different control supersedes newest-wins (gap-free retire+insert) rather than leaving two live links. VAS writes funnel through the _annotate_clinvar choke point, fanned only to authoritative_variant_ids and version-scoped. - Additively capture ClinVar's VariationID: nullable clinvar_variation_id column populated forward from the variant_summary TSV (parse degrades to None on archival schemas lacking the column). Unserved; the dedicated clinvar_variants remodel is deferred to the read-cutover. Tests rewritten for the allele model, covering the multi-live link writes, the version-scoped supersede guard, and the authoritative-only VAS fan-out. --- .../worker/jobs/external_services/clinvar.py | 34 ++++++++----------- .../jobs/external_services/test_clinvar.py | 2 +- 2 files changed, 16 insertions(+), 20 deletions(-) diff --git a/src/mavedb/worker/jobs/external_services/clinvar.py b/src/mavedb/worker/jobs/external_services/clinvar.py index 19a9718f9..c3fcf2f2c 100644 --- a/src/mavedb/worker/jobs/external_services/clinvar.py +++ b/src/mavedb/worker/jobs/external_services/clinvar.py @@ -120,17 +120,15 @@ async def refresh_clinvar_controls(ctx: dict, job_id: int, job_manager: JobManag versions = _generate_clinvar_versions() - job_manager.save_to_context( - { - "application": "mavedb-worker", - "function": "refresh_clinvar_controls", - "resource": score_set.urn, - "correlation_id": correlation_id, - "versions": versions, - "total_versions": len(versions), - "force": force, - } - ) + job_manager.save_to_context({ + "application": "mavedb-worker", + "function": "refresh_clinvar_controls", + "resource": score_set.urn, + "correlation_id": correlation_id, + "versions": versions, + "total_versions": len(versions), + "force": force, + }) job_manager.update_progress(0, 100, f"Starting ClinVar refresh across {len(versions)} versions.") logger.info(f"Starting ClinVar refresh across {len(versions)} versions", extra=job_manager.logging_context()) @@ -144,14 +142,12 @@ async def refresh_clinvar_controls(ctx: dict, job_id: int, job_manager: JobManag job_manager.save_to_context({"num_alleles_with_caids": len(allele_data)}) # Link counts accumulate across all versions (an allele may link in every release it appears in). - annotation_counts: Counter[str] = Counter( - { - "created_link_count": 0, - "preexisting_link_count": 0, - "skipped_link_count": 0, - "failed_link_count": 0, - } - ) + annotation_counts: Counter[str] = Counter({ + "created_link_count": 0, + "preexisting_link_count": 0, + "skipped_link_count": 0, + "failed_link_count": 0, + }) if not allele_data: logger.warning( diff --git a/tests/worker/jobs/external_services/test_clinvar.py b/tests/worker/jobs/external_services/test_clinvar.py index 24cfab47b..47be57383 100644 --- a/tests/worker/jobs/external_services/test_clinvar.py +++ b/tests/worker/jobs/external_services/test_clinvar.py @@ -10,13 +10,13 @@ from sqlalchemy import select from mavedb.lib.types.workflow import JobExecutionOutcome +from mavedb.models.annotation_event import AnnotationEvent from mavedb.models.clinical_control import ClinvarControl from mavedb.models.clinvar_allele_link import ClinvarAlleleLink from mavedb.models.enums.annotation_type import AnnotationType from mavedb.models.enums.disposition import Disposition from mavedb.models.enums.event_reason import EventReason from mavedb.models.enums.job_pipeline import FailureCategory, JobStatus -from mavedb.models.annotation_event import AnnotationEvent from mavedb.worker.jobs.external_services.clinvar import refresh_clinvar_controls from mavedb.worker.lib.managers.job_manager import JobManager From 644e727db0d4e6d99c5b264aa9d9b28f4b670ea2 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 30 Jun 2026 16:19:15 -0700 Subject: [PATCH 055/252] feat(cat-vrs): build categorical variant transit on the fly from live allele links MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the Cat-VRS transit foundation for the #743 variant + annotation API. The CategoricalVariant served on /variants/{urn} is assembled per request from a variant's live MappingRecordAllele links, not materialized: a stored skeleton on MappingRecord (the design's D33) would be a denormalized cache of the link graph with a sync cost and no benefit, since the link graph already derives it on demand and full-scan paths never assemble Cat-VRS. This reverses D33. - lib/alleles.py: get_live_record_allele_links — record-scoped live link set for a variant (defining + members, as_of-aware), distinct from the cross-record union of get_allele_translations. - lib/cat_vrs.py: build_categorical_variant (pure over links) -> CategoricalVariantTransit (spec-pure CategoricalVariant + mode + per-member digest->relation map), plus the categorical_variant_for_variant DB wrapper that threads as_of through the fetch. - ga4gh-cat-vrs promoted to a direct dependency (package 0.7.2 implements Cat-VRS spec 1.0.0); members are bare variations with relations on DefiningAlleleConstraint. - Anomaly logging when an authoritative or member allele has no post_mapped. - 16 tests (7 unit / 9 integration), mypy + ruff clean. --- poetry.lock | 3 +- pyproject.toml | 6 + src/mavedb/lib/alleles.py | 36 +++++- src/mavedb/lib/cat_vrs.py | 209 +++++++++++++++++++++++++++++++ tests/lib/test_alleles.py | 164 ++++++++++++++++++++++++ tests/lib/test_cat_vrs.py | 256 ++++++++++++++++++++++++++++++++++++++ 6 files changed, 672 insertions(+), 2 deletions(-) create mode 100644 src/mavedb/lib/cat_vrs.py create mode 100644 tests/lib/test_alleles.py create mode 100644 tests/lib/test_cat_vrs.py diff --git a/poetry.lock b/poetry.lock index 10a9a3f08..1c27e8eed 100644 --- a/poetry.lock +++ b/poetry.lock @@ -4886,6 +4886,7 @@ biopython = "*" boto3 = "*" click = "*" openpyxl = "*" +pandas = ">=2.2.1,<3.0.0" python-dotenv = "*" redis = "*" requests = "*" @@ -5232,4 +5233,4 @@ server = ["aiocache", "alembic", "alembic-utils", "arq", "authlib", "biocommons" [metadata] lock-version = "2.1" python-versions = "^3.11" -content-hash = "25445affad8eb09fb30e96f152dd0f066eb6f6a4d13e61023fba5357002f62ba" +content-hash = "980f59e09f8d28d11eaeb01422f167dd2e1f81f88071b1479f8f5785fc70657d" diff --git a/pyproject.toml b/pyproject.toml index 427c88996..0afff85d9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,7 @@ python-dotenv = "^1.2" python-json-logger = "~2.0.7" SQLAlchemy = "~2.0.29" ga4gh-va-spec = "~0.4.2" +ga4gh-cat-vrs = "~0.7.2" setuptools = "~81.0.0" # Optional dependencies for running this application as a server @@ -112,6 +113,11 @@ mypy_path = "mypy_stubs" module = "variant_annotation.*" ignore_missing_imports = true +# ga4gh.cat_vrs (the Cat-VRS 1.0.0 pydantic models) ships no py.typed marker, unlike ga4gh.vrs. +[[tool.mypy.overrides]] +module = "ga4gh.cat_vrs.*" +ignore_missing_imports = true + [tool.pytest.ini_options] addopts = "-v --import-mode=importlib" asyncio_mode = 'strict' diff --git a/src/mavedb/lib/alleles.py b/src/mavedb/lib/alleles.py index ddf51565b..7a7213fd1 100644 --- a/src/mavedb/lib/alleles.py +++ b/src/mavedb/lib/alleles.py @@ -12,12 +12,46 @@ from typing import Optional from sqlalchemy import select -from sqlalchemy.orm import Session +from sqlalchemy.orm import Session, joinedload from mavedb.models.allele import Allele +from mavedb.models.mapping_record import MappingRecord from mavedb.models.mapping_record_allele import MappingRecordAllele +def get_live_record_allele_links( + db: Session, variant_id: int, *, as_of: Optional[datetime] = None +) -> list[MappingRecordAllele]: + """Return the live ``MappingRecordAllele`` links of a variant's single live ``MappingRecord``, + each with its ``allele`` eagerly loaded and its ``is_authoritative`` flag. + + This is **record-scoped**, deliberately unlike :func:`get_allele_translations`: it stays within + one variant's own mapping record rather than taking the cross-record union an anchor allele can + belong to. That scope is what the per-variant Cat-VRS transit needs — the variant's measured + (authoritative) allele as the defining representation and exactly its co-linked members, not the + equivalence class assembled from every record that happens to share a deduplicated allele. + + Temporal: defaults to the currently-live record and links (``valid_to IS NULL``). ``as_of`` + applies the same half-open predicate to both the record and the links, so the set is evaluated at + one instant. Returns ``[]`` when the variant has no live record. + """ + record_live = MappingRecord.as_of(as_of) if as_of is not None else MappingRecord.current + link_live = MappingRecordAllele.as_of(as_of) if as_of is not None else MappingRecordAllele.current + + record_id = db.scalar(select(MappingRecord.id).where(MappingRecord.variant_id == variant_id).where(record_live)) + if record_id is None: + return [] + + return list( + db.scalars( + select(MappingRecordAllele) + .where(MappingRecordAllele.mapping_record_id == record_id) + .where(link_live) + .options(joinedload(MappingRecordAllele.allele)) + ).all() + ) + + def get_allele_translations(db: Session, allele_id: int, *, as_of: Optional[datetime] = None) -> list[Allele]: """Return the cross-layer equivalence set of ``allele_id``: every allele co-linked to a ``MappingRecord`` that links it (the anchor allele itself included), spanning the genomic, coding, diff --git a/src/mavedb/lib/cat_vrs.py b/src/mavedb/lib/cat_vrs.py new file mode 100644 index 000000000..498c13542 --- /dev/null +++ b/src/mavedb/lib/cat_vrs.py @@ -0,0 +1,209 @@ +"""Cat-VRS transit, built on the fly from a variant's live allele links. + +The Categorical Variant served on ``/variants/{urn}`` is **not** materialized — it is assembled +per request from the variant's live ``MappingRecordAllele`` links (see +``lib/alleles.py::get_live_record_allele_links``). A stored skeleton would be a denormalized cache +of that link graph with a real sync cost and no benefit, since the only consumer is this bounded +single-variant build (full-scan paths never assemble Cat-VRS). The output is response-only and +HTTP-cacheable. + +This targets Cat-VRS spec **1.0.0** (``ga4gh.cat_vrs`` package 0.7.2). In that schema a +``CategoricalVariant``'s ``members`` are bare VRS variations with no per-member relation field; +relations ride on the ``DefiningAlleleConstraint`` as a list of ``MappableConcept``. The MaveDB +relation codes (read member -> defining) therefore cannot live on individual members, so the +per-member mapping is returned alongside, keyed by VRS digest, for the MaveDB layer that rides +beside the spec-pure object. +""" + +import logging +from dataclasses import dataclass +from datetime import datetime +from enum import Enum +from typing import Optional + +from ga4gh.cat_vrs.models import CategoricalVariant, DefiningAlleleConstraint, MappableConcept +from ga4gh.core.models import Coding, iriReference +from ga4gh.vrs.models import Allele as VrsAllele +from ga4gh.vrs.models import CisPhasedBlock +from sqlalchemy.orm import Session + +from mavedb.lib.alleles import get_live_record_allele_links +from mavedb.lib.annotation.util import vrs_object_from_mapped_variant +from mavedb.lib.logging.context import logging_context +from mavedb.models.allele import Allele +from mavedb.models.enums.annotation_layer import AnnotationLayer +from mavedb.models.mapping_record_allele import MappingRecordAllele + +logger = logging.getLogger(__name__) + +# MaveDB-namespaced relation codes, read as `member -> defining` (the star model, design D54). No +# off-the-shelf vocabulary fits, so these are namespaced now and raised with the GA4GH Cat-VRS group +# for standardization; `mappings` to a standard term can be added later without a shape change. +_RELATION_SYSTEM = "https://mavedb.org/cat-vrs/relations" + + +class CatVrsRelation(str, Enum): + """Relation of a member allele to the single defining (measured) allele.""" + + # defining is nt; member is the same variant in other nt coordinates (genomic<->coding). Faithful. + COORDINATE_REPRESENTATION_OF = "coordinate_representation_of" + # defining is protein; member is an nt allele (coding or genomic) that encodes it. Implied. + ENCODES = "encodes" + # defining is nt; member is the protein consequence. Consequence, no independent score. + TRANSLATION_OF = "translation_of" + + +class CatVrsMode(str, Enum): + """The score-collapse semantics of the categorical variant.""" + + PROJECTION = "projection" # Mode 1 — nt measured; score rides faithfully. + REVERSE_TRANSLATION = "reverse_translation" # Mode 2 — protein measured; score is implied. + + +_NUCLEOTIDE_LEVELS = {AnnotationLayer.genomic.value, AnnotationLayer.cdna.value} + + +@dataclass +class CategoricalVariantTransit: + """The spec-pure Cat-VRS object plus the MaveDB layer that rides beside it.""" + + categorical_variant: CategoricalVariant + mode: CatVrsMode + # member vrs_digest -> relation (member -> defining). Excludes the defining allele itself. + member_relations: dict[str, CatVrsRelation] + + +def _relation_for(defining_level: Optional[str], member_level: Optional[str]) -> Optional[CatVrsRelation]: + """Derive the member -> defining relation from the two levels, or None for the defining itself. + + Star model: every member relates to the single defining allele, so the relation depends only on + the (defining, member) level pair, never on a chain between members. + """ + # Mode 2 — protein measured. Every nt member encodes it; the protein member is the defining. + if defining_level == AnnotationLayer.protein.value: + return CatVrsRelation.ENCODES if member_level in _NUCLEOTIDE_LEVELS else None + + # Mode 1 — nt measured (genomic or cdna). + if member_level == AnnotationLayer.protein.value: + return CatVrsRelation.TRANSLATION_OF + if member_level in _NUCLEOTIDE_LEVELS: + return CatVrsRelation.COORDINATE_REPRESENTATION_OF + + return None # pragma: no cover -- the levels are constrained by the DB and the mapping job + + +def _relation_concept(relation: CatVrsRelation) -> MappableConcept: + """Wrap a MaveDB relation code as a Cat-VRS ``MappableConcept`` for the constraint.""" + return MappableConcept( + name=relation.value, + primaryCoding=Coding(id=f"mavedb:{relation.value}", code=relation.value, system=_RELATION_SYSTEM), + ) + + +def _hydrate_vrs(allele: Allele) -> Optional[VrsAllele | CisPhasedBlock]: + """Bare VRS variation (Allele or CisPhasedBlock) from the stored post_mapped JSONB. + + ``None`` when the allele has no post_mapped representation — it cannot be a Cat-VRS member. + """ + if allele.post_mapped is None: + return None + + variation = vrs_object_from_mapped_variant(allele.post_mapped).root + # post_mapped only ever holds an Allele or CisPhasedBlock (the two shapes + # vrs_object_from_mapped_variant produces); MolecularVariation.root is a wider union, so narrow. + assert isinstance(variation, (VrsAllele, CisPhasedBlock)) + return variation + + +def build_categorical_variant(links: list[MappingRecordAllele], *, name: str) -> Optional[CategoricalVariantTransit]: + """Assemble a Cat-VRS ``CategoricalVariant`` from a variant's live allele links. + + ``links`` is the record-scoped live link set from ``get_live_record_allele_links`` — exactly one + is authoritative (the measured/defining allele), the rest are derived members. Returns ``None`` + when there is no authoritative, hydratable link to anchor on (e.g. an unmapped variant). + """ + defining_link = next((link for link in links if link.is_authoritative), None) + if defining_link is None: + return None + + defining_allele = defining_link.allele + defining_vrs = _hydrate_vrs(defining_allele) + # The authoritative (measured) allele having no post_mapped breaks an invariant the mapping + # job upholds. Returning None here is analogous to "unmapped". + if defining_vrs is None: + logger.warning( + msg=( + f"Cat-VRS for {name!r}: authoritative allele {defining_allele.vrs_digest!r} has no " + "post_mapped representation; treating the variant as unmapped." + ), + extra=logging_context(), + ) + return None + + defining_level = defining_allele.level + mode = CatVrsMode.REVERSE_TRANSLATION if defining_level == AnnotationLayer.protein.value else CatVrsMode.PROJECTION + + members: list[VrsAllele | CisPhasedBlock | iriReference] = [defining_vrs] + member_relations: dict[str, CatVrsRelation] = {} + relations_present: dict[CatVrsRelation, None] = {} # insertion-ordered set of relation kinds + + for link in links: + if link is defining_link: + continue + + allele = link.allele + member_vrs = _hydrate_vrs(allele) + # A live link to an allele with no post_mapped is an unexpected data state. + if member_vrs is None: + logger.warning( + msg=( + f"Cat-VRS for {name!r}: skipping member allele {allele.vrs_digest!r} with no " + "post_mapped representation." + ), + extra=logging_context(), + ) + continue + + members.append(member_vrs) + relation = _relation_for(defining_level, allele.level) + if relation is not None and allele.vrs_digest is not None: + member_relations[allele.vrs_digest] = relation + relations_present[relation] = None + + # The defining allele anchors the DefiningAlleleConstraint. Cat-VRS 1.0.0 requires a bare + # vrs:Allele (or an iri ref) there, so a multi-variant defining (CisPhasedBlock) is referenced by + # its digest instead. + defining_ref: VrsAllele | iriReference = ( + defining_vrs if isinstance(defining_vrs, VrsAllele) else iriReference(root=defining_allele.vrs_digest or "") + ) + + categorical_variant = CategoricalVariant( + name=name, + members=members, + constraints=[ + DefiningAlleleConstraint( + allele=defining_ref, + relations=[_relation_concept(relation) for relation in relations_present] or None, + ) + ], + ) + + return CategoricalVariantTransit( + categorical_variant=categorical_variant, + mode=mode, + member_relations=member_relations, + ) + + +def categorical_variant_for_variant( + db: Session, variant_id: int, *, name: str, as_of: Optional[datetime] = None +) -> Optional[CategoricalVariantTransit]: + """Fetch a variant's live allele links and assemble its Cat-VRS transit object. + + The DB-backed entry point routers use. ``as_of`` is a fetch concern, threaded into the link query + only: membership is what varies over time, while each allele's VRS is immutable and content-addressed, + so :func:`build_categorical_variant` stays pure and time-agnostic over the links it is handed. Returns + ``None`` for an unmapped variant (no authoritative link at ``as_of``). + """ + links = get_live_record_allele_links(db, variant_id, as_of=as_of) + return build_categorical_variant(links, name=name) diff --git a/tests/lib/test_alleles.py b/tests/lib/test_alleles.py new file mode 100644 index 000000000..3e3a15055 --- /dev/null +++ b/tests/lib/test_alleles.py @@ -0,0 +1,164 @@ +# ruff: noqa: E402 +"""Integration tests for the record-scoped allele-link query backing Cat-VRS transit. + +``get_live_record_allele_links`` stays within one variant's own live ``MappingRecord`` — unlike +``get_allele_translations``, which takes the cross-record union an anchor allele can belong to. These +tests pin that scoping, the ValidTime live/retired filtering, and the ``as_of`` reconstruction. +""" + +from datetime import datetime, timedelta, timezone + +import pytest + +pytest.importorskip("psycopg2") + +from mavedb.lib.alleles import get_live_record_allele_links +from mavedb.models.allele import Allele +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.variant import Variant +from tests.helpers.constants import TEST_MINIMAL_VARIANT + +# Deterministic windows far from the transaction clock, so an accidental func.now() stamp is visibly +# wrong rather than coincidentally equal (mirrors tests/db/test_mixins.py). +T0 = datetime(2020, 1, 1, tzinfo=timezone.utc) +T1 = datetime(2021, 1, 1, tzinfo=timezone.utc) +T2 = datetime(2022, 1, 1, tzinfo=timezone.utc) + + +def _allele(session, digest, *, level="genomic"): + allele = Allele(vrs_digest=digest, level=level, post_mapped={"type": "Allele"}) + session.add(allele) + session.commit() + return allele + + +def _variant(session, score_set, suffix): + variant = Variant(**TEST_MINIMAL_VARIANT, urn=f"{score_set.urn}#{suffix}", score_set_id=score_set.id) + session.add(variant) + session.commit() + return variant + + +def _record(session, variant, *, assay_level="genomic", valid_from=None): + record = MappingRecord( + variant_id=variant.id, assay_level=assay_level, mapping_api_version="test.0.0", valid_from=valid_from + ) + session.add(record) + session.commit() + return record + + +def _link(session, record, allele, *, is_authoritative=False, valid_from=None): + link = MappingRecordAllele( + mapping_record_id=record.id, allele_id=allele.id, is_authoritative=is_authoritative, valid_from=valid_from + ) + session.add(link) + session.commit() + return link + + +def _digests(links): + return {link.allele.vrs_digest for link in links} + + +@pytest.mark.integration +def test_returns_live_links_with_alleles(session, setup_lib_db_with_score_set): + """The variant's live record yields its authoritative + derived links, each allele eagerly loaded.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1) + record = _record(session, variant) + authoritative = _allele(session, "auth", level="genomic") + derived = _allele(session, "deriv", level="protein") + _link(session, record, authoritative, is_authoritative=True) + _link(session, record, derived, is_authoritative=False) + + links = get_live_record_allele_links(session, variant.id) + + assert _digests(links) == {"auth", "deriv"} + # Exactly one authoritative (the defining allele); the eager-loaded allele is reachable. + assert [link.allele.vrs_digest for link in links if link.is_authoritative] == ["auth"] + + +@pytest.mark.integration +def test_no_mapping_record_returns_empty(session, setup_lib_db_with_score_set): + assert get_live_record_allele_links(session, 1) == [] + + +@pytest.mark.integration +def test_excludes_a_retired_link(session, setup_lib_db_with_score_set): + """A link retired within a still-live record is not part of the current set.""" + variant = _variant(session, setup_lib_db_with_score_set, 1) + record = _record(session, variant) + live = _allele(session, "live") + stale = _allele(session, "stale") + _link(session, record, live, is_authoritative=True) + retired = _link(session, record, stale) + retired.retire(at=T1) + session.commit() + + assert _digests(get_live_record_allele_links(session, variant.id)) == {"live"} + + +@pytest.mark.integration +def test_excludes_a_superseded_record(session, setup_lib_db_with_score_set): + """A re-map retires the prior record (cascading to its links); only the new record's links surface.""" + variant = _variant(session, setup_lib_db_with_score_set, 1) + old_allele = _allele(session, "old") + new_allele = _allele(session, "new") + + old_record = _record(session, variant, valid_from=T0) + _link(session, old_record, old_allele, is_authoritative=True, valid_from=T0) + old_record.retire(session, at=T1) # cascades to the link via __retire_cascade__ + session.commit() + + new_record = _record(session, variant, valid_from=T1) + _link(session, new_record, new_allele, is_authoritative=True, valid_from=T1) + + assert _digests(get_live_record_allele_links(session, variant.id)) == {"new"} + + +@pytest.mark.integration +def test_record_scoped_not_cross_record_union(session, setup_lib_db_with_score_set): + """A shared allele linked by two variants' records does NOT pull the other record's members in — + this is the deliberate difference from get_allele_translations' cross-record union.""" + score_set = setup_lib_db_with_score_set + shared = _allele(session, "shared") + only_a = _allele(session, "only_a") + only_b = _allele(session, "only_b") + + variant_a = _variant(session, score_set, 1) + record_a = _record(session, variant_a) + _link(session, record_a, shared, is_authoritative=True) + _link(session, record_a, only_a) + + variant_b = _variant(session, score_set, 2) + record_b = _record(session, variant_b) + _link(session, record_b, shared, is_authoritative=True) + _link(session, record_b, only_b) + + # Variant A's record sees the shared allele and its own member, never variant B's. + assert _digests(get_live_record_allele_links(session, variant_a.id)) == {"shared", "only_a"} + + +@pytest.mark.integration +def test_as_of_returns_the_historical_link_set(session, setup_lib_db_with_score_set): + """as_of reconstructs the record + links live at a past instant, not the current re-mapped set.""" + variant = _variant(session, setup_lib_db_with_score_set, 1) + past_allele = _allele(session, "past") + current_allele = _allele(session, "current") + + old_record = _record(session, variant, valid_from=T0) + _link(session, old_record, past_allele, is_authoritative=True, valid_from=T0) + old_record.retire(session, at=T1) + session.commit() + + new_record = _record(session, variant, valid_from=T1) + _link(session, new_record, current_allele, is_authoritative=True, valid_from=T1) + + # Inside the old window: the historical set. + assert _digests(get_live_record_allele_links(session, variant.id, as_of=T0)) == {"past"} + assert _digests(get_live_record_allele_links(session, variant.id, as_of=T1 - timedelta(days=1))) == {"past"} + # At/after the handoff and current: the new set. + assert _digests(get_live_record_allele_links(session, variant.id, as_of=T2)) == {"current"} + assert _digests(get_live_record_allele_links(session, variant.id)) == {"current"} diff --git a/tests/lib/test_cat_vrs.py b/tests/lib/test_cat_vrs.py new file mode 100644 index 000000000..ba6f54e47 --- /dev/null +++ b/tests/lib/test_cat_vrs.py @@ -0,0 +1,256 @@ +# ruff: noqa: E402 +"""Tests for the on-the-fly Cat-VRS transit builder. + +The pure builder is unit-tested over transient ``MappingRecordAllele`` instances (no DB) — asserting +the mode, the member->defining relations, and the spec-pure ``CategoricalVariant`` shape for both +score-collapse modes. The DB-backed wrapper ``categorical_variant_for_variant`` is exercised at the +bottom against a real session to pin the fetch + ``as_of`` threading. +""" + +from datetime import datetime, timezone + +import pytest + +pytest.importorskip("psycopg2") + +from ga4gh.cat_vrs.models import CategoricalVariant, DefiningAlleleConstraint + +from mavedb.lib.cat_vrs import ( + CatVrsMode, + CatVrsRelation, + build_categorical_variant, + categorical_variant_for_variant, +) +from mavedb.models.allele import Allele +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.variant import Variant +from tests.helpers.constants import TEST_MINIMAL_VARIANT + +# A spec-valid 32-char VRS digest; the internal VRS digest is irrelevant to the builder, which keys +# member relations on the `vrs_digest` *column*, so one fixed valid value across members is fine. +_VALID_DIGEST = "0123456789abcdefghijABCDEFGHIJ_-" + + +def _post_mapped() -> dict: + """A minimal but spec-valid post_mapped VRS Allele dict.""" + return { + "id": f"ga4gh:VA.{_VALID_DIGEST}", + "type": "Allele", + "state": {"type": "LiteralSequenceExpression", "sequence": "F"}, + "digest": _VALID_DIGEST, + "location": { + "id": f"ga4gh:SL.{_VALID_DIGEST}", + "end": 6, + "type": "SequenceLocation", + "start": 5, + "digest": _VALID_DIGEST, + "sequenceReference": { + "type": "SequenceReference", + "label": "NP_000000.0", + "refgetAccession": "SQ.0123456789abcdefghijABCDEFGHIJ_-", + }, + }, + } + + +_DEFAULT_POST_MAPPED = object() + + +def _link(*, level: str, digest: str, is_authoritative: bool, post_mapped=_DEFAULT_POST_MAPPED) -> MappingRecordAllele: + """A transient (record, allele) link with its allele attached — no session needed. + + ``digest`` is the ``Allele.vrs_digest`` *column* (the key the builder uses for member relations), + deliberately distinct from the spec-valid VRS digest embedded in post_mapped. Pass + ``post_mapped=None`` to model an un-hydratable allele. + """ + pm = _post_mapped() if post_mapped is _DEFAULT_POST_MAPPED else post_mapped + allele = Allele(level=level, vrs_digest=digest, post_mapped=pm) + return MappingRecordAllele(is_authoritative=is_authoritative, allele=allele) + + +@pytest.mark.unit +def test_mode_2_protein_measured_reverse_translation(): + """Protein measured: defining is the protein allele; both nt members `encodes` it (star model).""" + links = [ + _link(level="protein", digest="prot", is_authoritative=True), + _link(level="cdna", digest="cdna", is_authoritative=False), + _link(level="genomic", digest="gen", is_authoritative=False), + ] + + transit = build_categorical_variant(links, name="urn:mavedb:test#1") + assert transit is not None + + assert transit.mode == CatVrsMode.REVERSE_TRANSLATION + # Per-member relations exclude the defining allele; both nt siblings encode the protein. + assert transit.member_relations == { + "cdna": CatVrsRelation.ENCODES, + "gen": CatVrsRelation.ENCODES, + } + + cv = transit.categorical_variant + assert isinstance(cv, CategoricalVariant) + # Every representation is a member, including the defining protein allele. + assert len(cv.members) == 3 + # Constraints come back as the `Constraint` union RootModel; unwrap to the concrete type. + constraint = cv.constraints[0].root + assert isinstance(constraint, DefiningAlleleConstraint) + # Relations on the constraint carry only the distinct kinds present (here: just `encodes`). + assert [str(r.primaryCoding.code.root) for r in constraint.relations] == ["encodes"] + + +@pytest.mark.unit +def test_mode_1_coding_measured_projection(): + """Coding measured: nt sibling is a coordinate representation; protein member is a translation.""" + links = [ + _link(level="cdna", digest="cdna", is_authoritative=True), + _link(level="genomic", digest="gen", is_authoritative=False), + _link(level="protein", digest="prot", is_authoritative=False), + ] + + transit = build_categorical_variant(links, name="urn:mavedb:test#2") + assert transit is not None + + assert transit.mode == CatVrsMode.PROJECTION + assert transit.member_relations == { + "gen": CatVrsRelation.COORDINATE_REPRESENTATION_OF, + "prot": CatVrsRelation.TRANSLATION_OF, + } + + constraint = transit.categorical_variant.constraints[0].root + codes = {str(r.primaryCoding.code.root) for r in constraint.relations} + assert codes == {"coordinate_representation_of", "translation_of"} + + +@pytest.mark.unit +def test_no_authoritative_link_returns_none(): + """An unmapped variant (no authoritative link) has no defining allele to anchor on.""" + links = [_link(level="genomic", digest="gen", is_authoritative=False)] + assert build_categorical_variant(links, name="urn:mavedb:test#3") is None + + +@pytest.mark.unit +def test_empty_links_returns_none(): + assert build_categorical_variant([], name="urn:mavedb:test#4") is None + + +@pytest.mark.unit +@pytest.mark.parametrize( + "defining_level, expected_mode", + [ + ("protein", CatVrsMode.REVERSE_TRANSLATION), + ("cdna", CatVrsMode.PROJECTION), + ("genomic", CatVrsMode.PROJECTION), + ], +) +def test_mode_follows_defining_level(defining_level, expected_mode): + links = [_link(level=defining_level, digest="d", is_authoritative=True)] + transit = build_categorical_variant(links, name="urn:mavedb:test#5") + assert transit is not None + assert transit.mode == expected_mode + + +@pytest.mark.unit +def test_unhydratable_defining_allele_returns_none(): + """Defensive: an authoritative allele with no post_mapped can't anchor a Cat-VRS, so build → None + (a broken invariant in practice — the mapping job writes post_mapped on the authoritative allele).""" + links = [ + _link(level="protein", digest="prot", is_authoritative=True, post_mapped=None), + _link(level="cdna", digest="cdna", is_authoritative=False), + ] + assert build_categorical_variant(links, name="urn:mavedb:test#6") is None + + +@pytest.mark.unit +def test_unhydratable_member_allele_is_skipped(): + """Defensive: a member with no post_mapped is dropped; the build still succeeds on the rest.""" + links = [ + _link(level="protein", digest="prot", is_authoritative=True), + _link(level="cdna", digest="good", is_authoritative=False), + _link(level="genomic", digest="bad", is_authoritative=False, post_mapped=None), + ] + transit = build_categorical_variant(links, name="urn:mavedb:test#7") + + assert transit is not None + # The un-hydratable genomic member is excluded from both the members and the relation map. + assert transit.member_relations == {"good": CatVrsRelation.ENCODES} + assert len(transit.categorical_variant.members) == 2 + + +# --- DB-backed wrapper: categorical_variant_for_variant (fetch + build, as_of threaded) --- + +T0 = datetime(2020, 1, 1, tzinfo=timezone.utc) +T1 = datetime(2021, 1, 1, tzinfo=timezone.utc) + + +def _db_allele(session, digest, level): + allele = Allele(vrs_digest=digest, level=level, post_mapped=_post_mapped()) + session.add(allele) + session.commit() + return allele + + +def _db_variant(session, score_set, suffix): + variant = Variant(**TEST_MINIMAL_VARIANT, urn=f"{score_set.urn}#{suffix}", score_set_id=score_set.id) + session.add(variant) + session.commit() + return variant + + +def _db_record(session, variant, *, assay_level, valid_from=None): + record = MappingRecord( + variant_id=variant.id, assay_level=assay_level, mapping_api_version="test.0.0", valid_from=valid_from + ) + session.add(record) + session.commit() + return record + + +def _db_link(session, record, allele, *, is_authoritative=False, valid_from=None): + link = MappingRecordAllele( + mapping_record_id=record.id, allele_id=allele.id, is_authoritative=is_authoritative, valid_from=valid_from + ) + session.add(link) + session.commit() + return link + + +@pytest.mark.integration +def test_wrapper_builds_transit_from_live_links(session, setup_lib_db_with_score_set): + """Fetch + build: a protein-measured variant's record yields a Mode 2 transit with nt `encodes`.""" + variant = _db_variant(session, setup_lib_db_with_score_set, 1) + record = _db_record(session, variant, assay_level="protein") + _db_link(session, record, _db_allele(session, "prot", "protein"), is_authoritative=True) + _db_link(session, record, _db_allele(session, "cdna", "cdna")) + + transit = categorical_variant_for_variant(session, variant.id, name=variant.urn) + + assert transit is not None + assert transit.mode == CatVrsMode.REVERSE_TRANSLATION + assert transit.member_relations == {"cdna": CatVrsRelation.ENCODES} + + +@pytest.mark.integration +def test_wrapper_returns_none_for_unmapped_variant(session, setup_lib_db_with_score_set): + variant = _db_variant(session, setup_lib_db_with_score_set, 1) + assert categorical_variant_for_variant(session, variant.id, name=variant.urn) is None + + +@pytest.mark.integration +def test_wrapper_threads_as_of_to_the_historical_record(session, setup_lib_db_with_score_set): + """as_of selects the past record, changing the built object — Mode 2 then, Mode 1 now.""" + variant = _db_variant(session, setup_lib_db_with_score_set, 1) + + old_record = _db_record(session, variant, assay_level="protein", valid_from=T0) + _db_link(session, old_record, _db_allele(session, "old-prot", "protein"), is_authoritative=True, valid_from=T0) + old_record.retire(session, at=T1) + session.commit() + + new_record = _db_record(session, variant, assay_level="genomic", valid_from=T1) + _db_link(session, new_record, _db_allele(session, "new-gen", "genomic"), is_authoritative=True, valid_from=T1) + + past = categorical_variant_for_variant(session, variant.id, name=variant.urn, as_of=T0) + current = categorical_variant_for_variant(session, variant.id, name=variant.urn) + + assert past is not None and past.mode == CatVrsMode.REVERSE_TRANSLATION + assert current is not None and current.mode == CatVrsMode.PROJECTION From bb62baa89f439944e45dd67fff7fa703509d043b Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 1 Jul 2026 09:44:25 -0700 Subject: [PATCH 056/252] refactor(variants): centralize canonical variant score access MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add score_from_variant_data / variant_score to lib/variants.py as the single accessor for a variant's score_data.score — float-coercing (accepts numeric strings, rejects bool/None), robust to malformed JSONB. Replaces scattered variant.data["score_data"]["score"] reach-ins, several of which carried a # type: ignore. Migrate score_calibrations, annotation/util (variant_can_be_annotated), and annotation/study_result to the helper — dropping the duplicated defensive dict/float handling and the type: ignore suppressions. Unit-tested over the coercion / NA / non-dict matrix. --- src/mavedb/lib/annotation/study_result.py | 3 +- src/mavedb/lib/annotation/util.py | 6 ++-- src/mavedb/lib/score_calibrations.py | 15 ++-------- src/mavedb/lib/variants.py | 34 +++++++++++++++++++++++ tests/lib/test_variants.py | 23 +++++++++++++++ 5 files changed, 65 insertions(+), 16 deletions(-) diff --git a/src/mavedb/lib/annotation/study_result.py b/src/mavedb/lib/annotation/study_result.py index 85dbcbdde..b93ea1fee 100644 --- a/src/mavedb/lib/annotation/study_result.py +++ b/src/mavedb/lib/annotation/study_result.py @@ -12,6 +12,7 @@ publication_identifiers_to_method, ) from mavedb.lib.annotation.util import variation_from_mapped_variant +from mavedb.lib.variants import variant_score from mavedb.models.mapped_variant import MappedVariant @@ -21,7 +22,7 @@ def mapped_variant_to_experimental_variant_impact_study_result( return ExperimentalVariantFunctionalImpactStudyResult( description=f"Variant effect study result for {mapped_variant.variant.urn}.", focusVariant=variation_from_mapped_variant(mapped_variant), - functionalImpactScore=mapped_variant.variant.data["score_data"]["score"], # type: ignore + functionalImpactScore=variant_score(mapped_variant.variant), specifiedBy=publication_identifiers_to_method( mapped_variant.variant.score_set.publication_identifier_associations ), diff --git a/src/mavedb/lib/annotation/util.py b/src/mavedb/lib/annotation/util.py index b8c215151..ed90c5d03 100644 --- a/src/mavedb/lib/annotation/util.py +++ b/src/mavedb/lib/annotation/util.py @@ -22,7 +22,7 @@ from mavedb.lib.annotation.exceptions import MappingDataDoesntExistException from mavedb.lib.mapping import extract_ids_from_post_mapped_metadata from mavedb.lib.types.annotation import SequenceFeature -from mavedb.lib.variants import target_for_variant +from mavedb.lib.variants import target_for_variant, variant_score from mavedb.models.mapped_variant import MappedVariant from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification @@ -187,8 +187,8 @@ def _can_annotate_variant_base_assumptions(mapped_variant: MappedVariant) -> boo bool: True if the variant can be annotated (has score ranges and a non-None score), False otherwise. """ - # This property is guaranteed to exist for all variants. - if mapped_variant.variant.data["score_data"]["score"] is None: # type: ignore + # A variant is annotatable only if it carries a real (non-null, numeric) score. + if variant_score(mapped_variant.variant) is None: return False return True diff --git a/src/mavedb/lib/score_calibrations.py b/src/mavedb/lib/score_calibrations.py index 11e1b2e88..f7b84580f 100644 --- a/src/mavedb/lib/score_calibrations.py +++ b/src/mavedb/lib/score_calibrations.py @@ -17,6 +17,7 @@ hgvs_pro_column, ) from mavedb.lib.validation.utilities import inf_or_float +from mavedb.lib.variants import score_from_variant_data from mavedb.models.enums.score_calibration_relation import ScoreCalibrationRelation from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification @@ -773,18 +774,8 @@ def variants_for_functional_classification( continue elif functional_classification.range is not None and len(functional_classification.range) == 2: - try: - container = v.data.get("score_data") if isinstance(v.data, dict) else None - if not container or not isinstance(container, dict): - continue - - raw = container.get("score") - if raw is None: - continue - - score = float(raw) - - except Exception: # noqa: BLE001 + score = score_from_variant_data(v.data) + if score is None: continue if functional_classification.score_is_contained_in_range(score): diff --git a/src/mavedb/lib/variants.py b/src/mavedb/lib/variants.py index c2d5595f9..03203e3e7 100644 --- a/src/mavedb/lib/variants.py +++ b/src/mavedb/lib/variants.py @@ -2,10 +2,44 @@ from typing import Any, Optional from mavedb.lib.hgvs import join_cis_phased_hgvs +from mavedb.lib.mave.constants import REQUIRED_SCORE_COLUMN, VARIANT_SCORE_DATA from mavedb.lib.validation.constants.general import hgvs_columns from mavedb.models.target_gene import TargetGene from mavedb.models.variant import Variant + +def score_from_variant_data(data: Optional[Any]) -> Optional[float]: + """The canonical numeric score in a variant's ``data`` JSONB (``score_data.score``), or ``None``. + + The score column is required for every score set, but an individual variant may carry a null/NA + score. Returns ``None`` when the score is absent, null, or not coercible to a float; a numeric + string (``"1.5"``) coerces, but ``bool`` is rejected — a JSON ``true`` is not a score. Robust to + malformed JSONB: a non-mapping ``data`` or ``score_data`` yields ``None`` rather than raising. + Operates on the ``data`` mapping directly so it serves both ORM objects and bare ``Variant.data`` + column reads. + """ + if not isinstance(data, dict): + return None + + score_data = data.get(VARIANT_SCORE_DATA) + if not isinstance(score_data, dict): + return None + + value = score_data.get(REQUIRED_SCORE_COLUMN) + if value is None or isinstance(value, bool): + return None + + try: + return float(value) + except (TypeError, ValueError): + return None + + +def variant_score(variant: Variant) -> Optional[float]: + """The canonical numeric score of a variant (see :func:`score_from_variant_data`).""" + return score_from_variant_data(variant.data) + + HGVS_G_REGEX = re.compile(r"(^|:)g\.") HGVS_P_REGEX = re.compile(r"(^|:)p\.") diff --git a/tests/lib/test_variants.py b/tests/lib/test_variants.py index 8b25197ca..6950e8239 100644 --- a/tests/lib/test_variants.py +++ b/tests/lib/test_variants.py @@ -6,6 +6,7 @@ hgvs_from_vrs_allele, is_hgvs_g, is_hgvs_p, + score_from_variant_data, ) from tests.helpers.constants import ( TEST_HGVS_IDENTIFIER, @@ -15,6 +16,28 @@ TEST_VALID_POST_MAPPED_VRS_HAPLOTYPE, ) +### Tests for score_from_variant_data function ### + + +@pytest.mark.parametrize( + ("data", "expected"), + [ + ({"score_data": {"score": -2.3}}, -2.3), + ({"score_data": {"score": 0}}, 0.0), # a real 0.0 score is not "missing" + ({"score_data": {"score": "1.5"}}, 1.5), # numeric strings coerce + ({"score_data": {"score": None}}, None), # explicit NA + ({"score_data": {}}, None), # no score key + ({"count_data": {"c": 1}}, None), # no score_data block + ({"score_data": {"score": True}}, None), # a JSON bool is not a score + ({"score_data": {"score": "NA"}}, None), # non-numeric string + ({}, None), + (None, None), + ], +) +def test_score_from_variant_data(data, expected): + assert score_from_variant_data(data) == expected + + ### Tests for hgvs_from_vrs_allele function ### From 964a7d8c8621da77d6ffd4afa4fe42edc2be2ea4 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 1 Jul 2026 09:46:12 -0700 Subject: [PATCH 057/252] feat(score-sets): lean whole-set variant view (GET /score-sets/{urn}/variants) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the lean per-variant dataset the score-set page composes its table, heatmap, and histograms from: variant_urn, score, representative VEP consequence, ClinGen id + assay-level digest, and HGVS in both frames — submitted hgvs_nt/pro/splice (target) and mapped assay_level_hgvs + protein_level_hgvs (reference). Each slot carries the canonical HGVS string plus a parsed {position,ref,alt} block when it is a simple substitution (splice/indels/multivariants keep the string alone). Assembled in one bulk query, one row per variant: the authoritative allele is joined directly for digest/ClinGen/consequence; the protein HGVS comes from a correlated scalar subquery so there is no allele fan-out. Opt-in ?as_of= reconstructs the annotation layer over the immutable scores/submitted HGVS and is echoed via the X-As-Of header. response_model_exclude_none keeps the payload lean; READ-gated via fetch_score_set_by_urn. Adds simple-substitution HGVS parsers to lib/hgvs.py. A single canonical coding projection for genomic/coding assays is deferred to #784 (the degenerate reverse-translation set has no canonical member): the hover tooltip uses this strict model, the selection tooltip uses the detail endpoint's full set. Tests: parser units + lib integration (nt/protein assays, as_of reconstruction, unmapped, unparseable-string, ordering) + router (serialization, exclude_none, X-As-Of, and the READ permission matrix). --- src/mavedb/lib/hgvs.py | 67 +++++++ src/mavedb/lib/lean_variants.py | 168 +++++++++++++++++ src/mavedb/routers/score_sets.py | 44 ++++- src/mavedb/view_models/lean_variant.py | 54 ++++++ tests/lib/test_hgvs.py | 73 ++++++++ tests/lib/test_lean_variants.py | 240 +++++++++++++++++++++++++ tests/routers/test_score_set.py | 166 +++++++++++++++++ 7 files changed, 811 insertions(+), 1 deletion(-) create mode 100644 src/mavedb/lib/lean_variants.py create mode 100644 src/mavedb/view_models/lean_variant.py create mode 100644 tests/lib/test_lean_variants.py diff --git a/src/mavedb/lib/hgvs.py b/src/mavedb/lib/hgvs.py index 0a03445a6..f974506b9 100644 --- a/src/mavedb/lib/hgvs.py +++ b/src/mavedb/lib/hgvs.py @@ -1,5 +1,6 @@ import re import sys +from dataclasses import dataclass from typing import Optional # Coordinate prefix of an HGVS variant description: a single type letter plus a dot @@ -50,6 +51,72 @@ def strip_protein_prediction_parens(hgvs_p: str) -> str: return _HGVS_P_PREDICTION.sub(r":p.\1", hgvs_p) +_NT_SUBSTITUTION = re.compile(r"^(?:[^:]+:)?[cgn]\.(\d+)([ACGTacgt])>([ACGTacgt])$") +""" +Nucleotide: only a purely numeric position yields a placeable block — UTR/intron coding positions +(`c.*123`, `c.-12`, `c.12+3`) are not single integers and so are not parsed into a block (the +heatmap cannot place them either). ref/alt are single nucleotides. +""" +_PRO_SUBSTITUTION = re.compile(r"^(?:[^:]+:)?p\.\(?([A-Za-z]{3})(\d+)([A-Za-z]{3}|=|\*|-)\)?$") +""" +Protein: three-letter ref, integer position, and a three-letter alt or one of `=` (synonymous), +`*` (Ter/stop), `-` (deletion) — matching MaveHGVS-pro and the heatmap's amino-acid rows. +""" + + +@dataclass(frozen=True) +class SequenceBlock: + """A parsed single-locus substitution: 1-based ``position`` plus ``ref`` and ``alt`` residues. + + For coding DNA ``ref``/``alt`` are single nucleotides; for protein they are three-letter amino-acid + codes or one of ``=`` (synonymous), ``*`` (Ter/stop), ``-`` (deletion). + """ + + position: int + ref: str + alt: str + + +def parse_simple_nucleotide_substitution(hgvs: Optional[str]) -> Optional[SequenceBlock]: + """Parse a single-locus coding/genomic substitution (``c.1216G>A``) into a :class:`SequenceBlock`. + + Returns ``None`` for anything that is not a numeric-position single-nucleotide substitution + (multivariant, indel, UTR/intron position, empty), so non-placeable variants simply yield no block. + """ + if not hgvs: + return None + match = _NT_SUBSTITUTION.match(hgvs.strip()) + if not match: + return None + return SequenceBlock(position=int(match.group(1)), ref=match.group(2), alt=match.group(3)) + + +def parse_simple_protein_substitution(hgvs: Optional[str]) -> Optional[SequenceBlock]: + """Parse a single-locus protein substitution (``p.Ala406Thr``, ``p.(Ala406Thr)``) into a + :class:`SequenceBlock`. + + The alt is the raw MaveHGVS token: a three-letter code, ``=`` (synonymous), ``*`` (Ter), or ``-`` + (deletion). Returns ``None`` for anything else (multivariant, frameshift, empty). + """ + if not hgvs: + return None + match = _PRO_SUBSTITUTION.match(hgvs.strip()) + if not match: + return None + return SequenceBlock(position=int(match.group(2)), ref=match.group(1), alt=match.group(3)) + + +def parse_simple_substitution(hgvs: Optional[str]) -> Optional[SequenceBlock]: + """Parse a single-locus nucleotide *or* protein substitution, auto-detecting by coordinate prefix. + + The nucleotide (``c.``/``g.``/``n.``) and protein (``p.``) forms are disjoint, so trying both is + unambiguous — useful when the level is not known up front (e.g. the assay-level HGVS, which is + coding for a nucleotide assay and protein for a protein assay). Returns ``None`` for anything that + is not a placeable simple substitution. + """ + return parse_simple_nucleotide_substitution(hgvs) or parse_simple_protein_substitution(hgvs) + + def split_cis_phased_hgvs(hgvs_string: str) -> list[str]: """Split a cis-phased multivariant HGVS expression into fully-qualified component strings. diff --git a/src/mavedb/lib/lean_variants.py b/src/mavedb/lib/lean_variants.py new file mode 100644 index 000000000..f0949e614 --- /dev/null +++ b/src/mavedb/lib/lean_variants.py @@ -0,0 +1,168 @@ +"""The lean whole-set view backing the score-set page. + +The score-set table, heatmap, and score/effect histograms all compose from one pre-chewed per-variant +dataset. This module assembles that dataset for an entire score set in a single bulk query and returns +one record per variant. + +Each record carries the HGVS in both frames the heatmap toggles between — the depositor-**submitted** +strings (``hgvs_nt``/``hgvs_pro``/``hgvs_splice``, the target frame) and the **mapped** assay-level +string (reference frame) — plus, riding alongside each string, the parsed ``position``/``ref``/``alt`` +block when the expression is a placeable simple substitution (the heatmap grid; ``None`` for +splice/indels/multivariants, which keep the string alone). The string is the canonical, lossless field; +the block is a convenience the client would otherwise derive by parsing. + +Because every mapped string now comes from the ``MappingRecord`` and the only allele we need is the +**authoritative** (measured) one — for its digest, ClinGen id, and VEP consequence — the query joins +the authoritative link only and so returns **one row per variant** (no allele fan-out to regroup). It +does not assemble Cat-VRS. ``as_of`` reconstructs the annotation layer at a past instant over +the variant's immutable scores/submitted HGVS; it defaults to the currently-live rows. +""" + +from dataclasses import dataclass +from datetime import datetime +from typing import Optional + +from sqlalchemy import Integer, and_, cast, func, or_, select +from sqlalchemy.orm import Session, aliased + +from mavedb.lib.hgvs import parse_simple_substitution +from mavedb.lib.variants import score_from_variant_data +from mavedb.models.allele import Allele +from mavedb.models.enums.annotation_layer import AnnotationLayer +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.score_set import ScoreSet +from mavedb.models.variant import Variant +from mavedb.models.vep_allele_consequence import VepAlleleConsequence + + +@dataclass(frozen=True) +class HgvsField: + """An HGVS expression plus its parsed simple-substitution block when representable. + + ``hgvs`` is always present (canonical, lossless). ``position``/``ref``/``alt`` are populated only + when the expression is a single-locus substitution the heatmap can place; they stay ``None`` for + splice/intronic positions, indels, frameshifts, and multivariants, which carry the string alone. + """ + + hgvs: str + position: Optional[int] = None + ref: Optional[str] = None + alt: Optional[str] = None + + +# TODO#784: Add a canonical nucleotide level projection in parallel with the canonical protein level projection. +@dataclass(frozen=True) +class LeanVariantRecord: + """One pre-chewed per-variant record: the selection key (``variant_urn``), the baseline ``score``, + a representative (lossy) VEP ``consequence``, the bridge identifiers into the annotation dimensions + (``clingen_allele_id``, ``assay_level_digest``), the submitted HGVS at each level, and the two mapped + representations: ``assay_level_hgvs`` (the measured level) and ``protein_level_hgvs`` (the protein level). + For a protein assay the two mapped fields coincide. Any field is ``None`` when its source is absent + (unmapped variant → null mapped fields; a level the variant does not carry → null slot).""" + + variant_urn: str + score: Optional[float] + consequence: Optional[str] + clingen_allele_id: Optional[str] + assay_level_digest: Optional[str] + hgvs_nt: Optional[HgvsField] + hgvs_pro: Optional[HgvsField] + hgvs_splice: Optional[HgvsField] + assay_level_hgvs: Optional[HgvsField] + protein_level_hgvs: Optional[HgvsField] + + +def _hgvs_field_for_str(hgvs: Optional[str]) -> Optional[HgvsField]: + """Wrap an HGVS string as an :class:`HgvsField`, attaching a parsed block when it is a placeable + simple substitution. ``None`` for an absent string.""" + if not hgvs: + return None + block = parse_simple_substitution(hgvs) + if block is None: + return HgvsField(hgvs=hgvs) + return HgvsField(hgvs=hgvs, position=block.position, ref=block.ref, alt=block.alt) + + +def get_lean_score_set_variants( + db: Session, score_set: ScoreSet, *, as_of: Optional[datetime] = None +) -> list[LeanVariantRecord]: + """Assemble the lean whole-set view for ``score_set`` — one record per variant, ordered by variant + number. Unmapped variants are retained (left joins) with null mapped fields so the table and score + histogram still see them. ``as_of`` reconstructs the annotation layer at a past instant (submitted + HGVS and scores are immutable and unaffected); it defaults to the currently-live rows. + """ + # Content valid-time: each ValidTime layer is evaluated at the same instant, or at the current tail. + record_live = MappingRecord.as_of(as_of) if as_of is not None else MappingRecord.current + link_live = MappingRecordAllele.as_of(as_of) if as_of is not None else MappingRecordAllele.current + vep_live = VepAlleleConsequence.as_of(as_of) if as_of is not None else VepAlleleConsequence.current + + # The post-mapped protein HGVS is the forward-translated protein consequence, stored in the hgvs_p + # column of the record's protein-level allele. Pull it as a correlated scalar subquery so the main + # query stays one row per variant. It is null when no forward-translated protein is available. For + # a protein assay this is the measured allele, so it coincides with assay_level_hgvs. + prot_link = aliased(MappingRecordAllele) + prot_allele = aliased(Allele) + prot_link_live = ( + and_(prot_link.valid_from <= as_of, or_(prot_link.valid_to.is_(None), prot_link.valid_to > as_of)) + if as_of is not None + else prot_link.valid_to.is_(None) + ) + protein_hgvs_subquery = ( + select(prot_allele.hgvs_p) + .join(prot_link, prot_link.allele_id == prot_allele.id) + .where(prot_link.mapping_record_id == MappingRecord.id) + .where(prot_link_live) + .where(prot_allele.level == AnnotationLayer.protein.value) + .order_by(prot_allele.id) + .limit(1) + .correlate(MappingRecord) + .scalar_subquery() + ) + + statement = ( + select( + Variant.urn, + Variant.data, + Variant.hgvs_nt, + Variant.hgvs_pro, + Variant.hgvs_splice, + MappingRecord.hgvs_assay_level, + protein_hgvs_subquery.label("protein_hgvs"), + Allele.vrs_digest, + Allele.clingen_allele_id, + VepAlleleConsequence.functional_consequence, + ) + .outerjoin(MappingRecord, and_(MappingRecord.variant_id == Variant.id, record_live)) + # Only the authoritative (measured) allele is needed, so the link join stays 1:1 with the + # variant — one authoritative link per live record (the invariant the mapping job upholds). + .outerjoin( + MappingRecordAllele, + and_( + MappingRecordAllele.mapping_record_id == MappingRecord.id, + MappingRecordAllele.is_authoritative.is_(True), + link_live, + ), + ) + .outerjoin(Allele, Allele.id == MappingRecordAllele.allele_id) + .outerjoin(VepAlleleConsequence, and_(VepAlleleConsequence.allele_id == Allele.id, vep_live)) + .where(Variant.score_set_id == score_set.id) + # Variant number (the integer after '#') is the natural table order; id breaks ties stably. + .order_by(cast(func.split_part(Variant.urn, "#", 2), Integer), Variant.id) + ) + + return [ + LeanVariantRecord( + variant_urn=row.urn, + score=score_from_variant_data(row.data), + consequence=row.functional_consequence, + clingen_allele_id=row.clingen_allele_id, + assay_level_digest=row.vrs_digest, + hgvs_nt=_hgvs_field_for_str(row.hgvs_nt), + hgvs_pro=_hgvs_field_for_str(row.hgvs_pro), + hgvs_splice=_hgvs_field_for_str(row.hgvs_splice), + assay_level_hgvs=_hgvs_field_for_str(row.hgvs_assay_level), + protein_level_hgvs=_hgvs_field_for_str(row.protein_hgvs), + ) + for row in db.execute(statement) + ] diff --git a/src/mavedb/routers/score_sets.py b/src/mavedb/routers/score_sets.py index 04f798a70..300191285 100644 --- a/src/mavedb/routers/score_sets.py +++ b/src/mavedb/routers/score_sets.py @@ -9,7 +9,7 @@ import pandas as pd import requests from arq import ArqRedis -from fastapi import APIRouter, Depends, File, Query, Request, UploadFile +from fastapi import APIRouter, Depends, File, Query, Request, Response, UploadFile from fastapi.encoders import jsonable_encoder from fastapi.exceptions import HTTPException, RequestValidationError from fastapi.responses import StreamingResponse @@ -41,6 +41,7 @@ find_or_create_doi_identifier, find_or_create_publication_identifier, ) +from mavedb.lib.lean_variants import get_lean_score_set_variants from mavedb.lib.logging import LoggedRoute from mavedb.lib.logging.context import ( correlation_id_for_context, @@ -98,6 +99,7 @@ from mavedb.view_models import clinical_control, gnomad_variant, mapped_variant, score_set from mavedb.view_models.contributor import ContributorCreate from mavedb.view_models.doi_identifier import DoiIdentifierCreate +from mavedb.view_models.lean_variant import LeanVariant from mavedb.view_models.publication_identifier import PublicationIdentifierCreate from mavedb.view_models.score_set_dataset_columns import DatasetColumnMetadata from mavedb.view_models.search import ScoreSetsSearch, ScoreSetsSearchFilterOptionsResponse, ScoreSetsSearchResponse @@ -865,6 +867,46 @@ async def show_score_set( return score_set.ScoreSet.model_validate(item).copy(update={"experiment": enriched_experiment}) +@router.get( + "/score-sets/{urn}/variants", + status_code=200, + response_model=List[LeanVariant], + response_model_exclude_none=True, + responses={**ACCESS_CONTROL_ERROR_RESPONSES}, + summary="Get the lean whole-set variant view for a score set", +) +async def get_score_set_lean_variants( + *, + urn: str, + response: Response, + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the annotation layer (mapping, allele links, VEP consequence) as it stood at " + "this instant, over the score set's fixed scores. ISO 8601, ideally timezone-aware. This is " + "content valid-time only — it never re-selects a score-set version. Defaults to current." + ), + ), + db: Session = Depends(deps.get_db), + user_data: Optional[UserData] = Depends(get_current_user), +) -> Any: + """ + Return the lean whole-set view for a score set: one pre-chewed record per variant carrying the + selection key (variant URN), score, a representative consequence, the bridge identifiers into the + annotation dimensions (ClinGen allele id, assay-level digest), and the DNA + protein parsed + position/ref/alt blocks that drive the heatmap's level toggle. + + The full set is returned in one payload — the score-set page bins/sorts/filters across every + variant client-side. as_of time-travels the annotation layer only (scores are immutable); the + resolved value is echoed in the X-As-Of response header so the content-time is a visible fact. + """ + save_to_logging_context({"requested_resource": urn, "resource_property": "lean-variants", "as_of": as_of}) + response.headers["X-As-Of"] = as_of.isoformat() if as_of is not None else "current" + + score_set = await fetch_score_set_by_urn(db, urn, user_data, None, False) + return get_lean_score_set_variants(db, score_set, as_of=as_of) + + @router.get( "/score-sets/{urn}/variants/data", status_code=200, diff --git a/src/mavedb/view_models/lean_variant.py b/src/mavedb/view_models/lean_variant.py new file mode 100644 index 000000000..1820cfc71 --- /dev/null +++ b/src/mavedb/view_models/lean_variant.py @@ -0,0 +1,54 @@ +"""Response view models for the lean whole-set view (``GET /score-sets/{urn}/variants``). + +The pydantic serialization boundary over the ``lib.lean_variants`` transit dataclasses. ``from_attributes`` +lets the route return the ``LeanVariantRecord`` dataclasses directly and have FastAPI's ``response_model`` +coerce them (field names line up); aliases camelize per the shared base config, so clients see +``variantUrn`` / ``assayLevelHgvs`` etc. ``response_model_exclude_none`` drops absent fields, so a slot +with no parsed block serializes as just ``{"hgvs": "..."}``. +""" + +from typing import Optional + +from mavedb.view_models.base.base import BaseModel + + +class HgvsField(BaseModel): + """An HGVS expression with its parsed substitution block riding alongside when representable. + + ``hgvs`` is always present; ``position``/``ref``/``alt`` appear only for a placeable simple + substitution (the heatmap grid) and are omitted for splice/indels/multivariants. + """ + + hgvs: str + position: Optional[int] = None + ref: Optional[str] = None + alt: Optional[str] = None + + class Config: + from_attributes = True + + +class LeanVariant(BaseModel): + """One pre-chewed per-variant record feeding the score-set table, heatmap, and histograms. + + ``variantUrn`` is the universal selection key; ``assayLevelDigest`` bridges into the digest-keyed + annotation dimensions. The submitted HGVS (``hgvsNt``/``hgvsPro``/``hgvsSplice``, target frame) and + the mapped ``assayLevelHgvs`` (reference frame) carry both sides of the heatmap's frame toggle, each + with an optional parsed block for the level toggle. ``proteinLevelHgvs`` is the mapped protein + representation (distinct from the *submitted* ``hgvsPro``); for a protein assay it coincides with + ``assayLevelHgvs``. Fields are omitted when null. + """ + + variant_urn: str + score: Optional[float] = None + consequence: Optional[str] = None + clingen_allele_id: Optional[str] = None + assay_level_digest: Optional[str] = None + hgvs_nt: Optional[HgvsField] = None + hgvs_pro: Optional[HgvsField] = None + hgvs_splice: Optional[HgvsField] = None + assay_level_hgvs: Optional[HgvsField] = None + protein_level_hgvs: Optional[HgvsField] = None + + class Config: + from_attributes = True diff --git a/tests/lib/test_hgvs.py b/tests/lib/test_hgvs.py index e1e49978c..5ecb40e9d 100644 --- a/tests/lib/test_hgvs.py +++ b/tests/lib/test_hgvs.py @@ -1,13 +1,86 @@ import pytest from mavedb.lib.hgvs import ( + SequenceBlock, extract_accession, join_cis_phased_hgvs, + parse_simple_nucleotide_substitution, + parse_simple_protein_substitution, + parse_simple_substitution, split_cis_phased_hgvs, strip_protein_prediction_parens, ) +@pytest.mark.unit +@pytest.mark.parametrize( + ("hgvs", "expected"), + [ + # auto-detects the level from the disjoint coordinate prefix + ("NM_000546.6:c.1216G>A", SequenceBlock(position=1216, ref="G", alt="A")), + ("NP_000537.3:p.Ala406Thr", SequenceBlock(position=406, ref="Ala", alt="Thr")), + ("g.1000A>G", SequenceBlock(position=1000, ref="A", alt="G")), + # non-placeable in either grammar + ("c.122-6T>A", None), + ("c.[197A>G;472T>C]", None), + (None, None), + ], +) +def test_parse_simple_substitution(hgvs, expected): + assert parse_simple_substitution(hgvs) == expected + + +@pytest.mark.unit +@pytest.mark.parametrize( + ("hgvs", "expected"), + [ + # accession-qualified and bare coding substitutions + ("NM_000546.6:c.1216G>A", SequenceBlock(position=1216, ref="G", alt="A")), + ("c.5A>G", SequenceBlock(position=5, ref="A", alt="G")), + # genomic and non-coding levels parse the same way + ("NC_000001.11:g.1000A>G", SequenceBlock(position=1000, ref="A", alt="G")), + ("n.42C>T", SequenceBlock(position=42, ref="C", alt="T")), + # lowercase nucleotides are tolerated + ("c.5a>g", SequenceBlock(position=5, ref="a", alt="g")), + # non-placeable: UTR/intron positions, multivariant, indels, non-substitutions, empty + ("c.*123A>G", None), + ("c.-12A>G", None), + ("c.12+3A>G", None), + ("NM_000546.6:c.[197A>G;472T>C]", None), + ("c.76_78del", None), + ("p.Ala406Thr", None), + ("", None), + (None, None), + ], +) +def test_parse_simple_nucleotide_substitution(hgvs, expected): + assert parse_simple_nucleotide_substitution(hgvs) == expected + + +@pytest.mark.unit +@pytest.mark.parametrize( + ("hgvs", "expected"), + [ + # accession-qualified, bare, and prediction-wrapped substitutions + ("NP_000537.3:p.Ala406Thr", SequenceBlock(position=406, ref="Ala", alt="Thr")), + ("p.Ala406Thr", SequenceBlock(position=406, ref="Ala", alt="Thr")), + ("NP_000537.3:p.(Ala406Thr)", SequenceBlock(position=406, ref="Ala", alt="Thr")), + # synonymous, stop, and deletion tokens are preserved as the raw alt + ("p.Ala406=", SequenceBlock(position=406, ref="Ala", alt="=")), + ("p.Tyr745*", SequenceBlock(position=745, ref="Tyr", alt="*")), + ("p.Ala406-", SequenceBlock(position=406, ref="Ala", alt="-")), + # non-placeable: multivariant, frameshift, single-letter codes, empty + ("p.[Ala406Thr;Gly12Cys]", None), + ("p.Arg97fs", None), + ("p.A406T", None), + ("", None), + (None, None), + ], +) +def test_parse_simple_protein_substitution(hgvs, expected): + assert parse_simple_protein_substitution(hgvs) == expected + + @pytest.mark.parametrize( ("hgvs", "expected"), [ diff --git a/tests/lib/test_lean_variants.py b/tests/lib/test_lean_variants.py new file mode 100644 index 000000000..79d630815 --- /dev/null +++ b/tests/lib/test_lean_variants.py @@ -0,0 +1,240 @@ +# ruff: noqa: E402 +"""Integration tests for the lean whole-set view assembly (``lib/lean_variants.py``). + +These pin the one-row-per-variant assembly: the submitted HGVS come off the variant, the mapped +assay-level HGVS off the live mapping record, the digest/ClinGen id/VEP consequence off the +authoritative allele; each HGVS string always rides even when it is not a placeable simple +substitution; unmapped variants are retained with null mapped fields; ``as_of`` reconstructs the +annotation layer while the immutable submitted HGVS and score stay put; and the set is ordered by +variant number. +""" + +from datetime import datetime, timedelta, timezone + +import pytest + +pytest.importorskip("psycopg2") + +from mavedb.lib.lean_variants import HgvsField, get_lean_score_set_variants +from mavedb.models.allele import Allele +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.variant import Variant +from mavedb.models.vep_allele_consequence import VepAlleleConsequence +from tests.helpers.constants import TEST_MINIMAL_VARIANT + +# Deterministic windows far from the transaction clock (mirrors tests/lib/test_alleles.py). +T0 = datetime(2020, 1, 1, tzinfo=timezone.utc) +T1 = datetime(2021, 1, 1, tzinfo=timezone.utc) +T2 = datetime(2022, 1, 1, tzinfo=timezone.utc) + + +def _variant(session, score_set, suffix, *, score=None, hgvs_nt=None, hgvs_pro=None, hgvs_splice=None): + data = {"score_data": {"score": score}} if score is not None else TEST_MINIMAL_VARIANT["data"] + variant = Variant( + urn=f"{score_set.urn}#{suffix}", + score_set_id=score_set.id, + data=data, + hgvs_nt=hgvs_nt, + hgvs_pro=hgvs_pro, + hgvs_splice=hgvs_splice, + creation_date=TEST_MINIMAL_VARIANT["creation_date"], + modification_date=TEST_MINIMAL_VARIANT["modification_date"], + ) + session.add(variant) + session.commit() + return variant + + +def _allele(session, digest, *, level="cdna", clingen_allele_id=None, hgvs_p=None): + allele = Allele( + vrs_digest=digest, + level=level, + post_mapped={"type": "Allele"}, + clingen_allele_id=clingen_allele_id, + hgvs_p=hgvs_p, + ) + session.add(allele) + session.commit() + return allele + + +def _record(session, variant, *, assay_level="cdna", hgvs_assay_level=None, valid_from=None): + record = MappingRecord( + variant_id=variant.id, + assay_level=assay_level, + hgvs_assay_level=hgvs_assay_level, + mapping_api_version="test.0.0", + valid_from=valid_from, + ) + session.add(record) + session.commit() + return record + + +def _link(session, record, allele, *, is_authoritative=True, valid_from=None): + link = MappingRecordAllele( + mapping_record_id=record.id, allele_id=allele.id, is_authoritative=is_authoritative, valid_from=valid_from + ) + session.add(link) + session.commit() + return link + + +def _consequence(session, allele, value): + row = VepAlleleConsequence( + allele_id=allele.id, functional_consequence=value, source_version="116", access_date="2026-01-01" + ) + session.add(row) + session.commit() + return row + + +@pytest.mark.integration +def test_nucleotide_assay(session, setup_lib_db_with_score_set): + """Coding assay: submitted nt+pro from the variant, mapped coding string from the record, and the + digest/ClinGen id/consequence off the authoritative allele. Each parseable string gets its block.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, score=-2.3, hgvs_nt="c.1A>T", hgvs_pro="p.Thr1Ser") + record = _record(session, variant, assay_level="cdna", hgvs_assay_level="NM_000546.6:c.1216G>A") + measured = _allele(session, "cdna-digest", level="cdna", clingen_allele_id="CA123") + protein = _allele(session, "prot-digest", level="protein", hgvs_p="NP_000537.3:p.Ala406Thr") + _link(session, record, measured, is_authoritative=True) + _link(session, record, protein, is_authoritative=False) + _consequence(session, measured, "missense_variant") + + [record_out] = get_lean_score_set_variants(session, score_set) + + assert record_out.variant_urn == variant.urn + assert record_out.score == -2.3 + assert record_out.consequence == "missense_variant" + assert record_out.clingen_allele_id == "CA123" + assert record_out.assay_level_digest == "cdna-digest" # the measured (authoritative) allele + assert record_out.hgvs_nt == HgvsField(hgvs="c.1A>T", position=1, ref="A", alt="T") + assert record_out.hgvs_pro == HgvsField(hgvs="p.Thr1Ser", position=1, ref="Thr", alt="Ser") + assert record_out.hgvs_splice is None + assert record_out.assay_level_hgvs == HgvsField(hgvs="NM_000546.6:c.1216G>A", position=1216, ref="G", alt="A") + # The mapped protein comes off the protein-level member allele's hgvs_p, not the measured coding allele. + assert record_out.protein_level_hgvs == HgvsField( + hgvs="NP_000537.3:p.Ala406Thr", position=406, ref="Ala", alt="Thr" + ) + + +@pytest.mark.integration +def test_protein_assay_maps_assay_level_to_a_protein_block(session, setup_lib_db_with_score_set): + """Protein assay: the mapped assay-level string is a p. expression, parsed into a protein block; the + digest comes off the authoritative protein allele.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, score=1.0, hgvs_pro="p.Ala406Thr") + record = _record(session, variant, assay_level="protein", hgvs_assay_level="NP_000537.3:p.Ala406Thr") + measured = _allele( + session, "prot-digest", level="protein", clingen_allele_id="PA9", hgvs_p="NP_000537.3:p.Ala406Thr" + ) + _link(session, record, measured, is_authoritative=True) + + [record_out] = get_lean_score_set_variants(session, score_set) + + assert record_out.assay_level_digest == "prot-digest" + assert record_out.clingen_allele_id == "PA9" + block = HgvsField(hgvs="NP_000537.3:p.Ala406Thr", position=406, ref="Ala", alt="Thr") + # For a protein assay the measured allele *is* the protein one, so the two mapped fields coincide. + assert record_out.assay_level_hgvs == block + assert record_out.protein_level_hgvs == block + + +@pytest.mark.integration +def test_hgvs_string_rides_even_when_unparseable(session, setup_lib_db_with_score_set): + """The canonical string is always present; a splice/multivariant expression simply carries no block.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, hgvs_nt="NM_000546.6:c.[197A>G;472T>C]", hgvs_splice="c.122-6T>A") + record = _record(session, variant, assay_level="cdna", hgvs_assay_level="c.76_78del") + measured = _allele(session, "cdna-digest", level="cdna") + _link(session, record, measured, is_authoritative=True) + + [record_out] = get_lean_score_set_variants(session, score_set) + + # Multivariant, intronic, and indel expressions are all string-only (no position/ref/alt block). + assert record_out.hgvs_nt == HgvsField(hgvs="NM_000546.6:c.[197A>G;472T>C]") + assert record_out.hgvs_nt.position is None + assert record_out.hgvs_splice == HgvsField(hgvs="c.122-6T>A") + assert record_out.assay_level_hgvs == HgvsField(hgvs="c.76_78del") + + +@pytest.mark.integration +def test_unmapped_variant_has_null_mapped_fields(session, setup_lib_db_with_score_set): + """A variant with no live mapping record keeps its submitted HGVS + score but null mapped fields.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, score=0.5, hgvs_nt="c.1A>T", hgvs_pro="p.Thr1Ser") + + [record_out] = get_lean_score_set_variants(session, score_set) + + assert record_out.variant_urn == variant.urn + assert record_out.score == 0.5 + assert record_out.hgvs_nt == HgvsField(hgvs="c.1A>T", position=1, ref="A", alt="T") + assert record_out.consequence is None + assert record_out.clingen_allele_id is None + assert record_out.assay_level_digest is None + assert record_out.assay_level_hgvs is None + assert record_out.protein_level_hgvs is None + + +@pytest.mark.integration +def test_no_vep_consequence_is_none(session, setup_lib_db_with_score_set): + """A mapped variant with no live VEP consequence has consequence None but keeps its digest + mapped HGVS.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, hgvs_nt="c.1A>T") + record = _record(session, variant, assay_level="cdna", hgvs_assay_level="NM_000546.6:c.1216G>A") + measured = _allele(session, "cdna-digest", level="cdna") + _link(session, record, measured, is_authoritative=True) + + [record_out] = get_lean_score_set_variants(session, score_set) + + assert record_out.consequence is None + assert record_out.assay_level_digest == "cdna-digest" + assert record_out.assay_level_hgvs == HgvsField(hgvs="NM_000546.6:c.1216G>A", position=1216, ref="G", alt="A") + + +@pytest.mark.integration +def test_ordered_by_variant_number(session, setup_lib_db_with_score_set): + """Records come back ordered by the integer after '#', not insertion or lexical order.""" + score_set = setup_lib_db_with_score_set + for suffix in (10, 2, 1): + _variant(session, score_set, suffix, score=float(suffix)) + + records = get_lean_score_set_variants(session, score_set) + + assert [r.variant_urn for r in records] == [f"{score_set.urn}#{n}" for n in (1, 2, 10)] + + +@pytest.mark.integration +def test_as_of_reconstructs_the_historical_annotation_layer(session, setup_lib_db_with_score_set): + """A re-map retires the old record and inserts a new one; as_of reconstructs the mapped layer live at + a past instant, while the immutable submitted HGVS and score are unaffected.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, score=-2.3, hgvs_nt="c.1A>T") + + old_record = _record(session, variant, assay_level="cdna", hgvs_assay_level="NM_000546.6:c.1216G>A", valid_from=T0) + old_allele = _allele(session, "old-digest", level="cdna") + _link(session, old_record, old_allele, is_authoritative=True, valid_from=T0) + old_record.retire(session, at=T1) # cascades to the link via __retire_cascade__ + + new_record = _record(session, variant, assay_level="cdna", hgvs_assay_level="NM_000546.6:c.500A>T", valid_from=T1) + new_allele = _allele(session, "new-digest", level="cdna") + _link(session, new_record, new_allele, is_authoritative=True, valid_from=T1) + session.commit() + + [current] = get_lean_score_set_variants(session, score_set) + assert current.assay_level_digest == "new-digest" + assert current.assay_level_hgvs == HgvsField(hgvs="NM_000546.6:c.500A>T", position=500, ref="A", alt="T") + + [historical] = get_lean_score_set_variants(session, score_set, as_of=T1 - timedelta(days=1)) + assert historical.assay_level_digest == "old-digest" + assert historical.assay_level_hgvs == HgvsField(hgvs="NM_000546.6:c.1216G>A", position=1216, ref="G", alt="A") + # Submitted HGVS and score are immutable — unaffected by as_of. + assert historical.hgvs_nt == HgvsField(hgvs="c.1A>T", position=1, ref="A", alt="T") + assert historical.score == -2.3 + + # Before anything was mapped, the variant still appears with null mapped fields. + [pre_mapping] = get_lean_score_set_variants(session, score_set, as_of=T0 - timedelta(days=1)) + assert pre_mapping.assay_level_digest is None + assert pre_mapping.assay_level_hgvs is None diff --git a/tests/routers/test_score_set.py b/tests/routers/test_score_set.py index 4a896c3cb..055d6bbb2 100644 --- a/tests/routers/test_score_set.py +++ b/tests/routers/test_score_set.py @@ -24,9 +24,14 @@ from mavedb.models.experiment import Experiment as ExperimentDbModel from mavedb.models.job_run import JobRun from mavedb.models.pipeline import Pipeline +from mavedb.models.allele import Allele from mavedb.models.mapped_variant import MappedVariant as MappedVariantDbModel +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.score_set import ScoreSet as ScoreSetDbModel from mavedb.models.variant import Variant as VariantDbModel +from mavedb.models.vep_allele_consequence import VepAlleleConsequence +from mavedb.view_models.lean_variant import LeanVariant from mavedb.view_models.orcid import OrcidUser from mavedb.view_models.score_set import ScoreSet, ScoreSetCreate from tests.helpers.constants import ( @@ -4559,3 +4564,164 @@ def test_cannot_fetch_gnomad_variants_for_score_set_when_none_exist( f"No gnomad variants matching the provided filters associated with score set URN {score_set['urn']} were found" in response_data["detail"] ) + + +def _seed_lean_mapping(session, variant_urn): + """Give one variant a live coding-measured mapping record (with an assay-level HGVS) whose + authoritative allele carries a digest + ClinGen id + a live VEP consequence.""" + variant = session.scalar(select(VariantDbModel).where(VariantDbModel.urn == variant_urn)) + record = MappingRecord( + variant_id=variant.id, + assay_level="cdna", + hgvs_assay_level="NM_000546.6:c.1216G>A", + mapping_api_version="test.0.0", + ) + session.add(record) + session.commit() + + measured = Allele(vrs_digest="cdna-digest", level="cdna", post_mapped={"type": "Allele"}, clingen_allele_id="CA123") + protein = Allele( + vrs_digest="prot-digest", level="protein", post_mapped={"type": "Allele"}, hgvs_p="NP_000537.3:p.Ala406Thr" + ) + session.add_all([measured, protein]) + session.commit() + + session.add_all( + [ + MappingRecordAllele(mapping_record_id=record.id, allele_id=measured.id, is_authoritative=True), + MappingRecordAllele(mapping_record_id=record.id, allele_id=protein.id, is_authoritative=False), + VepAlleleConsequence( + allele_id=measured.id, + functional_consequence="missense_variant", + source_version="116", + access_date="2026-01-01", + ), + ] + ) + session.commit() + + +def test_get_lean_variants(client, session, data_provider, data_files, setup_router_db): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + _seed_lean_mapping(session, f"{score_set['urn']}#1") + + response = client.get(f"/api/v1/score-sets/{score_set['urn']}/variants") + + assert response.status_code == 200 + records = response.json() + # All three variants are returned, ordered by variant number. + assert [r["variantUrn"] for r in records] == [f"{score_set['urn']}#{n}" for n in (1, 2, 3)] + for record in records: + LeanVariant.model_validate_json(json.dumps(record)) + + mapped = records[0] + assert mapped["score"] == 0.3 + assert mapped["consequence"] == "missense_variant" + assert mapped["clingenAlleleId"] == "CA123" + assert mapped["assayLevelDigest"] == "cdna-digest" + # Submitted HGVS (from the uploaded CSV), each with its parsed block riding alongside. + assert mapped["hgvsNt"] == {"hgvs": "c.1A>T", "position": 1, "ref": "A", "alt": "T"} + assert mapped["hgvsPro"] == {"hgvs": "p.Thr1Ser", "position": 1, "ref": "Thr", "alt": "Ser"} + # Mapped assay-level HGVS (reference frame) and the mapped protein representation. + assert mapped["assayLevelHgvs"] == {"hgvs": "NM_000546.6:c.1216G>A", "position": 1216, "ref": "G", "alt": "A"} + assert mapped["proteinLevelHgvs"] == { + "hgvs": "NP_000537.3:p.Ala406Thr", + "position": 406, + "ref": "Ala", + "alt": "Thr", + } + + # An unmapped variant keeps its submitted HGVS + score; the mapped fields are dropped (exclude_none). + unmapped = records[1] + assert unmapped["score"] == 1.0 + assert unmapped["hgvsNt"]["hgvs"] == "c.2C>T" + for omitted in ("consequence", "clingenAlleleId", "assayLevelDigest", "assayLevelHgvs"): + assert omitted not in unmapped + + +def test_get_lean_variants_unknown_score_set_is_404(client, setup_router_db): + response = client.get("/api/v1/score-sets/urn:mavedb:00000000-a-1/variants") + assert response.status_code == 404 + + +def test_get_lean_variants_echoes_as_of_header(client, session, data_provider, data_files, setup_router_db): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + + # Default is current — the resolved content-time is echoed for the client. + current = client.get(f"/api/v1/score-sets/{score_set['urn']}/variants") + assert current.status_code == 200 + assert current.headers["X-As-Of"] == "current" + + # An explicit as_of is accepted and echoed back; before mapping exists everything is unmapped. + historical = client.get(f"/api/v1/score-sets/{score_set['urn']}/variants", params={"as_of": "2020-01-01T00:00:00Z"}) + assert historical.status_code == 200 + assert historical.headers["X-As-Of"] == "2020-01-01T00:00:00+00:00" + assert all("assayLevelDigest" not in record for record in historical.json()) + + +def test_get_lean_variants_other_user_cannot_read_private(client, session, data_provider, data_files, setup_router_db): + """The endpoint gates on READ: a non-owner cannot read another user's private (unpublished) set.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + change_ownership(session, score_set["urn"], ScoreSetDbModel) + + response = client.get(f"/api/v1/score-sets/{score_set['urn']}/variants") + + assert response.status_code == 404 + assert f"score set with URN '{score_set['urn']}' not found" in response.json()["detail"] + + +def test_get_lean_variants_anonymous_cannot_read_private( + client, session, data_provider, data_files, setup_router_db, anonymous_app_overrides +): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + change_ownership(session, score_set["urn"], ScoreSetDbModel) + + with DependencyOverrider(anonymous_app_overrides): + response = client.get(f"/api/v1/score-sets/{score_set['urn']}/variants") + + assert response.status_code == 404 + + +def test_get_lean_variants_anonymous_can_read_published( + client, session, data_provider, data_files, setup_router_db, anonymous_app_overrides +): + """A published score set is world-readable, so an anonymous caller gets the full lean set.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + published = publish_score_set(client, score_set["urn"]) + + with DependencyOverrider(anonymous_app_overrides): + response = client.get(f"/api/v1/score-sets/{published['urn']}/variants") + + assert response.status_code == 200 + assert len(response.json()) == 3 + + +def test_get_lean_variants_admin_can_read_private( + client, session, data_provider, data_files, setup_router_db, admin_app_overrides +): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + change_ownership(session, score_set["urn"], ScoreSetDbModel) + + with DependencyOverrider(admin_app_overrides): + response = client.get(f"/api/v1/score-sets/{score_set['urn']}/variants") + + assert response.status_code == 200 + assert len(response.json()) == 3 From f563760c0330d97d747227af596cc572eeedb7c3 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 1 Jul 2026 11:07:47 -0700 Subject: [PATCH 058/252] refactor(valid-time): add ValidTime.live_at for uniform temporal filtering MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a live_at(as_of) hybrid_method to the ValidTime mixin — the one-call form of `as_of(ts) if ts is not None else current`. It is alias-aware (like as_of/current, its column refs adapt to an aliased() class), so a per-layer subquery can filter an aliased ValidTime table directly rather than hand-rolling a valid_to predicate. Migrate the repeated ternary in alleles.py to it. Covered by TestLiveAt: instance both branches, expression agrees with current/as_of, and alias-awareness. --- src/mavedb/db/mixins.py | 23 +++++++++++++++++++-- src/mavedb/lib/alleles.py | 17 ++++++++-------- tests/db/test_mixins.py | 43 ++++++++++++++++++++++++++++++++++++++- 3 files changed, 71 insertions(+), 12 deletions(-) diff --git a/src/mavedb/db/mixins.py b/src/mavedb/db/mixins.py index 4bc5c1ca9..c9b17f4b3 100644 --- a/src/mavedb/db/mixins.py +++ b/src/mavedb/db/mixins.py @@ -3,8 +3,9 @@ from datetime import datetime from typing import ClassVar, Optional, Sequence, TypeVar -from sqlalchemy import ColumnElement, DateTime, Update, func, inspect as sa_inspect, select, update -from sqlalchemy.ext.hybrid import hybrid_property +from sqlalchemy import ColumnElement, DateTime, Update, func, select, update +from sqlalchemy import inspect as sa_inspect +from sqlalchemy.ext.hybrid import hybrid_method, hybrid_property from sqlalchemy.orm import Mapped, Session, mapped_column T = TypeVar("T", bound="ValidTime") @@ -97,6 +98,24 @@ def as_of(cls, ts: datetime) -> ColumnElement[bool]: """Filter clause selecting the rows live at ``ts`` (half-open ``[valid_from, valid_to)``).""" return (cls.valid_from <= ts) & (cls.valid_to.is_(None) | (cls.valid_to > ts)) + @hybrid_method + def live_at(self, as_of: Optional[datetime]) -> bool: + """Live at ``as_of``, or currently live when ``as_of`` is ``None``. + + The one-call form of the ``as_of(ts) if ts is not None else current`` branch every temporal + read otherwise repeats. Same result as :attr:`current` / :meth:`as_of`, but usable uniformly on + the class, an instance, and an ``aliased()`` class (SQLAlchemy adapts the column refs to it).""" + if as_of is None: + return self.valid_to is None + return self.valid_from <= as_of and (self.valid_to is None or self.valid_to > as_of) + + @live_at.expression + @classmethod + def _live_at_expression(cls, as_of: Optional[datetime]) -> ColumnElement[bool]: + if as_of is None: + return cls.valid_to.is_(None) + return (cls.valid_from <= as_of) & (cls.valid_to.is_(None) | (cls.valid_to > as_of)) + def retire(self, session: Optional[Session] = None, at: Optional[datetime] = None) -> None: """Close this row's validity window, cascading to the live child links named in ``__retire_cascade__``. Idempotent — an already-closed row keeps its original valid_to. diff --git a/src/mavedb/lib/alleles.py b/src/mavedb/lib/alleles.py index 7a7213fd1..bf6a340d4 100644 --- a/src/mavedb/lib/alleles.py +++ b/src/mavedb/lib/alleles.py @@ -35,10 +35,9 @@ def get_live_record_allele_links( applies the same half-open predicate to both the record and the links, so the set is evaluated at one instant. Returns ``[]`` when the variant has no live record. """ - record_live = MappingRecord.as_of(as_of) if as_of is not None else MappingRecord.current - link_live = MappingRecordAllele.as_of(as_of) if as_of is not None else MappingRecordAllele.current - - record_id = db.scalar(select(MappingRecord.id).where(MappingRecord.variant_id == variant_id).where(record_live)) + record_id = db.scalar( + select(MappingRecord.id).where(MappingRecord.variant_id == variant_id).where(MappingRecord.live_at(as_of)) + ) if record_id is None: return [] @@ -46,7 +45,7 @@ def get_live_record_allele_links( db.scalars( select(MappingRecordAllele) .where(MappingRecordAllele.mapping_record_id == record_id) - .where(link_live) + .where(MappingRecordAllele.live_at(as_of)) .options(joinedload(MappingRecordAllele.allele)) ).all() ) @@ -69,10 +68,10 @@ def get_allele_translations(db: Session, allele_id: int, *, as_of: Optional[date instant. The retire-cascade invariant (a live link implies a live record) holds under ``as_of`` too, so filtering the links alone is sufficient. """ - link_live = MappingRecordAllele.as_of(as_of) if as_of is not None else MappingRecordAllele.current - record_ids = db.scalars( - select(MappingRecordAllele.mapping_record_id).where(MappingRecordAllele.allele_id == allele_id).where(link_live) + select(MappingRecordAllele.mapping_record_id) + .where(MappingRecordAllele.allele_id == allele_id) + .where(MappingRecordAllele.live_at(as_of)) ).all() if not record_ids: return [] @@ -82,7 +81,7 @@ def get_allele_translations(db: Session, allele_id: int, *, as_of: Optional[date select(Allele) .join(MappingRecordAllele, MappingRecordAllele.allele_id == Allele.id) .where(MappingRecordAllele.mapping_record_id.in_(record_ids)) - .where(link_live) + .where(MappingRecordAllele.live_at(as_of)) .distinct() ).all() ) diff --git a/tests/db/test_mixins.py b/tests/db/test_mixins.py index ad0333a5d..938d2062e 100644 --- a/tests/db/test_mixins.py +++ b/tests/db/test_mixins.py @@ -15,7 +15,7 @@ import pytest from sqlalchemy import Column, ForeignKey, Index, Integer, select, text from sqlalchemy.exc import IntegrityError -from sqlalchemy.orm import declarative_base, relationship +from sqlalchemy.orm import aliased, declarative_base, relationship from mavedb.db.mixins import ValidTime @@ -120,6 +120,47 @@ def live_ids(ts): assert live_ids(T2 + timedelta(days=1)) == [] # after the window closes +class TestLiveAt: + """``live_at(as_of)`` is the one-call ``as_of(ts) if ts else current`` — same result, but it also + works on an ``aliased()`` entity, which the raw ternary at call sites does too, but only clumsily.""" + + def test_live_at_none_is_current_on_instance(self, session): + live = _parent(session, 1) + retired = _parent(session, 2) + retired.retire(session, at=T1) + assert live.live_at(None) is True + assert retired.live_at(None) is False + + def test_live_at_ts_matches_the_window_on_instance(self, session): + p = _parent(session, 1, valid_from=T0) + p.retire(session, at=T2) + assert p.live_at(T0 - timedelta(days=1)) is False # before the window opens + assert p.live_at(T0) is True # at valid_from (inclusive) + assert p.live_at(T1) is True # inside the window + assert p.live_at(T2) is False # at valid_to (exclusive) + + def test_live_at_expression_agrees_with_current_and_as_of(self, session): + p = _parent(session, 1, valid_from=T0) + p.retire(session, at=T2) + session.commit() + + def ids(pred): + return [r.id for r in session.scalars(select(_VtParent).where(pred)).all()] + + assert ids(_VtParent.live_at(None)) == ids(_VtParent.current) == [] # retired -> not current + assert ids(_VtParent.live_at(T1)) == ids(_VtParent.as_of(T1)) == [p.id] + + def test_live_at_is_alias_aware(self, session): + # The point of live_at: it scopes to the alias, not the base table (a plain classmethod on an + # aliased entity is the trap this avoids). Both the compiled SQL and a query through the alias. + p = _parent(session, 1, valid_from=T0) + session.commit() + alias = aliased(_VtParent, name="pa") + assert "pa." in str(alias.live_at(T1)) + rows = session.scalars(select(alias).where(alias.live_at(T1))).all() + assert [r.id for r in rows] == [p.id] + + class TestRetire: def test_retire_closes_valid_to(self, session): c = _child(session, _parent(session, 1), 1) From a5f3da700def00da47586ab1bb0df6159d19090b Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 1 Jul 2026 11:09:09 -0700 Subject: [PATCH 059/252] refactor(lean_variants): use live_at helper and join-based protein hgvs Rework get_lean_score_set_variants to simplify ValidTime handling and the post-mapped protein HGVS projection. - Replace the inline record/link/vep liveness predicates with a shared live_at(as_of) helper on each ValidTime model, removing the ad-hoc and_/or_ valid_from/valid_to comparisons (and the now-unused or_ import) - Swap the correlated scalar subquery for protein_hgvs for a single DISTINCT ON subquery scoped to the score set and joined once, so its cost scales with the response rather than per-row correlation - Expand comments to document why live_at rides in each ON clause (a WHERE would collapse the LEFT joins to inner joins for unmapped variants) --- src/mavedb/lib/lean_variants.py | 77 ++++++++++++++++++++------------- 1 file changed, 46 insertions(+), 31 deletions(-) diff --git a/src/mavedb/lib/lean_variants.py b/src/mavedb/lib/lean_variants.py index f0949e614..4ac9df53f 100644 --- a/src/mavedb/lib/lean_variants.py +++ b/src/mavedb/lib/lean_variants.py @@ -22,7 +22,7 @@ from datetime import datetime from typing import Optional -from sqlalchemy import Integer, and_, cast, func, or_, select +from sqlalchemy import Integer, and_, cast, func, select from sqlalchemy.orm import Session, aliased from mavedb.lib.hgvs import parse_simple_substitution @@ -92,32 +92,34 @@ def get_lean_score_set_variants( histogram still see them. ``as_of`` reconstructs the annotation layer at a past instant (submitted HGVS and scores are immutable and unaffected); it defaults to the currently-live rows. """ - # Content valid-time: each ValidTime layer is evaluated at the same instant, or at the current tail. - record_live = MappingRecord.as_of(as_of) if as_of is not None else MappingRecord.current - link_live = MappingRecordAllele.as_of(as_of) if as_of is not None else MappingRecordAllele.current - vep_live = VepAlleleConsequence.as_of(as_of) if as_of is not None else VepAlleleConsequence.current - - # The post-mapped protein HGVS is the forward-translated protein consequence, stored in the hgvs_p - # column of the record's protein-level allele. Pull it as a correlated scalar subquery so the main - # query stays one row per variant. It is null when no forward-translated protein is available. For - # a protein assay this is the measured allele, so it coincides with assay_level_hgvs. + # Per record, the forward-translated protein HGVS (hgvs_p) of its protein-level allele. prot_link = aliased(MappingRecordAllele) prot_allele = aliased(Allele) - prot_link_live = ( - and_(prot_link.valid_from <= as_of, or_(prot_link.valid_to.is_(None), prot_link.valid_to > as_of)) - if as_of is not None - else prot_link.valid_to.is_(None) - ) - protein_hgvs_subquery = ( - select(prot_allele.hgvs_p) - .join(prot_link, prot_link.allele_id == prot_allele.id) - .where(prot_link.mapping_record_id == MappingRecord.id) - .where(prot_link_live) + prot_record = aliased(MappingRecord) + prot_variant = aliased(Variant) + protein_allele = ( + select( + prot_link.mapping_record_id.label("mapping_record_id"), + prot_allele.hgvs_p.label("protein_hgvs"), + ) + # DISTINCT ON -> at most one protein level row per record. + .distinct(prot_link.mapping_record_id) + # link -> allele: level and hgvs_p live on the *allele*. + .join(prot_allele, prot_allele.id == prot_link.allele_id) + # link -> record -> variant: the bridge to scoreset_id, to scope the subquery (next). + .join(prot_record, prot_record.id == prot_link.mapping_record_id) + .join(prot_variant, prot_variant.id == prot_record.variant_id) + # Scope to this score set so the subquery's cost scales with the response. + .where(prot_variant.score_set_id == score_set.id) + # Live links only. A live link implies a live parent record (ValidTime invariant), so this also + # covers record-liveness — no prot_record.live_at needed; the MR join is just the scoping bridge. + .where(prot_link.live_at(as_of)) + # Only the protein-level allele. .where(prot_allele.level == AnnotationLayer.protein.value) - .order_by(prot_allele.id) - .limit(1) - .correlate(MappingRecord) - .scalar_subquery() + # DISTINCT ON requires ORDER BY to lead with its column; allele.id then picks which row survives + # per record — inert today (only one protein allele exists). + .order_by(prot_link.mapping_record_id, prot_allele.id) + .subquery() ) statement = ( @@ -128,26 +130,39 @@ def get_lean_score_set_variants( Variant.hgvs_pro, Variant.hgvs_splice, MappingRecord.hgvs_assay_level, - protein_hgvs_subquery.label("protein_hgvs"), + protein_allele.c.protein_hgvs.label("protein_hgvs"), Allele.vrs_digest, Allele.clingen_allele_id, VepAlleleConsequence.functional_consequence, ) - .outerjoin(MappingRecord, and_(MappingRecord.variant_id == Variant.id, record_live)) - # Only the authoritative (measured) allele is needed, so the link join stays 1:1 with the - # variant — one authoritative link per live record (the invariant the mapping job upholds). + # Variant -> its live mapping record. LEFT join so unmapped variants stay (with null mapped fields) + # for the table + score histogram. live_at rides in the ON clause, never a WHERE: a WHERE would + # reject the null-record row an outer join makes for an unmapped variant, silently collapsing the + # LEFT join to an inner one. Same pattern on every join below. + .outerjoin(MappingRecord, and_(MappingRecord.variant_id == Variant.id, MappingRecord.live_at(as_of))) + # The one authoritative (measured) allele. is_authoritative keeps this 1:1 with the variant + # (one authoritative link per live record — the invariant the mapping job upholds). .outerjoin( MappingRecordAllele, and_( MappingRecordAllele.mapping_record_id == MappingRecord.id, MappingRecordAllele.is_authoritative.is_(True), - link_live, + MappingRecordAllele.live_at(as_of), ), ) + # The authoritative allele row itself (digest, ClinGen id). No live_at: alleles are immutable and + # deduplicated, not ValidTime — the live link already establishes what applies. .outerjoin(Allele, Allele.id == MappingRecordAllele.allele_id) - .outerjoin(VepAlleleConsequence, and_(VepAlleleConsequence.allele_id == Allele.id, vep_live)) + # Its VEP consequence. + .outerjoin( + VepAlleleConsequence, and_(VepAlleleConsequence.allele_id == Allele.id, VepAlleleConsequence.live_at(as_of)) + ) + # The protein subquery, matched back by record id (a globally-unique PK -> the right protein row, + # no cross-set risk). LEFT: some rows have no protein projection (UTR/intronic) -> null. + .outerjoin(protein_allele, protein_allele.c.mapping_record_id == MappingRecord.id) + # The anchor: just this score set's variants. .where(Variant.score_set_id == score_set.id) - # Variant number (the integer after '#') is the natural table order; id breaks ties stably. + # Natural table order: variant number (the integer after '#'), id breaks ties stably. .order_by(cast(func.split_part(Variant.urn, "#", 2), Integer), Variant.id) ) From fc6c8ad70413e0454606a94b4b5584630b01e9df Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Mon, 6 Jul 2026 08:21:36 -0700 Subject: [PATCH 060/252] perf(lean-variants): materialize protein projection to avoid O(N^2) plan The protein-level HGVS projection was a plain subquery. Postgres badly under-counted its DISTINCT ON cardinality (est. ~5 vs ~2.8k actual) and buried it on the inner side of a nested loop, re-running the whole score-set-scoped projection once per variant. That is O(N^2) in the score set size and made the whole-set variants query hang on large sets. Rendering it as a MATERIALIZED CTE forces single evaluation, collapsing the query back to O(N). Results are unchanged; only the plan differs. --- src/mavedb/lib/lean_variants.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/mavedb/lib/lean_variants.py b/src/mavedb/lib/lean_variants.py index 4ac9df53f..134a4eae2 100644 --- a/src/mavedb/lib/lean_variants.py +++ b/src/mavedb/lib/lean_variants.py @@ -119,7 +119,12 @@ def get_lean_score_set_variants( # DISTINCT ON requires ORDER BY to lead with its column; allele.id then picks which row survives # per record — inert today (only one protein allele exists). .order_by(prot_link.mapping_record_id, prot_allele.id) - .subquery() + # MATERIALIZED forces this to run exactly once. The planner badly under-counted the DISTINCT ON + # cardinality (est. ~5 vs ~2.8k actual) and, left as a plain subquery, buries it on the inner + # side of a nested loop, re-running the whole projection once per variant. That is O(N^2) in the + # score set size. Materializing collapses it back to O(N). + .cte("protein_allele") + .prefix_with("MATERIALIZED") ) statement = ( @@ -157,8 +162,9 @@ def get_lean_score_set_variants( .outerjoin( VepAlleleConsequence, and_(VepAlleleConsequence.allele_id == Allele.id, VepAlleleConsequence.live_at(as_of)) ) - # The protein subquery, matched back by record id (a globally-unique PK -> the right protein row, - # no cross-set risk). LEFT: some rows have no protein projection (UTR/intronic) -> null. + # The protein projection (materialized above), matched back by record id (a globally-unique PK -> + # the right protein row, no cross-set risk). LEFT: some rows have no protein projection + # (UTR/intronic) -> null. .outerjoin(protein_allele, protein_allele.c.mapping_record_id == MappingRecord.id) # The anchor: just this score set's variants. .where(Variant.score_set_id == score_set.id) From b6ea2dcefa940741fe5d76e6384bfcc88bdb9798 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 7 Jul 2026 11:44:38 -0700 Subject: [PATCH 061/252] refactor(score-sets): rename lean_variants module to score_set_variants The module serves the whole-set variant view for a score set, not a generic 'lean variant' concept; the new name pairs it with lib/score_sets.py and lib/alleles.py. Pure rename plus reference/docstring updates. --- src/mavedb/lib/{lean_variants.py => score_set_variants.py} | 0 src/mavedb/routers/score_sets.py | 2 +- src/mavedb/view_models/lean_variant.py | 2 +- .../lib/{test_lean_variants.py => test_score_set_variants.py} | 4 ++-- 4 files changed, 4 insertions(+), 4 deletions(-) rename src/mavedb/lib/{lean_variants.py => score_set_variants.py} (100%) rename tests/lib/{test_lean_variants.py => test_score_set_variants.py} (99%) diff --git a/src/mavedb/lib/lean_variants.py b/src/mavedb/lib/score_set_variants.py similarity index 100% rename from src/mavedb/lib/lean_variants.py rename to src/mavedb/lib/score_set_variants.py diff --git a/src/mavedb/routers/score_sets.py b/src/mavedb/routers/score_sets.py index 300191285..34ef50df2 100644 --- a/src/mavedb/routers/score_sets.py +++ b/src/mavedb/routers/score_sets.py @@ -41,7 +41,6 @@ find_or_create_doi_identifier, find_or_create_publication_identifier, ) -from mavedb.lib.lean_variants import get_lean_score_set_variants from mavedb.lib.logging import LoggedRoute from mavedb.lib.logging.context import ( correlation_id_for_context, @@ -50,6 +49,7 @@ ) from mavedb.lib.permissions import Action, assert_permission, has_permission from mavedb.lib.score_calibrations import create_score_calibration +from mavedb.lib.score_set_variants import get_lean_score_set_variants from mavedb.lib.score_sets import ( CLINVAR_NS_PATTERN, csv_data_to_df, diff --git a/src/mavedb/view_models/lean_variant.py b/src/mavedb/view_models/lean_variant.py index 1820cfc71..029a217a6 100644 --- a/src/mavedb/view_models/lean_variant.py +++ b/src/mavedb/view_models/lean_variant.py @@ -1,6 +1,6 @@ """Response view models for the lean whole-set view (``GET /score-sets/{urn}/variants``). -The pydantic serialization boundary over the ``lib.lean_variants`` transit dataclasses. ``from_attributes`` +The pydantic serialization boundary over the ``lib.score_set_variants`` transit dataclasses. ``from_attributes`` lets the route return the ``LeanVariantRecord`` dataclasses directly and have FastAPI's ``response_model`` coerce them (field names line up); aliases camelize per the shared base config, so clients see ``variantUrn`` / ``assayLevelHgvs`` etc. ``response_model_exclude_none`` drops absent fields, so a slot diff --git a/tests/lib/test_lean_variants.py b/tests/lib/test_score_set_variants.py similarity index 99% rename from tests/lib/test_lean_variants.py rename to tests/lib/test_score_set_variants.py index 79d630815..22f4f43d0 100644 --- a/tests/lib/test_lean_variants.py +++ b/tests/lib/test_score_set_variants.py @@ -1,5 +1,5 @@ # ruff: noqa: E402 -"""Integration tests for the lean whole-set view assembly (``lib/lean_variants.py``). +"""Integration tests for the lean whole-set view assembly (``lib/score_set_variants.py``). These pin the one-row-per-variant assembly: the submitted HGVS come off the variant, the mapped assay-level HGVS off the live mapping record, the digest/ClinGen id/VEP consequence off the @@ -15,7 +15,7 @@ pytest.importorskip("psycopg2") -from mavedb.lib.lean_variants import HgvsField, get_lean_score_set_variants +from mavedb.lib.score_set_variants import HgvsField, get_lean_score_set_variants from mavedb.models.allele import Allele from mavedb.models.mapping_record import MappingRecord from mavedb.models.mapping_record_allele import MappingRecordAllele From 6a6112af4ac636614eb32aebf1d961e38561d307 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 7 Jul 2026 11:46:37 -0700 Subject: [PATCH 062/252] feat(variants): variant detail envelope for GET /variants/{urn} Replace the flat measurement response with a two-tier envelope: flat assay-level fields (assay level, target/reference HGVS, digest, ClinGen id), per-calibration classifications, the spec-pure Cat-VRS molecular representation, and a digest-keyed annotation map (VEP/gnomAD/ClinVar). Adds ?as_of= reconstruction of the molecular layer (scores/classifications carry no ValidTime, so they are as-of-invariant) and folds in supersession self-describe (is_current/superseded_by) on this single-resource route. New lib/allele_annotations.py (digest-keyed annotation assembler, reused by the allele page) and lib/variant_detail.py (envelope composition); view models split into allele_annotation + variant_detail. Model columns gain Mapped[...] typing to back the annotation reads. Deletes the dead VariantEffectMeasurementWithScoreSet. --- src/mavedb/lib/allele_annotations.py | 151 ++++++++++++ src/mavedb/lib/variant_detail.py | 180 ++++++++++++++ src/mavedb/models/allele.py | 2 +- src/mavedb/models/clinical_control.py | 16 +- src/mavedb/models/gnomad_variant.py | 18 +- src/mavedb/models/vep_allele_consequence.py | 4 +- src/mavedb/routers/variants.py | 62 ++++- src/mavedb/view_models/allele_annotation.py | 60 +++++ src/mavedb/view_models/variant.py | 22 +- src/mavedb/view_models/variant_detail.py | 63 +++++ tests/lib/test_allele_annotations.py | 162 +++++++++++++ tests/lib/test_variant_detail.py | 248 ++++++++++++++++++++ tests/routers/test_variant.py | 172 ++++++++++++++ 13 files changed, 1116 insertions(+), 44 deletions(-) create mode 100644 src/mavedb/lib/allele_annotations.py create mode 100644 src/mavedb/lib/variant_detail.py create mode 100644 src/mavedb/view_models/allele_annotation.py create mode 100644 src/mavedb/view_models/variant_detail.py create mode 100644 tests/lib/test_allele_annotations.py create mode 100644 tests/lib/test_variant_detail.py create mode 100644 tests/routers/test_variant.py diff --git a/src/mavedb/lib/allele_annotations.py b/src/mavedb/lib/allele_annotations.py new file mode 100644 index 000000000..1b6cf112b --- /dev/null +++ b/src/mavedb/lib/allele_annotations.py @@ -0,0 +1,151 @@ +"""Digest-keyed annotation assembly over the deduplicated allele model. + +Gathers a set of alleles' external annotations — VEP consequence, gnomAD frequency, ClinVar +assertions — keyed by ``vrs_digest``. The digest is the join key the serving envelopes use to ride +annotations *alongside* the spec-pure Cat-VRS members (design §8): Cat-VRS itself carries no +annotation slot, so annotations travel beside it in a flat digest-keyed map. + +All three sources are ``ValidTime`` and ``allele_id``-keyed with no reverse collection on +``Allele`` (they are navigated set-wise from the link tables). ``as_of`` reconstructs them at a past +instant via ``live_at``, defaulting to the currently-live rows. Absence is normal and encoded as +``None``/empty — annotations are sparse and computed at all levels with no fixed level→source +mapping (D44/D44b): a genomic allele *may* carry VEP, a coding allele *may* match gnomAD. + +VEP and gnomAD are one-live-per-allele, so each is at most one value; ClinVar is multi-live (one +live link per release), so it is a list. + +This is the shared assembler for both ``GET /variants/{urn}`` (all of a variant's alleles) and +``GET /alleles/{digest}`` (a single allele) — pass whichever allele set the caller has in hand. +""" + +from dataclasses import dataclass, field +from datetime import datetime +from typing import Optional, Sequence + +from sqlalchemy import Result, select +from sqlalchemy.orm import Session + +from mavedb.models.allele import Allele +from mavedb.models.clinical_control import ClinvarControl +from mavedb.models.clinvar_allele_link import ClinvarAlleleLink +from mavedb.models.gnomad_allele_link import GnomadAlleleLink +from mavedb.models.gnomad_variant import GnomADVariant +from mavedb.models.vep_allele_consequence import VepAlleleConsequence + + +@dataclass(frozen=True) +class VepAnnotation: + """A VEP most-severe functional consequence and the Ensembl release it was resolved under.""" + + consequence: Optional[str] + source_version: str + + +@dataclass(frozen=True) +class GnomadAnnotation: + """gnomAD population frequency for an allele, from its live gnomAD link.""" + + allele_frequency: float + allele_count: int + allele_number: int + faf95_max: Optional[float] + db_version: str + db_identifier: str + + +@dataclass(frozen=True) +class ClinvarAnnotation: + """One ClinVar assertion for an allele (one per live release — an allele may carry several).""" + + clinical_significance: str + clinical_review_status: str + clinvar_variation_id: Optional[str] + clinvar_allele_id: str + db_version: str + + +@dataclass +class AlleleAnnotations: + """The external annotations for a single allele. Every field is sparse — a source with no data + for the allele is ``None`` (vep/gnomad) or an empty list (clinvar).""" + + vep: Optional[VepAnnotation] = None + gnomad: Optional[GnomadAnnotation] = None + clinvar: list[ClinvarAnnotation] = field(default_factory=list) + + +def get_allele_annotations( + db: Session, alleles: Sequence[Allele], *, as_of: Optional[datetime] = None +) -> dict[str, AlleleAnnotations]: + """Assemble the digest-keyed annotation map for ``alleles``. + + Returns one :class:`AlleleAnnotations` per distinct ``vrs_digest`` present in ``alleles``. An + allele with no annotations still gets an (empty) entry, so the map's keys mirror the alleles + handed in — the envelope can join every Cat-VRS member to a (possibly empty) annotation block. + ``as_of`` reconstructs each source at a past instant; it defaults to the currently-live rows. + """ + # id -> digest for the alleles we were handed; digest is the map key the envelope joins on. + digest_by_id = {allele.id: allele.vrs_digest for allele in alleles} + annotations: dict[str, AlleleAnnotations] = { + digest: AlleleAnnotations() for digest in digest_by_id.values() if digest is not None + } + if not digest_by_id: + return annotations + + allele_ids = list(digest_by_id.keys()) + + # VEP — at most one live consequence per allele. + vep_rows = db.execute( + select( + VepAlleleConsequence.allele_id, + VepAlleleConsequence.functional_consequence, + VepAlleleConsequence.source_version, + ) + .where(VepAlleleConsequence.allele_id.in_(allele_ids)) + .where(VepAlleleConsequence.live_at(as_of)) + ).tuples() + for allele_id, consequence, source_version in vep_rows: + digest = digest_by_id[allele_id] + if digest is not None: + annotations[digest].vep = VepAnnotation(consequence=consequence, source_version=source_version) + + # gnomAD — at most one live link per allele, resolving to a frequency record. + gnomad_rows: Result[tuple[int, GnomADVariant]] = db.execute( + select(GnomadAlleleLink.allele_id, GnomADVariant) + .join(GnomADVariant, GnomADVariant.id == GnomadAlleleLink.gnomad_variant_id) + .where(GnomadAlleleLink.allele_id.in_(allele_ids)) + .where(GnomadAlleleLink.live_at(as_of)) + ) + for allele_id, gnomad_variant in gnomad_rows: + digest = digest_by_id[allele_id] + if digest is not None: + annotations[digest].gnomad = GnomadAnnotation( + allele_frequency=gnomad_variant.allele_frequency, + allele_count=gnomad_variant.allele_count, + allele_number=gnomad_variant.allele_number, + faf95_max=gnomad_variant.faf95_max, + db_version=gnomad_variant.db_version, + db_identifier=gnomad_variant.db_identifier, + ) + + # ClinVar — multi-live (one live link per release), so each allele accumulates a list. + clinvar_rows: Result[tuple[int, ClinvarControl]] = db.execute( + select(ClinvarAlleleLink.allele_id, ClinvarControl) + .join(ClinvarControl, ClinvarControl.id == ClinvarAlleleLink.clinvar_control_id) + .where(ClinvarAlleleLink.allele_id.in_(allele_ids)) + .where(ClinvarAlleleLink.live_at(as_of)) + ) + for allele_id, clinvar_control in clinvar_rows: + digest = digest_by_id[allele_id] + if digest is not None: + annotations[digest].clinvar.append( + ClinvarAnnotation( + clinical_significance=clinvar_control.clinical_significance, + clinical_review_status=clinvar_control.clinical_review_status, + clinvar_variation_id=clinvar_control.clinvar_variation_id, + clinvar_allele_id=clinvar_control.db_identifier, + db_version=clinvar_control.db_version, + ) + ) + + return annotations diff --git a/src/mavedb/lib/variant_detail.py b/src/mavedb/lib/variant_detail.py new file mode 100644 index 000000000..09e48a072 --- /dev/null +++ b/src/mavedb/lib/variant_detail.py @@ -0,0 +1,180 @@ +"""The assayed variant-detail view backing ``GET /variants/{urn}``. + +Assembles the two-tier envelope for a single variant: flat, UI-ergonomic assay fields (the +assay-level HGVS pair, digest, ClinGen id) plus the spec-pure GA4GH ``CategoricalVariant`` (built +on the fly by :mod:`lib.cat_vrs`) and, riding alongside it keyed by VRS digest, the MaveDB layer — +per-member relations and the digest-keyed external-annotation map (:mod:`lib.annotations`). Also +carries the per-calibration functional ``classifications`` the variant falls into, and its version +standing (``is_current`` / ``superseded_by``) so a superseded variant self-describes rather than +reading as current (design §9.2). + +Temporal scope of ``as_of``: only the **molecular** layer is versioned — Cat-VRS membership and the +VEP/gnomAD/ClinVar annotations reconstruct at the past instant. Scores are immutable (``Variant`` is +not ``ValidTime``) and calibrations carry no ``ValidTime`` at all (they are frozen/append-only), so +scores and classifications are as-of-invariant and returned as they stand now. ``as_of`` defaults to +the currently-live rows. +""" + +from dataclasses import dataclass +from datetime import datetime +from typing import Any, Optional + +from sqlalchemy import select +from sqlalchemy.orm import Session + +from mavedb.lib.allele_annotations import AlleleAnnotations, get_allele_annotations +from mavedb.lib.alleles import get_live_record_allele_links +from mavedb.lib.cat_vrs import categorical_variant_for_variant +from mavedb.models.enums.annotation_layer import AnnotationLayer +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.score_calibration import ScoreCalibration +from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification +from mavedb.models.score_calibration_functional_classification_variant_association import ( + score_calibration_functional_classification_variants_association_table as classification_variants, +) +from mavedb.models.score_set import ScoreSet +from mavedb.models.variant import Variant + + +@dataclass(frozen=True) +class VariantClassificationRecord: + """One functional classification the variant falls into, tagged with its calibration context. + + ``calibration_id`` / ``primary`` locate the classification among the score set's (possibly + several) calibrations — ``primary`` is the UI default. ``classification`` is the ORM + :class:`ScoreCalibrationFunctionalClassification` (functional call, ACMG, oddspath), serialized + at the view-model boundary. + """ + + calibration_id: int + primary: bool + classification: ScoreCalibrationFunctionalClassification + + +@dataclass(frozen=True) +class VariantDetail: + """The assembled variant-detail envelope (transit; serialized by ``view_models.variant``).""" + + urn: str + scores: Optional[dict[str, Any]] + counts: Optional[dict[str, Any]] + classifications: list[VariantClassificationRecord] + + # Flat, UI-ergonomic assay-level fields. + assay_level: Optional[str] + target_hgvs: Optional[str] # submitted, target/assay coordinates + reference_hgvs: Optional[str] # mapped, reference coordinates, assay level + assay_level_digest: Optional[str] + clingen_allele_id: Optional[str] + + # Spec-pure GA4GH Cat-VRS (no MaveDB fields inside), plus the MaveDB layer riding alongside. + molecular_representation: Optional[dict[str, Any]] + mode: Optional[str] # CatVrsMode value — projection | reverse_translation + member_relations: dict[str, str] # member vrs_digest -> relation (member -> defining) + + # External annotations, keyed by VRS digest, joined to the Cat-VRS members. + annotations: dict[str, AlleleAnnotations] + + # Version standing — self-descriptive for a superseded variant. + is_current: bool + superseded_by: Optional[str] # URN of the superseding variant's score set, if any (and readable) + + +def _classifications_for_variant( + db: Session, variant: Variant, *, visible_calibration_ids: Optional[set[int]] +) -> list[VariantClassificationRecord]: + """The functional classifications the variant belongs to (membership is materialized on the + calibration→classification→variant m2m). One row per calibration that classifies the variant; + the primary calibration sorts first. ``visible_calibration_ids`` restricts to calibrations the + caller resolved as readable (``None`` = no restriction, for lib-level use).""" + statement = ( + select(ScoreCalibrationFunctionalClassification, ScoreCalibration.primary, ScoreCalibration.id) + .join( + classification_variants, + classification_variants.c.functional_classification_id == ScoreCalibrationFunctionalClassification.id, + ) + .join(ScoreCalibration, ScoreCalibration.id == ScoreCalibrationFunctionalClassification.calibration_id) + .where(classification_variants.c.variant_id == variant.id) + .order_by(ScoreCalibration.primary.desc(), ScoreCalibration.id) + ) + if visible_calibration_ids is not None: + statement = statement.where(ScoreCalibration.id.in_(visible_calibration_ids)) + + return [ + VariantClassificationRecord(calibration_id=calibration_id, primary=primary, classification=classification) + for classification, primary, calibration_id in db.execute(statement) + ] + + +def _submitted_assay_level_hgvs(variant: Variant, assay_level: Optional[str]) -> Optional[str]: + """The depositor-submitted HGVS in the variant's assay frame: protein for a protein assay, + otherwise the nucleotide expression (genomic or coding share ``hgvs_nt``).""" + if assay_level == AnnotationLayer.protein.value: + return variant.hgvs_pro + return variant.hgvs_nt + + +def get_variant_detail( + db: Session, + variant: Variant, + *, + superseding_score_set: Optional[ScoreSet] = None, + visible_calibration_ids: Optional[set[int]] = None, + as_of: Optional[datetime] = None, +) -> VariantDetail: + """Assemble the variant-detail envelope for ``variant``. + + ``superseding_score_set`` is the newer version the caller has already resolved for visibility + (blanked when the user cannot read it, mirroring ``fetch_score_set_by_urn``); its presence drives + ``is_current``/``superseded_by``. ``visible_calibration_ids`` restricts classifications to + readable calibrations. ``as_of`` reconstructs the molecular layer (Cat-VRS membership + + annotations); scores and classifications are as-of-invariant. See the module docstring. + """ + data = variant.data if isinstance(variant.data, dict) else {} + scores = data.get("score_data") + counts = data.get("count_data") + + # The live mapping record supplies the assay level and the mapped (reference-frame) assay HGVS. + record = db.scalar( + select(MappingRecord).where(MappingRecord.variant_id == variant.id).where(MappingRecord.live_at(as_of)) + ) + assay_level = record.assay_level if record is not None else None + reference_hgvs = record.hgvs_assay_level if record is not None else None + + # The live allele links: the authoritative allele gives the assay-level digest + ClinGen id; all + # linked alleles seed the digest-keyed annotation map. + links = get_live_record_allele_links(db, variant.id, as_of=as_of) + authoritative = next((link for link in links if link.is_authoritative), None) + assay_level_digest = authoritative.allele.vrs_digest if authoritative is not None else None + clingen_allele_id = authoritative.allele.clingen_allele_id if authoritative is not None else None + + annotations = get_allele_annotations(db, [link.allele for link in links], as_of=as_of) + + # Spec-pure Cat-VRS built on the fly, plus the MaveDB layer (mode + per-member relations). + transit = categorical_variant_for_variant(db, variant.id, name=variant.urn or "", as_of=as_of) + if transit is not None: + molecular_representation = transit.categorical_variant.model_dump(mode="json", exclude_none=True) + mode: Optional[str] = transit.mode.value + member_relations = {digest: relation.value for digest, relation in transit.member_relations.items()} + else: + molecular_representation = None + mode = None + member_relations = {} + + return VariantDetail( + urn=variant.urn or "", + scores=scores, + counts=counts, + classifications=_classifications_for_variant(db, variant, visible_calibration_ids=visible_calibration_ids), + assay_level=assay_level, + target_hgvs=_submitted_assay_level_hgvs(variant, assay_level), + reference_hgvs=reference_hgvs, + assay_level_digest=assay_level_digest, + clingen_allele_id=clingen_allele_id, + molecular_representation=molecular_representation, + mode=mode, + member_relations=member_relations, + annotations=annotations, + is_current=superseding_score_set is None, + superseded_by=superseding_score_set.urn if superseding_score_set is not None else None, + ) diff --git a/src/mavedb/models/allele.py b/src/mavedb/models/allele.py index a57c4c802..67eadd519 100644 --- a/src/mavedb/models/allele.py +++ b/src/mavedb/models/allele.py @@ -18,7 +18,7 @@ class Allele(Base): id: Mapped[int] = Column(Integer, primary_key=True) - vrs_digest = Column(String, nullable=False) + vrs_digest: Mapped[str] = Column(String, nullable=False) level = Column(String(length=16), nullable=False) hgvs_g = Column(String, nullable=True) diff --git a/src/mavedb/models/clinical_control.py b/src/mavedb/models/clinical_control.py index 694aad58e..388444a09 100644 --- a/src/mavedb/models/clinical_control.py +++ b/src/mavedb/models/clinical_control.py @@ -1,5 +1,5 @@ from datetime import date -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Optional from sqlalchemy import Column, Date, Integer, String, UniqueConstraint from sqlalchemy.orm import Mapped, relationship @@ -22,18 +22,18 @@ class ClinvarControl(Base): id: Mapped[int] = Column(Integer, primary_key=True) - gene_symbol = Column(String, nullable=False, index=True) + gene_symbol: Mapped[str] = Column(String, nullable=False, index=True) - clinical_significance = Column(String, nullable=False) - clinical_review_status = Column(String, nullable=False) + clinical_significance: Mapped[str] = Column(String, nullable=False) + clinical_review_status: Mapped[str] = Column(String, nullable=False) - db_name = Column(String, nullable=False, index=True) + db_name: Mapped[str] = Column(String, nullable=False, index=True) # ClinVar Allele ID (row level link). - db_identifier = Column(String, nullable=False, index=True) - db_version = Column(String, nullable=False, index=True) + db_identifier: Mapped[str] = Column(String, nullable=False, index=True) + db_version: Mapped[str] = Column(String, nullable=False, index=True) # ClinVar Variation ID (variation level link). - clinvar_variation_id = Column(String, nullable=True) + clinvar_variation_id: Mapped[Optional[str]] = Column(String, nullable=True) creation_date = Column(Date, nullable=False, default=date.today) modification_date = Column(Date, nullable=False, default=date.today, onupdate=date.today) diff --git a/src/mavedb/models/gnomad_variant.py b/src/mavedb/models/gnomad_variant.py index bfc7c8026..bb9a1faf6 100644 --- a/src/mavedb/models/gnomad_variant.py +++ b/src/mavedb/models/gnomad_variant.py @@ -1,5 +1,5 @@ from datetime import date -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Optional from sqlalchemy import Column, Date, Float, Integer, String from sqlalchemy.orm import Mapped, relationship @@ -17,16 +17,16 @@ class GnomADVariant(Base): id: Mapped[int] = Column(Integer, primary_key=True) - db_name = Column(String, nullable=False) - db_identifier = Column(String, nullable=False, index=True) - db_version = Column(String, nullable=False) + db_name: Mapped[str] = Column(String, nullable=False) + db_identifier: Mapped[str] = Column(String, nullable=False, index=True) + db_version: Mapped[str] = Column(String, nullable=False) - allele_count = Column(Integer, nullable=False) - allele_number = Column(Integer, nullable=False) - allele_frequency = Column(Float, nullable=False) + allele_count: Mapped[int] = Column(Integer, nullable=False) + allele_number: Mapped[int] = Column(Integer, nullable=False) + allele_frequency: Mapped[float] = Column(Float, nullable=False) - faf95_max = Column(Float, nullable=True) - faf95_max_ancestry = Column(String, nullable=True) + faf95_max: Mapped[Optional[float]] = Column(Float, nullable=True) + faf95_max_ancestry: Mapped[Optional[str]] = Column(String, nullable=True) creation_date = Column(Date, nullable=False, default=date.today) modification_date = Column(Date, nullable=False, default=date.today, onupdate=date.today) diff --git a/src/mavedb/models/vep_allele_consequence.py b/src/mavedb/models/vep_allele_consequence.py index de41230a5..4133a1231 100644 --- a/src/mavedb/models/vep_allele_consequence.py +++ b/src/mavedb/models/vep_allele_consequence.py @@ -1,5 +1,5 @@ from datetime import date -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Optional from sqlalchemy import Column, Date, ForeignKey, Index, Integer, String, text from sqlalchemy.orm import Mapped, relationship @@ -52,7 +52,7 @@ class VepAlleleConsequence(ValidTime, Base): ForeignKey("alleles.id", ondelete="RESTRICT"), nullable=False, ) - functional_consequence = Column(String, nullable=True) + functional_consequence: Mapped[Optional[str]] = Column(String, nullable=True) source_version: Mapped[str] = Column(String, nullable=False) access_date: Mapped[date] = Column(Date, nullable=False) diff --git a/src/mavedb/routers/variants.py b/src/mavedb/routers/variants.py index c195f9030..97352a8ec 100644 --- a/src/mavedb/routers/variants.py +++ b/src/mavedb/routers/variants.py @@ -1,8 +1,10 @@ import itertools import logging import re +from datetime import datetime +from typing import Optional -from fastapi import APIRouter, Depends +from fastapi import APIRouter, Depends, Query, Response from fastapi.exceptions import HTTPException from sqlalchemy import select from sqlalchemy.exc import MultipleResultsFound @@ -15,6 +17,7 @@ from mavedb.lib.logging.context import logging_context, save_to_logging_context from mavedb.lib.permissions import Action, assert_permission, has_permission from mavedb.lib.types.authentication import UserData +from mavedb.lib.variant_detail import get_variant_detail from mavedb.models.mapped_variant import MappedVariant from mavedb.models.score_set import ScoreSet from mavedb.models.variant import Variant @@ -28,8 +31,8 @@ from mavedb.view_models.variant import ( ClingenAlleleIdVariantLookupResponse, ClingenAlleleIdVariantLookupsRequest, - VariantEffectMeasurementWithScoreSet, ) +from mavedb.view_models.variant_detail import VariantDetail TAG_NAME = "Variants" @@ -437,19 +440,37 @@ def lookup_variants( @router.get( "/variants/{urn}", status_code=200, - response_model=VariantEffectMeasurementWithScoreSet, + response_model=VariantDetail, responses={**ACCESS_CONTROL_ERROR_RESPONSES}, response_model_exclude_none=True, - summary="Fetch variant by URN", + summary="Fetch assayed variant detail by URN", ) -def get_variant(*, urn: str, db: Session = Depends(deps.get_db), user_data: UserData = Depends(get_current_user)): - """ - Fetch a single variant by URN. +def get_variant( + *, + urn: str, + response: Response, + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the molecular layer (Cat-VRS membership, VEP/gnomAD/ClinVar annotations) as it " + "stood at this instant, over the variant's fixed score. ISO 8601, ideally timezone-aware. " + "Content valid-time only — it never re-selects a score-set version, and scores/classifications " + "are as-of-invariant. Defaults to current." + ), + ), + db: Session = Depends(deps.get_db), + user_data: Optional[UserData] = Depends(get_current_user), +): + """Fetch the two-tier detail envelope for a single assayed variant by URN. + + Flat assay-level fields for the common UI case plus the spec-pure GA4GH CategoricalVariant and a + digest-keyed annotation map for machine/standard consumers. A superseded variant is served (it is + the citable unit) but self-describes via isCurrent/supersededBy rather than reading as current. """ - save_to_logging_context({"requested_resource": urn}) + save_to_logging_context({"requested_resource": urn, "as_of": as_of}) + response.headers["X-As-Of"] = as_of.isoformat() if as_of is not None else "current" try: - query = db.query(Variant).filter(Variant.urn == urn) - variant = query.one_or_none() + variant = db.query(Variant).filter(Variant.urn == urn).one_or_none() except MultipleResultsFound: logger.info(msg="Could not fetch the requested variant; Multiple such variants exist.", extra=logging_context()) raise HTTPException(status_code=500, detail=f"multiple variants with URN '{urn}' were found") @@ -459,4 +480,23 @@ def get_variant(*, urn: str, db: Session = Depends(deps.get_db), user_data: User raise HTTPException(status_code=404, detail=f"variant with URN '{urn}' not found") assert_permission(user_data, variant.score_set, Action.READ) - return variant + + # Version standing: resolve the superseding version's visibility exactly as fetch_score_set_by_urn + # does — a newer version the user cannot read is not leaked (the variant then reads as current). + superseding = variant.score_set.superseding_score_set + if superseding is not None and not has_permission(user_data, superseding, Action.READ).permitted: + superseding = None + + visible_calibration_ids = { + sc.id + for sc in variant.score_set.score_calibrations + if sc.id is not None and has_permission(user_data, sc, Action.READ).permitted + } + + return get_variant_detail( + db, + variant, + superseding_score_set=superseding, + visible_calibration_ids=visible_calibration_ids, + as_of=as_of, + ) diff --git a/src/mavedb/view_models/allele_annotation.py b/src/mavedb/view_models/allele_annotation.py new file mode 100644 index 000000000..b538d4639 --- /dev/null +++ b/src/mavedb/view_models/allele_annotation.py @@ -0,0 +1,60 @@ +"""Response view models for an allele's external annotations (VEP / gnomAD / ClinVar). + +The pydantic serialization boundary over the ``lib.allele_annotations`` transit dataclasses +(``from_attributes`` coerces them directly). These are the sparse, digest-keyed annotation blocks +that ride *alongside* the spec-pure Cat-VRS on ``GET /variants/{urn}`` and are the allele's own block +on ``GET /alleles/{digest}`` — so they live here, shared by both serving views rather than in either +one's file. +""" + +from typing import Optional + +from mavedb.view_models.base.base import BaseModel + + +class VepAnnotation(BaseModel): + """VEP most-severe functional consequence and the Ensembl release it resolved under.""" + + consequence: Optional[str] = None + source_version: str + + class Config: + from_attributes = True + + +class GnomadAnnotation(BaseModel): + """gnomAD population frequency for an allele.""" + + allele_frequency: float + allele_count: int + allele_number: int + faf95_max: Optional[float] = None + db_version: str + db_identifier: str + + class Config: + from_attributes = True + + +class ClinvarAnnotation(BaseModel): + """One ClinVar assertion for an allele (an allele may carry one per release).""" + + clinical_significance: str + clinical_review_status: str + clinvar_variation_id: Optional[str] = None + clinvar_allele_id: str + db_version: str + + class Config: + from_attributes = True + + +class AlleleAnnotations(BaseModel): + """The external annotations for one allele, sparse — each source absent unless it has data.""" + + vep: Optional[VepAnnotation] = None + gnomad: Optional[GnomadAnnotation] = None + clinvar: list[ClinvarAnnotation] = [] + + class Config: + from_attributes = True diff --git a/src/mavedb/view_models/variant.py b/src/mavedb/view_models/variant.py index b01b01832..ad02023df 100644 --- a/src/mavedb/view_models/variant.py +++ b/src/mavedb/view_models/variant.py @@ -9,7 +9,7 @@ from mavedb.view_models.mapped_variant import MappedVariant, SavedMappedVariant if TYPE_CHECKING: - from mavedb.view_models.score_set import ScoreSet, ShortScoreSet + from mavedb.view_models.score_set import ShortScoreSet class VariantEffectMeasurementBase(BaseModel): @@ -76,13 +76,6 @@ class VariantEffectMeasurement(SavedVariantEffectMeasurement): pass -class VariantEffectMeasurementWithScoreSet(SavedVariantEffectMeasurement): - """Variant effect measurement view model with mapped variants and score set""" - - score_set: "ScoreSet" - mapped_variants: list[MappedVariant] - - class VariantEffectMeasurementWithShortScoreSet(SavedVariantEffectMeasurement): """Variant effect measurement view model with mapped variants and a limited set of score set details""" @@ -96,8 +89,11 @@ class ClingenAlleleIdVariantLookupsRequest(BaseModel): clingen_allele_ids: list[str] -class Variant(BaseModel): - """View model for a variant, defined by its ClinGen allele id, with associated variant effect measurements""" +class ClingenAlleleVariants(BaseModel): + """The assayed variants sharing one ClinGen allele id, with their variant effect measurements. + + An allele-side grouping (keyed by ClinGen allele id), it is the mapped-allele view of "which + measurements share this molecular identity," not the assayed variant itself.""" clingen_allele_id: str variant_effect_measurements: list[VariantEffectMeasurementWithShortScoreSet] @@ -107,6 +103,6 @@ class ClingenAlleleIdVariantLookupResponse(BaseModel): """Response model for a variant lookup by ClinGen allele ID""" clingen_allele_id: str - exact_match: Optional[Variant] = None - equivalent_nt: list[Variant] = [] - equivalent_aa: list[Variant] = [] + exact_match: Optional[ClingenAlleleVariants] = None + equivalent_nt: list[ClingenAlleleVariants] = [] + equivalent_aa: list[ClingenAlleleVariants] = [] diff --git a/src/mavedb/view_models/variant_detail.py b/src/mavedb/view_models/variant_detail.py new file mode 100644 index 000000000..18a78f844 --- /dev/null +++ b/src/mavedb/view_models/variant_detail.py @@ -0,0 +1,63 @@ +"""Response view models for the assayed variant-detail view (``GET /variants/{urn}``, design §7.1). + +The pydantic serialization boundary over the ``lib.variant_detail`` transit dataclasses +(``from_attributes`` coerces them directly); aliases camelize per the shared base config. Kept in its +own module — mirroring ``lib/variant_detail.py`` and the ``lean_variant`` precedent — so +``view_models/variant.py`` stays the core measurement + lookup file. +""" + +from typing import Any, Optional + +from mavedb.view_models.allele_annotation import AlleleAnnotations +from mavedb.view_models.base.base import BaseModel +from mavedb.view_models.score_calibration import SavedFunctionalClassification + + +class VariantClassification(BaseModel): + """A functional classification the variant falls into, tagged with its calibration context. + + A score set may carry several calibrations, so a variant has one classification per calibration; + ``primary`` flags the UI default. The classifications are calibration-derived but as-of-invariant + (calibrations carry no valid-time), so they are always the current calibration state. + """ + + calibration_id: int + primary: bool + classification: SavedFunctionalClassification + + class Config: + from_attributes = True + + +class VariantDetail(BaseModel): + """The assayed variant-detail envelope (``GET /variants/{urn}``, design §7.1). + + Two tiers: flat, UI-ergonomic assay fields (the ``targetHgvs``/``referenceHgvs`` coordinate pair + is a client-side toggle, no refetch) plus the spec-pure GA4GH ``molecularRepresentation`` + (``CategoricalVariant``, no MaveDB fields inside). The MaveDB layer rides alongside keyed by VRS + digest: ``memberRelations`` (member→defining relation) and the ``annotations`` map. ``isCurrent`` + /``supersededBy`` let a superseded variant self-describe. Absent fields are omitted. + """ + + urn: str + scores: Optional[dict[str, Any]] = None + counts: Optional[dict[str, Any]] = None + classifications: list[VariantClassification] = [] + + assay_level: Optional[str] = None + target_hgvs: Optional[str] = None + reference_hgvs: Optional[str] = None + assay_level_digest: Optional[str] = None + clingen_allele_id: Optional[str] = None + + molecular_representation: Optional[dict[str, Any]] = None + mode: Optional[str] = None + member_relations: dict[str, str] = {} + + annotations: dict[str, AlleleAnnotations] = {} + + is_current: bool + superseded_by: Optional[str] = None + + class Config: + from_attributes = True diff --git a/tests/lib/test_allele_annotations.py b/tests/lib/test_allele_annotations.py new file mode 100644 index 000000000..cedb4267c --- /dev/null +++ b/tests/lib/test_allele_annotations.py @@ -0,0 +1,162 @@ +# ruff: noqa: E402 +"""Integration tests for the digest-keyed annotation assembler (``lib/allele_annotations.py``). + +These pin: the map is keyed by ``vrs_digest`` and covers every allele handed in (an allele with no +annotations still gets an empty entry); VEP/gnomAD resolve to one value each while ClinVar is a list +(multi-live, one per release); absence is ``None``/empty; and ``as_of`` reconstructs each source at a +past instant over the immutable, content-addressed alleles. +""" + +from datetime import datetime, timedelta, timezone + +import pytest + +pytest.importorskip("psycopg2") + +from mavedb.lib.allele_annotations import get_allele_annotations +from mavedb.models.allele import Allele +from mavedb.models.clinical_control import ClinvarControl +from mavedb.models.clinvar_allele_link import ClinvarAlleleLink +from mavedb.models.gnomad_allele_link import GnomadAlleleLink +from mavedb.models.gnomad_variant import GnomADVariant +from mavedb.models.vep_allele_consequence import VepAlleleConsequence + +T0 = datetime(2020, 1, 1, tzinfo=timezone.utc) +T1 = datetime(2021, 1, 1, tzinfo=timezone.utc) + + +def _allele(session, digest, *, level="cdna"): + allele = Allele(vrs_digest=digest, level=level, post_mapped={"type": "Allele"}) + session.add(allele) + session.commit() + return allele + + +def _vep(session, allele, value, *, source_version="116", valid_from=None): + row = VepAlleleConsequence( + allele_id=allele.id, + functional_consequence=value, + source_version=source_version, + access_date="2026-01-01", + valid_from=valid_from, + ) + session.add(row) + session.commit() + return row + + +def _gnomad(session, allele, *, allele_frequency=0.0123, db_version="4.1.0"): + gv = GnomADVariant( + db_name="gnomAD", + db_identifier=f"gnomad-{allele.vrs_digest}", + db_version=db_version, + allele_count=42, + allele_number=10000, + allele_frequency=allele_frequency, + faf95_max=0.05, + ) + session.add(gv) + session.commit() + link = GnomadAlleleLink(allele_id=allele.id, gnomad_variant_id=gv.id) + session.add(link) + session.commit() + return gv + + +def _clinvar(session, allele, *, significance="Likely benign", db_version="11_2024", variation_id="12345"): + control = ClinvarControl( + db_identifier=f"cv-{db_version}-{allele.vrs_digest}", + gene_symbol="PTEN", + clinical_significance=significance, + clinical_review_status="criteria provided, multiple submitters, no conflicts", + db_name="ClinVar", + db_version=db_version, + clinvar_variation_id=variation_id, + ) + session.add(control) + session.commit() + link = ClinvarAlleleLink(allele_id=allele.id, clinvar_control_id=control.id) + session.add(link) + session.commit() + return control + + +@pytest.mark.integration +def test_empty_allele_list_returns_empty_map(session, setup_lib_db): + assert get_allele_annotations(session, []) == {} + + +@pytest.mark.integration +def test_allele_without_annotations_gets_an_empty_entry(session, setup_lib_db): + """The map's keys mirror the alleles handed in — a fully un-annotated allele still gets an + (empty) block, so the envelope can join every Cat-VRS member to something.""" + allele = _allele(session, "bare-digest") + + annotations = get_allele_annotations(session, [allele]) + + assert set(annotations) == {"bare-digest"} + block = annotations["bare-digest"] + assert block.vep is None + assert block.gnomad is None + assert block.clinvar == [] + + +@pytest.mark.integration +def test_all_three_sources_present(session, setup_lib_db): + allele = _allele(session, "rich-digest") + _vep(session, allele, "missense_variant") + _gnomad(session, allele, allele_frequency=0.0123) + _clinvar(session, allele, significance="Likely benign") + + block = get_allele_annotations(session, [allele])["rich-digest"] + + assert block.vep is not None and block.vep.consequence == "missense_variant" + assert block.vep.source_version == "116" + assert block.gnomad is not None and block.gnomad.allele_frequency == 0.0123 + assert block.gnomad.faf95_max == 0.05 + assert [c.clinical_significance for c in block.clinvar] == ["Likely benign"] + + +@pytest.mark.integration +def test_clinvar_is_multi_live_across_releases(session, setup_lib_db): + """ClinVar stacks one live link per release, so an allele can carry several assertions at once.""" + allele = _allele(session, "cv-digest") + _clinvar(session, allele, significance="Likely benign", db_version="11_2024") + _clinvar(session, allele, significance="Pathogenic", db_version="05_2025") + + block = get_allele_annotations(session, [allele])["cv-digest"] + + assert {(c.db_version, c.clinical_significance) for c in block.clinvar} == { + ("11_2024", "Likely benign"), + ("05_2025", "Pathogenic"), + } + + +@pytest.mark.integration +def test_map_is_keyed_by_digest_across_multiple_alleles(session, setup_lib_db): + a1 = _allele(session, "digest-1", level="cdna") + a2 = _allele(session, "digest-2", level="protein") + _vep(session, a1, "missense_variant") + _gnomad(session, a2) + + annotations = get_allele_annotations(session, [a1, a2]) + + assert set(annotations) == {"digest-1", "digest-2"} + assert annotations["digest-1"].vep is not None and annotations["digest-1"].gnomad is None + assert annotations["digest-2"].gnomad is not None and annotations["digest-2"].vep is None + + +@pytest.mark.integration +def test_as_of_reconstructs_the_historical_annotation(session, setup_lib_db): + """A changed VEP consequence supersedes the old row; as_of selects the value live at the instant.""" + allele = _allele(session, "vep-digest") + old = _vep(session, allele, "missense_variant", source_version="110", valid_from=T0) + old.retire(session, at=T1) + session.commit() + _vep(session, allele, "stop_gained", source_version="116", valid_from=T1) + + current = get_allele_annotations(session, [allele])["vep-digest"] + historical = get_allele_annotations(session, [allele], as_of=T1 - timedelta(days=1))["vep-digest"] + + assert current.vep is not None and current.vep.consequence == "stop_gained" + assert historical.vep is not None and historical.vep.consequence == "missense_variant" diff --git a/tests/lib/test_variant_detail.py b/tests/lib/test_variant_detail.py new file mode 100644 index 000000000..61594e4f0 --- /dev/null +++ b/tests/lib/test_variant_detail.py @@ -0,0 +1,248 @@ +# ruff: noqa: E402 +"""Integration tests for the variant-detail envelope assembly (``lib/variant_detail.py``). + +These pin the composition: the flat assay fields (level, target/reference HGVS, digest, ClinGen id) +off the live record + authoritative allele; the spec-pure Cat-VRS + MaveDB mode/member-relations off +the on-the-fly builder; the digest-keyed annotation map; the per-calibration classifications +(primary first, filterable by visibility); scores/counts passthrough; and the ``is_current`` / +``superseded_by`` version standing. Cat-VRS/annotation internals are covered in their own suites — here +we assert they are wired in and keyed correctly. +""" + +from types import SimpleNamespace + +import pytest + +pytest.importorskip("psycopg2") + +from mavedb.lib.variant_detail import get_variant_detail +from mavedb.models.allele import Allele +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.score_calibration import ScoreCalibration +from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification +from mavedb.models.user import User +from mavedb.models.variant import Variant +from mavedb.models.vep_allele_consequence import VepAlleleConsequence +from tests.helpers.constants import TEST_MINIMAL_VARIANT, TEST_USER + +# A spec-valid post_mapped VRS Allele — the Cat-VRS builder hydrates this, so it must parse. +_VALID_DIGEST = "0123456789abcdefghijABCDEFGHIJ_-" + + +def _post_mapped() -> dict: + return { + "id": f"ga4gh:VA.{_VALID_DIGEST}", + "type": "Allele", + "state": {"type": "LiteralSequenceExpression", "sequence": "F"}, + "digest": _VALID_DIGEST, + "location": { + "id": f"ga4gh:SL.{_VALID_DIGEST}", + "end": 6, + "type": "SequenceLocation", + "start": 5, + "digest": _VALID_DIGEST, + "sequenceReference": { + "type": "SequenceReference", + "label": "NP_000000.0", + "refgetAccession": "SQ.0123456789abcdefghijABCDEFGHIJ_-", + }, + }, + } + + +def _variant(session, score_set, suffix, *, data=None, hgvs_nt=None, hgvs_pro=None): + variant = Variant( + urn=f"{score_set.urn}#{suffix}", + score_set_id=score_set.id, + data=data if data is not None else TEST_MINIMAL_VARIANT["data"], + hgvs_nt=hgvs_nt, + hgvs_pro=hgvs_pro, + creation_date=TEST_MINIMAL_VARIANT["creation_date"], + modification_date=TEST_MINIMAL_VARIANT["modification_date"], + ) + session.add(variant) + session.commit() + return variant + + +def _allele(session, digest, *, level="cdna", clingen_allele_id=None): + allele = Allele(vrs_digest=digest, level=level, post_mapped=_post_mapped(), clingen_allele_id=clingen_allele_id) + session.add(allele) + session.commit() + return allele + + +def _record(session, variant, *, assay_level="cdna", hgvs_assay_level=None): + record = MappingRecord( + variant_id=variant.id, + assay_level=assay_level, + hgvs_assay_level=hgvs_assay_level, + mapping_api_version="test.0.0", + ) + session.add(record) + session.commit() + return record + + +def _link(session, record, allele, *, is_authoritative=False): + link = MappingRecordAllele(mapping_record_id=record.id, allele_id=allele.id, is_authoritative=is_authoritative) + session.add(link) + session.commit() + return link + + +def _vep(session, allele, value): + session.add( + VepAlleleConsequence( + allele_id=allele.id, functional_consequence=value, source_version="116", access_date="2026-01-01" + ) + ) + session.commit() + + +def _calibration(session, score_set, *, primary, classifications): + """A calibration with functional classifications; each ``classifications`` entry is + ``(functional, [variants...])`` and its m2m membership is set to those variants.""" + user = session.query(User).filter(User.username == TEST_USER["username"]).one() + calibration = ScoreCalibration( + score_set_id=score_set.id, + title=f"cal-{'primary' if primary else 'secondary'}", + primary=primary, + private=False, + created_by_id=user.id, + modified_by_id=user.id, + ) + session.add(calibration) + session.commit() + for functional, variants in classifications: + fc = ScoreCalibrationFunctionalClassification( + calibration_id=calibration.id, label=functional, functional_classification=functional + ) + fc.variants = variants + session.add(fc) + session.commit() + return calibration + + +@pytest.mark.integration +def test_full_envelope_for_a_coding_assay(session, setup_lib_db_with_score_set): + """Mode 1 (coding measured): flat assay fields off the record + authoritative allele, spec-pure + Cat-VRS in projection mode, and a digest-keyed annotation map covering the linked alleles.""" + score_set = setup_lib_db_with_score_set + variant = _variant( + session, score_set, 1, data={"score_data": {"score": -2.3}}, hgvs_nt="c.1216G>A", hgvs_pro="p.Ala406Thr" + ) + record = _record(session, variant, assay_level="cdna", hgvs_assay_level="NM_000546.6:c.1216G>A") + measured = _allele(session, "cdna-digest", level="cdna", clingen_allele_id="CA123") + protein = _allele(session, "prot-digest", level="protein") + _link(session, record, measured, is_authoritative=True) + _link(session, record, protein) + _vep(session, measured, "missense_variant") + + detail = get_variant_detail(session, variant) + + assert detail.urn == variant.urn + assert detail.scores == {"score": -2.3} + assert detail.assay_level == "cdna" + assert detail.target_hgvs == "c.1216G>A" # submitted, target frame + assert detail.reference_hgvs == "NM_000546.6:c.1216G>A" # mapped, reference frame + assert detail.assay_level_digest == "cdna-digest" + assert detail.clingen_allele_id == "CA123" + assert detail.mode == "projection" + assert detail.molecular_representation is not None + assert detail.molecular_representation["type"] == "CategoricalVariant" + # The protein member is a translation of the measured coding allele (member -> defining). + assert detail.member_relations == {"prot-digest": "translation_of"} + # Annotations keyed by digest, covering both linked alleles; VEP rode in on the measured allele. + assert set(detail.annotations) == {"cdna-digest", "prot-digest"} + assert detail.annotations["cdna-digest"].vep is not None + assert detail.annotations["cdna-digest"].vep.consequence == "missense_variant" + assert detail.is_current is True + assert detail.superseded_by is None + + +@pytest.mark.integration +def test_protein_assay_targets_the_protein_frame(session, setup_lib_db_with_score_set): + """Mode 2 (protein measured): assay level is protein, target HGVS is the submitted p. string, and + the nt member `encodes` the defining protein allele.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, hgvs_nt="c.1216G>A", hgvs_pro="p.Ala406Thr") + record = _record(session, variant, assay_level="protein", hgvs_assay_level="NP_000537.3:p.Ala406Thr") + measured = _allele(session, "prot-digest", level="protein", clingen_allele_id="PA9") + coding = _allele(session, "cdna-digest", level="cdna") + _link(session, record, measured, is_authoritative=True) + _link(session, record, coding) + + detail = get_variant_detail(session, variant) + + assert detail.assay_level == "protein" + assert detail.target_hgvs == "p.Ala406Thr" # protein assay -> submitted protein string + assert detail.reference_hgvs == "NP_000537.3:p.Ala406Thr" + assert detail.assay_level_digest == "prot-digest" + assert detail.mode == "reverse_translation" + assert detail.member_relations == {"cdna-digest": "encodes"} + + +@pytest.mark.integration +def test_unmapped_variant_has_null_molecular_layer(session, setup_lib_db_with_score_set): + """No live record: flat mapped fields and Cat-VRS are null, the annotation map is empty, and the + variant reads as current.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, data={"score_data": {"score": 0.5}}, hgvs_nt="c.1A>T") + + detail = get_variant_detail(session, variant) + + assert detail.scores == {"score": 0.5} + assert detail.assay_level is None + assert detail.target_hgvs == "c.1A>T" # submitted still surfaces (no assay level -> nucleotide) + assert detail.reference_hgvs is None + assert detail.assay_level_digest is None + assert detail.clingen_allele_id is None + assert detail.molecular_representation is None + assert detail.mode is None + assert detail.member_relations == {} + assert detail.annotations == {} + assert detail.is_current is True + + +@pytest.mark.integration +def test_classifications_are_per_calibration_primary_first(session, setup_lib_db_with_score_set): + """A variant carries one classification per calibration that classifies it; the primary sorts first.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, data={"score_data": {"score": 0.8}}) + _calibration(session, score_set, primary=False, classifications=[("normal", [variant])]) + _calibration(session, score_set, primary=True, classifications=[("abnormal", [variant])]) + + detail = get_variant_detail(session, variant) + + assert [(c.primary, c.classification.functional_classification.value) for c in detail.classifications] == [ + (True, "abnormal"), + (False, "normal"), + ] + + +@pytest.mark.integration +def test_visible_calibration_ids_filters_classifications(session, setup_lib_db_with_score_set): + """Only classifications from calibrations the caller resolved as visible are returned.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, data={"score_data": {"score": 0.8}}) + visible = _calibration(session, score_set, primary=True, classifications=[("abnormal", [variant])]) + _calibration(session, score_set, primary=False, classifications=[("normal", [variant])]) + + detail = get_variant_detail(session, variant, visible_calibration_ids={visible.id}) + + assert [c.calibration_id for c in detail.classifications] == [visible.id] + + +@pytest.mark.integration +def test_superseded_variant_self_describes(session, setup_lib_db_with_score_set): + """When a superseding version is passed in, the variant reports it rather than reading as current.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1) + newer = SimpleNamespace(urn="urn:mavedb:00000001-a-2") # only .urn is read + + detail = get_variant_detail(session, variant, superseding_score_set=newer) + + assert detail.is_current is False + assert detail.superseded_by == "urn:mavedb:00000001-a-2" diff --git a/tests/routers/test_variant.py b/tests/routers/test_variant.py new file mode 100644 index 000000000..68b48cecc --- /dev/null +++ b/tests/routers/test_variant.py @@ -0,0 +1,172 @@ +# ruff: noqa: E402 +"""Router tests for the assayed variant-detail endpoint (``GET /variants/{urn}``). + +Exercise the HTTP surface end to end: the two-tier envelope serializes and validates, the ``as_of`` +content-time is echoed in the ``X-As-Of`` header, a superseded variant self-describes rather than +reading as current, and the READ gate holds for private score sets. +""" + +import pytest + +arq = pytest.importorskip("arq") +cdot = pytest.importorskip("cdot") +fastapi = pytest.importorskip("fastapi") + +from sqlalchemy import select + +from mavedb.models.allele import Allele +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.score_set import ScoreSet as ScoreSetDbModel +from mavedb.models.variant import Variant as VariantDbModel +from mavedb.models.vep_allele_consequence import VepAlleleConsequence +from mavedb.view_models.variant_detail import VariantDetail +from tests.helpers.dependency_overrider import DependencyOverrider +from tests.helpers.util.experiment import create_experiment +from tests.helpers.util.score_set import create_seq_score_set_with_variants +from tests.helpers.util.user import change_ownership + +_VALID_DIGEST = "0123456789abcdefghijABCDEFGHIJ_-" + + +def _post_mapped() -> dict: + """A spec-valid post_mapped VRS Allele — the Cat-VRS builder hydrates this on the detail path.""" + return { + "id": f"ga4gh:VA.{_VALID_DIGEST}", + "type": "Allele", + "state": {"type": "LiteralSequenceExpression", "sequence": "F"}, + "digest": _VALID_DIGEST, + "location": { + "id": f"ga4gh:SL.{_VALID_DIGEST}", + "end": 6, + "type": "SequenceLocation", + "start": 5, + "digest": _VALID_DIGEST, + "sequenceReference": { + "type": "SequenceReference", + "label": "NP_000000.0", + "refgetAccession": "SQ.0123456789abcdefghijABCDEFGHIJ_-", + }, + }, + } + + +def _seed_mapping(session, variant_urn): + """Give a variant a live coding-measured mapping record whose authoritative allele carries a + digest + ClinGen id + a live VEP consequence, plus a protein member — enough to build Cat-VRS.""" + variant = session.scalar(select(VariantDbModel).where(VariantDbModel.urn == variant_urn)) + record = MappingRecord( + variant_id=variant.id, + assay_level="cdna", + hgvs_assay_level="NM_000546.6:c.1216G>A", + mapping_api_version="test.0.0", + ) + session.add(record) + session.commit() + + measured = Allele(vrs_digest="cdna-digest", level="cdna", post_mapped=_post_mapped(), clingen_allele_id="CA123") + protein = Allele(vrs_digest="prot-digest", level="protein", post_mapped=_post_mapped()) + session.add_all([measured, protein]) + session.commit() + + session.add_all( + [ + MappingRecordAllele(mapping_record_id=record.id, allele_id=measured.id, is_authoritative=True), + MappingRecordAllele(mapping_record_id=record.id, allele_id=protein.id, is_authoritative=False), + VepAlleleConsequence( + allele_id=measured.id, + functional_consequence="missense_variant", + source_version="116", + access_date="2026-01-01", + ), + ] + ) + session.commit() + + +def test_get_variant_detail_envelope(client, session, data_provider, data_files, setup_router_db): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + urn = f"{score_set['urn']}#1" + _seed_mapping(session, urn) + + response = client.get(f"/api/v1/variants/{urn.replace(chr(35), '%23')}") + + assert response.status_code == 200 + body = response.json() + VariantDetail.model_validate(body) + + assert body["urn"] == urn + assert body["assayLevel"] == "cdna" + assert body["referenceHgvs"] == "NM_000546.6:c.1216G>A" + assert body["assayLevelDigest"] == "cdna-digest" + assert body["clingenAlleleId"] == "CA123" + assert body["mode"] == "projection" + # Spec-pure Cat-VRS carries its own field names (no camelization of the nested GA4GH object). + assert body["molecularRepresentation"]["type"] == "CategoricalVariant" + assert body["memberRelations"] == {"prot-digest": "translation_of"} + assert body["annotations"]["cdna-digest"]["vep"]["consequence"] == "missense_variant" + assert body["isCurrent"] is True + assert "supersededBy" not in body # dropped by exclude_none when current + assert response.headers["X-As-Of"] == "current" + + +def test_get_variant_detail_unknown_urn_is_404(client, setup_router_db): + response = client.get("/api/v1/variants/urn:mavedb:00000000-a-1#1") + assert response.status_code == 404 + + +def test_get_variant_detail_echoes_as_of_header(client, session, data_provider, data_files, setup_router_db): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + urn = f"{score_set['urn']}#1" + + historical = client.get(f"/api/v1/variants/{urn.replace(chr(35), '%23')}", params={"as_of": "2020-01-01T00:00:00Z"}) + + assert historical.status_code == 200 + assert historical.headers["X-As-Of"] == "2020-01-01T00:00:00+00:00" + # Before any mapping existed, the molecular layer is empty. + assert "molecularRepresentation" not in historical.json() + + +def test_superseded_variant_self_describes(client, session, data_provider, data_files, setup_router_db): + """A variant on a superseded score set is still served, but flags its version standing.""" + experiment = create_experiment(client) + older = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + newer = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + + # Link newer as the version that replaces older (replaces_id / superseding relationship). + older_db = session.scalar(select(ScoreSetDbModel).where(ScoreSetDbModel.urn == older["urn"])) + newer_db = session.scalar(select(ScoreSetDbModel).where(ScoreSetDbModel.urn == newer["urn"])) + newer_db.superseded_score_set_id = older_db.id + session.commit() + + response = client.get(f"/api/v1/variants/{older['urn']}%231") + + assert response.status_code == 200 + body = response.json() + assert body["isCurrent"] is False + assert body["supersededBy"] == newer["urn"] + + +def test_get_variant_detail_anonymous_cannot_read_private( + client, session, data_provider, data_files, setup_router_db, anonymous_app_overrides +): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + change_ownership(session, score_set["urn"], ScoreSetDbModel) + + with DependencyOverrider(anonymous_app_overrides): + response = client.get(f"/api/v1/variants/{score_set['urn']}%231") + + assert response.status_code == 404 From 3016dc90154a01a149d7c24b7a8042f688124401 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 7 Jul 2026 11:46:57 -0700 Subject: [PATCH 063/252] refactor(clinical-controls): serve from allele-link substrate with as_of Rework the score-set clinical-controls routes off the legacy MappedVariant.clinical_controls association onto the new-model substrate (ClinvarAlleleLink -> Allele -> MappingRecordAllele -> MappingRecord), matching how the annotation pipeline now writes links. Adds ?as_of= (X-As-Of echo) to reconstruct the allele->ClinVar link state at an instant, and sorts control options newest-version-first. Test infra: add the reusable new-substrate seeder tests/helpers/util/annotation.py (seed_mapping_record + AlleleSpec building the MappingRecord/Allele/link graph), migrate the clinical-controls fixtures (link_clinical_controls_to_alleles) and the lean-view seeder onto it, and retire ClinvarAlleleLinks (not MappedVariant.current) to exercise the live-link filter. --- src/mavedb/routers/score_sets.py | 114 ++++++++++++++++++++++------ tests/helpers/util/annotation.py | 123 +++++++++++++++++++++++++++++++ tests/helpers/util/score_set.py | 32 ++++---- tests/routers/test_score_set.py | 74 +++++++------------ 4 files changed, 260 insertions(+), 83 deletions(-) create mode 100644 tests/helpers/util/annotation.py diff --git a/src/mavedb/routers/score_sets.py b/src/mavedb/routers/score_sets.py index 34ef50df2..9d150fed0 100644 --- a/src/mavedb/routers/score_sets.py +++ b/src/mavedb/routers/score_sets.py @@ -17,8 +17,8 @@ from ga4gh.va_spec.base.core import ExperimentalVariantFunctionalImpactStudyResult, Statement from pydantic import ValidationError from sqlalchemy import or_, select -from sqlalchemy.exc import MultipleResultsFound, IntegrityError -from sqlalchemy.orm import Session, contains_eager +from sqlalchemy.exc import IntegrityError, MultipleResultsFound +from sqlalchemy.orm import Session from mavedb import deps from mavedb.data_providers.services import CSV_UPLOAD_S3_BUCKET_NAME, s3_client @@ -74,13 +74,17 @@ generate_score_set_urn, ) from mavedb.lib.workflow.pipeline_factory import PipelineFactory +from mavedb.models.allele import Allele from mavedb.models.clinical_control import ClinvarControl +from mavedb.models.clinvar_allele_link import ClinvarAlleleLink from mavedb.models.contributor import Contributor from mavedb.models.enums.processing_state import ProcessingState from mavedb.models.experiment import Experiment from mavedb.models.gnomad_variant import GnomADVariant from mavedb.models.license import License from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.pipeline import Pipeline from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.score_set import ScoreSet @@ -2448,18 +2452,27 @@ async def publish_score_set( async def get_clinical_controls_for_score_set( *, urn: str, + response: Response, + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the allele → ClinVar link state as it stood at this instant. " + "ISO 8601, ideally timezone-aware. Defaults to current." + ), + ), # We'd prefer to reserve `db` as a query parameter. _db: Session = Depends(deps.get_db), user_data: UserData = Depends(get_current_user), db: Optional[str] = None, version: Optional[str] = None, -) -> Sequence[ClinvarControl]: +) -> list[clinical_control.ClinicalControlWithMappedVariants]: """ Fetch relevant clinical controls for a given score set. """ - save_to_logging_context({"requested_resource": urn, "resource_property": "clinical_controls"}) + save_to_logging_context({"requested_resource": urn, "resource_property": "clinical_controls", "as_of": as_of}) + response.headers["X-As-Of"] = as_of.isoformat() if as_of is not None else "current" - # Rename user facing kwargs for consistency with code base naming conventions. My-py doesn't care for us redefining db. + # Rename user facing kwargs for consistency with code base naming conventions. db_name = db db_version = version @@ -2474,25 +2487,38 @@ async def get_clinical_controls_for_score_set( assert_permission(user_data, item, Action.READ) clinical_controls_query = ( - select(ClinvarControl) - .join(ClinvarControl.mapped_variants) - .join(MappedVariant.variant) - .options(contains_eager(ClinvarControl.mapped_variants).contains_eager(MappedVariant.variant)) - .filter(MappedVariant.current.is_(True)) - .filter(Variant.score_set_id == item.id) + select(ClinvarControl, Variant.urn.label("variant_urn")) + .join(ClinvarAlleleLink, ClinvarAlleleLink.clinvar_control_id == ClinvarControl.id) + .join(Allele, Allele.id == ClinvarAlleleLink.allele_id) + .join(MappingRecordAllele, MappingRecordAllele.allele_id == Allele.id) + .join(MappingRecord, MappingRecord.id == MappingRecordAllele.mapping_record_id) + .join(Variant, Variant.id == MappingRecord.variant_id) + .where(ClinvarAlleleLink.live_at(as_of)) + .where(MappingRecordAllele.live_at(as_of)) + .where(MappingRecord.live_at(as_of)) + .where(Variant.score_set_id == item.id) + .distinct() ) if db_name is not None: save_to_logging_context({"db_name": db_name}) - clinical_controls_query = clinical_controls_query.filter(ClinvarControl.db_name == db_name) + clinical_controls_query = clinical_controls_query.where(ClinvarControl.db_name == db_name) if db_version is not None: save_to_logging_context({"db_version": db_version}) - clinical_controls_query = clinical_controls_query.filter(ClinvarControl.db_version == db_version) + clinical_controls_query = clinical_controls_query.where(ClinvarControl.db_version == db_version) - clinical_controls: Sequence[ClinvarControl] = _db.scalars(clinical_controls_query).unique().all() + rows = _db.execute(clinical_controls_query).tuples().all() - if not clinical_controls: + controls: dict[int, ClinvarControl] = {} + urns_by_control: dict[int, list[str]] = {} + for ctrl, variant_urn in rows: + if ctrl.id not in controls: + controls[ctrl.id] = ctrl + urns_by_control[ctrl.id] = [] + urns_by_control[ctrl.id].append(variant_urn) + + if not controls: logger.info( msg="No clinical control variants matching the provided filters are associated with the requested score set.", extra=logging_context(), @@ -2502,9 +2528,25 @@ async def get_clinical_controls_for_score_set( detail=f"No clinical control variants matching the provided filters associated with score set URN {urn} were found", ) - save_to_logging_context({"resource_count": len(clinical_controls)}) + save_to_logging_context({"resource_count": len(controls)}) - return clinical_controls + return [ + clinical_control.ClinicalControlWithMappedVariants.model_validate( + { + "id": ctrl.id, + "db_identifier": ctrl.db_identifier, + "gene_symbol": ctrl.gene_symbol, + "clinical_significance": ctrl.clinical_significance, + "clinical_review_status": ctrl.clinical_review_status, + "db_version": ctrl.db_version, + "db_name": ctrl.db_name, + "modification_date": ctrl.modification_date, + "creation_date": ctrl.creation_date, + "mapped_variants": [{"variant_urn": v_urn} for v_urn in urns_by_control[ctrl.id]], + } + ) + for ctrl in controls.values() + ] @router.get( @@ -2518,6 +2560,13 @@ async def get_clinical_controls_for_score_set( async def get_clinical_controls_options_for_score_set( *, urn: str, + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the allele → ClinVar link state as it stood at this instant. " + "ISO 8601, ideally timezone-aware. Defaults to current." + ), + ), # We'd prefer to reserve `db` as a query parameter. db: Session = Depends(deps.get_db), user_data: UserData = Depends(get_current_user), @@ -2525,7 +2574,13 @@ async def get_clinical_controls_options_for_score_set( """ Fetch clinical control options for a given score set. """ - save_to_logging_context({"requested_resource": urn, "resource_property": "clinical_control_options"}) + save_to_logging_context( + { + "requested_resource": urn, + "resource_property": "clinical_control_options", + "as_of": as_of, + } + ) item: Optional[ScoreSet] = db.scalars(select(ScoreSet).where(ScoreSet.urn == urn)).one_or_none() if not item: @@ -2539,13 +2594,19 @@ async def get_clinical_controls_options_for_score_set( clinical_controls_query = ( select(ClinvarControl.db_name, ClinvarControl.db_version) - .join(MappedVariant, ClinvarControl.mapped_variants) - .join(Variant) - .where(MappedVariant.current.is_(True)) + .join(ClinvarAlleleLink, ClinvarAlleleLink.clinvar_control_id == ClinvarControl.id) + .join(Allele, Allele.id == ClinvarAlleleLink.allele_id) + .join(MappingRecordAllele, MappingRecordAllele.allele_id == Allele.id) + .join(MappingRecord, MappingRecord.id == MappingRecordAllele.mapping_record_id) + .join(Variant, Variant.id == MappingRecord.variant_id) + .where(ClinvarAlleleLink.live_at(as_of)) + .where(MappingRecordAllele.live_at(as_of)) + .where(MappingRecord.live_at(as_of)) .where(Variant.score_set_id == item.id) + .distinct() ) - clinical_controls_for_item = db.execute(clinical_controls_query).unique() + clinical_controls_for_item = db.execute(clinical_controls_query) # NOTE: We return options even for pairwise groupings which may have no associated mapped variants # and 404 when ultimately requested together. @@ -2563,8 +2624,15 @@ async def get_clinical_controls_options_for_score_set( detail=f"no clinical control variants associated with score set URN {urn} were found", ) + def _version_sort_key(v: str) -> tuple[int, int]: + try: + month, year = v.split("_") + return int(year), int(month) + except (ValueError, AttributeError): + return (0, 0) + return [ - dict(zip(("db_name", "available_versions"), (db_name, db_versions))) + {"db_name": db_name, "available_versions": sorted(db_versions, key=_version_sort_key, reverse=True)} for db_name, db_versions in clinical_control_options.items() ] diff --git a/tests/helpers/util/annotation.py b/tests/helpers/util/annotation.py new file mode 100644 index 000000000..7e80a7c73 --- /dev/null +++ b/tests/helpers/util/annotation.py @@ -0,0 +1,123 @@ +"""Seeding helpers for the new-model annotation substrate. + +The serving layer (the lean whole-set view, the ``/variants/{urn}`` detail envelope, allele +annotations, and the score-set clinical-controls routes) reads a variant's mapped alleles and +their annotations through the ``MappingRecord`` / ``Allele`` / ``MappingRecordAllele`` graph and +the per-source link tables (``ClinvarAlleleLink`` / ``GnomadAlleleLink`` / ``VepAlleleConsequence``) +— not the legacy ``MappedVariant`` association tables. These helpers build that graph in tests so a +score set's variants carry alleles and annotations the way the mapping/annotation pipeline would. + +``seed_mapping_record`` is the single reusable builder; higher-level helpers (e.g. +``link_clinical_controls_to_alleles``) compose it. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import datetime +from typing import Any, Optional, Sequence, Union + +from sqlalchemy import select +from sqlalchemy.orm import Session + +from mavedb.models.allele import Allele +from mavedb.models.clinvar_allele_link import ClinvarAlleleLink +from mavedb.models.gnomad_allele_link import GnomadAlleleLink +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.variant import Variant +from mavedb.models.vep_allele_consequence import VepAlleleConsequence + + +@dataclass +class AlleleSpec: + """One allele on a mapping record, plus the annotations that hang off it. + + Digests are the allele's identity in the serving layer, so they must be distinct within a test. + ``clinvar_control_ids`` / ``gnomad_variant_ids`` reference rows seeded elsewhere (the router + fixtures seed ClinVar controls with ids 1 and 2); each id becomes a live link on this allele. + """ + + digest: str + level: str = "cdna" + is_authoritative: bool = False + clingen_allele_id: Optional[str] = None + hgvs_c: Optional[str] = None + hgvs_p: Optional[str] = None + hgvs_g: Optional[str] = None + post_mapped: Optional[dict] = None + vep_consequence: Optional[str] = None + clinvar_control_ids: Sequence[int] = field(default_factory=tuple) + gnomad_variant_ids: Sequence[int] = field(default_factory=tuple) + + +def seed_mapping_record( + session: Session, + variant: Union[Variant, str], + *, + alleles: Sequence[AlleleSpec], + assay_level: str = "cdna", + hgvs_assay_level: Optional[str] = None, + mapping_api_version: str = "test.0.0", + valid_from: Optional[datetime] = None, +) -> MappingRecord: + """Give ``variant`` a live mapping record carrying ``alleles`` and their annotations. + + ``variant`` may be a ``Variant`` instance or its URN. When ``valid_from`` is set it stamps the + record, its allele links, and every annotation link, so ``as_of`` reconstruction tests can place + the whole graph at a chosen instant (absent it, ``valid_from`` defaults to ``now()`` and the row + is live). Returns the created ``MappingRecord``. + """ + if isinstance(variant, str): + resolved = session.scalar(select(Variant).where(Variant.urn == variant)) + assert resolved is not None, f"variant with URN '{variant}' not found" + variant = resolved + + record = MappingRecord( + variant_id=variant.id, + assay_level=assay_level, + hgvs_assay_level=hgvs_assay_level, + mapping_api_version=mapping_api_version, + ) + if valid_from is not None: + record.valid_from = valid_from + session.add(record) + session.commit() + + for spec in alleles: + allele = Allele( + vrs_digest=spec.digest, + level=spec.level, + clingen_allele_id=spec.clingen_allele_id, + hgvs_c=spec.hgvs_c, + hgvs_p=spec.hgvs_p, + hgvs_g=spec.hgvs_g, + post_mapped=spec.post_mapped if spec.post_mapped is not None else {"type": "Allele"}, + ) + session.add(allele) + session.commit() + + links: list[Any] = [ + MappingRecordAllele( + mapping_record_id=record.id, allele_id=allele.id, is_authoritative=spec.is_authoritative + ) + ] + if spec.vep_consequence is not None: + links.append( + VepAlleleConsequence( + allele_id=allele.id, + functional_consequence=spec.vep_consequence, + source_version="116", + access_date="2026-01-01", + ) + ) + links.extend(ClinvarAlleleLink(allele_id=allele.id, clinvar_control_id=cid) for cid in spec.clinvar_control_ids) + links.extend(GnomadAlleleLink(allele_id=allele.id, gnomad_variant_id=gid) for gid in spec.gnomad_variant_ids) + + if valid_from is not None: + for link in links: + link.valid_from = valid_from + session.add_all(links) + + session.commit() + return record diff --git a/tests/helpers/util/score_set.py b/tests/helpers/util/score_set.py index 83ebcbca8..067c627fe 100644 --- a/tests/helpers/util/score_set.py +++ b/tests/helpers/util/score_set.py @@ -24,6 +24,7 @@ TEST_VALID_PRE_MAPPED_VRS_ALLELE_VRS2_X, TEST_VALID_PRE_MAPPED_VRS_CIS_PHASED_BLOCK, ) +from tests.helpers.util.annotation import AlleleSpec, seed_mapping_record from tests.helpers.util.variant import mock_worker_variant_insertion @@ -204,26 +205,31 @@ def create_acc_score_set_with_variants( return score_set -def link_clinical_controls_to_mapped_variants(db, score_set): - mapped_variants = db.scalars( - select(MappedVariantDbModel) - .join(VariantDbModel) +def link_clinical_controls_to_alleles(db, score_set): + """Seed the new-model annotation graph for a score set's first two variants and link the two + seeded ClinVar controls to their alleles via ``ClinvarAlleleLink``. + + The first variant's allele gets the ClinVar control (id 1), the second gets the generic control + (id 2) — mirroring the two-control shape the clinical-controls routes are asserted against. + """ + variants = db.scalars( + select(VariantDbModel) .join(ScoreSetDbModel) .where(ScoreSetDbModel.urn == score_set["urn"]) + .order_by(VariantDbModel.id) ).all() - # The first mapped variant gets the clinvar control, the second gets the generic control. - mapped_variants[0].clinical_controls.append( - db.scalar(select(ClinicalControlDbModel).where(ClinicalControlDbModel.id == 1)) + seed_mapping_record( + db, + variants[0], + alleles=[AlleleSpec(digest="clinical-control-allele-0", is_authoritative=True, clinvar_control_ids=[1])], ) - mapped_variants[1].clinical_controls.append( - db.scalar(select(ClinicalControlDbModel).where(ClinicalControlDbModel.id == 2)) + seed_mapping_record( + db, + variants[1], + alleles=[AlleleSpec(digest="clinical-control-allele-1", is_authoritative=True, clinvar_control_ids=[2])], ) - db.add(mapped_variants[0]) - db.add(mapped_variants[1]) - db.commit() - def link_clinvar_control_to_mapped_variant(db, score_set): """Link the seeded ClinVar clinical control (id=1) to the first mapped variant of a score set.""" diff --git a/tests/routers/test_score_set.py b/tests/routers/test_score_set.py index 055d6bbb2..30e5af017 100644 --- a/tests/routers/test_score_set.py +++ b/tests/routers/test_score_set.py @@ -4,7 +4,7 @@ import json import re from copy import deepcopy -from datetime import date +from datetime import date, datetime, timezone from io import StringIO from unittest.mock import patch @@ -24,13 +24,9 @@ from mavedb.models.experiment import Experiment as ExperimentDbModel from mavedb.models.job_run import JobRun from mavedb.models.pipeline import Pipeline -from mavedb.models.allele import Allele -from mavedb.models.mapped_variant import MappedVariant as MappedVariantDbModel -from mavedb.models.mapping_record import MappingRecord -from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.clinvar_allele_link import ClinvarAlleleLink from mavedb.models.score_set import ScoreSet as ScoreSetDbModel from mavedb.models.variant import Variant as VariantDbModel -from mavedb.models.vep_allele_consequence import VepAlleleConsequence from mavedb.view_models.lean_variant import LeanVariant from mavedb.view_models.orcid import OrcidUser from mavedb.view_models.score_set import ScoreSet, ScoreSetCreate @@ -80,11 +76,12 @@ create_publish_and_promote_score_calibration, create_test_score_calibration_in_score_set_via_client, ) +from tests.helpers.util.annotation import AlleleSpec, seed_mapping_record from tests.helpers.util.score_set import ( create_seq_score_set, create_seq_score_set_with_mapped_variants, create_seq_score_set_with_variants, - link_clinical_controls_to_mapped_variants, + link_clinical_controls_to_alleles, link_clinvar_control_to_mapped_variant, link_gnomad_variants_to_mapped_variants, publish_score_set, @@ -3862,7 +3859,7 @@ def test_can_fetch_current_clinical_controls_for_score_set(client, setup_router_ score_set = create_seq_score_set_with_mapped_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) - link_clinical_controls_to_mapped_variants(session, score_set) + link_clinical_controls_to_alleles(session, score_set) response = client.get(f"/api/v1/score-sets/{score_set['urn']}/clinical-controls") assert response.status_code == 200 @@ -3890,7 +3887,7 @@ def test_can_fetch_current_clinical_controls_for_score_set_with_parameters( score_set = create_seq_score_set_with_mapped_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) - link_clinical_controls_to_mapped_variants(session, score_set) + link_clinical_controls_to_alleles(session, score_set) query_string = "?" for param, accessor in parameters: @@ -3915,7 +3912,7 @@ def test_cannot_fetch_clinical_controls_for_nonexistent_score_set( score_set = create_seq_score_set_with_mapped_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) - link_clinical_controls_to_mapped_variants(session, score_set) + link_clinical_controls_to_alleles(session, score_set) response = client.get(f"/api/v1/score-sets/{score_set['urn'] + 'xxx'}/clinical-controls") @@ -3950,7 +3947,7 @@ def test_can_fetch_current_clinical_control_options_for_score_set( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) - link_clinical_controls_to_mapped_variants(session, score_set) + link_clinical_controls_to_alleles(session, score_set) response = client.get(f"/api/v1/score-sets/{score_set['urn']}/clinical-controls/options") assert response.status_code == 200 @@ -3975,17 +3972,14 @@ def test_clinical_control_options_exclude_non_current(client, setup_router_db, s score_set = create_seq_score_set_with_mapped_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) - link_clinical_controls_to_mapped_variants(session, score_set) + link_clinical_controls_to_alleles(session, score_set) - # Mark all mapped variants as non-current to simulate stale mapping data. - mapped_variants = session.scalars( - select(MappedVariantDbModel) - .join(VariantDbModel) - .join(ScoreSetDbModel) - .where(ScoreSetDbModel.urn == score_set["urn"]) - ).all() - for mv in mapped_variants: - mv.current = False + # The options query only surfaces live allele → ClinVar links; retiring them (as a re-map would, + # by stamping valid_to) must leave no current controls and 404. Test-isolated DB, so these are + # the only links present. + links = session.scalars(select(ClinvarAlleleLink).where(ClinvarAlleleLink.valid_to.is_(None))).all() + for link in links: + link.valid_to = datetime.now(timezone.utc) session.commit() response = client.get(f"/api/v1/score-sets/{score_set['urn']}/clinical-controls/options") @@ -4569,36 +4563,22 @@ def test_cannot_fetch_gnomad_variants_for_score_set_when_none_exist( def _seed_lean_mapping(session, variant_urn): """Give one variant a live coding-measured mapping record (with an assay-level HGVS) whose authoritative allele carries a digest + ClinGen id + a live VEP consequence.""" - variant = session.scalar(select(VariantDbModel).where(VariantDbModel.urn == variant_urn)) - record = MappingRecord( - variant_id=variant.id, + seed_mapping_record( + session, + variant_urn, assay_level="cdna", hgvs_assay_level="NM_000546.6:c.1216G>A", - mapping_api_version="test.0.0", - ) - session.add(record) - session.commit() - - measured = Allele(vrs_digest="cdna-digest", level="cdna", post_mapped={"type": "Allele"}, clingen_allele_id="CA123") - protein = Allele( - vrs_digest="prot-digest", level="protein", post_mapped={"type": "Allele"}, hgvs_p="NP_000537.3:p.Ala406Thr" - ) - session.add_all([measured, protein]) - session.commit() - - session.add_all( - [ - MappingRecordAllele(mapping_record_id=record.id, allele_id=measured.id, is_authoritative=True), - MappingRecordAllele(mapping_record_id=record.id, allele_id=protein.id, is_authoritative=False), - VepAlleleConsequence( - allele_id=measured.id, - functional_consequence="missense_variant", - source_version="116", - access_date="2026-01-01", + alleles=[ + AlleleSpec( + digest="cdna-digest", + level="cdna", + is_authoritative=True, + clingen_allele_id="CA123", + vep_consequence="missense_variant", ), - ] + AlleleSpec(digest="prot-digest", level="protein", hgvs_p="NP_000537.3:p.Ala406Thr"), + ], ) - session.commit() def test_get_lean_variants(client, session, data_provider, data_files, setup_router_db): From a5918d19786ba46c9ed7623e0212790776d31b94 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 7 Jul 2026 14:10:46 -0700 Subject: [PATCH 064/252] refactor(clinical-controls): serve list and options from a shared lib module The list and options handlers each carried the same ClinvarAlleleLink -> Allele -> MappingRecordAllele -> MappingRecord -> Variant join chain inline, plus an ad-hoc MM_YYYY version sort. Extract the traversal, grouping, and sort into lib/clinical_controls.py so the two endpoints share one definition and cannot drift, matching how the rest of the redesign pushes query logic into lib/. Drop the now-unused model imports and add a unit test pinning the version-sort contract the UI parses in parallel. --- src/mavedb/lib/clinical_controls.py | 105 ++++++++++++++++++++++++++++ src/mavedb/routers/score_sets.py | 82 ++++------------------ tests/lib/test_clinical_controls.py | 32 +++++++++ 3 files changed, 149 insertions(+), 70 deletions(-) create mode 100644 src/mavedb/lib/clinical_controls.py create mode 100644 tests/lib/test_clinical_controls.py diff --git a/src/mavedb/lib/clinical_controls.py b/src/mavedb/lib/clinical_controls.py new file mode 100644 index 000000000..5aa10b1ce --- /dev/null +++ b/src/mavedb/lib/clinical_controls.py @@ -0,0 +1,105 @@ +"""Live ClinVar-control lookups for a score set, over the allele-link substrate. + +A score set's clinical controls are the ClinVar assertions whose allele is live-linked to one of the +score set's variants, reached by walking ``ClinvarAlleleLink → Allele → MappingRecordAllele → +MappingRecord → Variant`` with every ``ValidTime`` hop constrained to the same instant (``as_of``, +defaulting to currently-live). Both serving endpoints — the controls list +(``GET /score-sets/{urn}/clinical-controls``) and its options facet (``.../options``) — walk that same +chain, so it lives here once: the two cannot drift, and the ``as_of`` semantics are defined in a single +place. +""" + +from datetime import datetime +from typing import Any, Optional, TypeVar + +from sqlalchemy import Select, select +from sqlalchemy.orm import Session + +from mavedb.models.allele import Allele +from mavedb.models.clinical_control import ClinvarControl +from mavedb.models.clinvar_allele_link import ClinvarAlleleLink +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.variant import Variant + +SelectT = TypeVar("SelectT", bound=Select[Any]) + + +def _scope_to_live_score_set_controls(stmt: SelectT, score_set_id: int, as_of: Optional[datetime]) -> SelectT: + """Join ``stmt`` (already selecting from :class:`ClinvarControl`) down the allele-link chain to the + score set's variants and constrain every ``ValidTime`` hop to ``as_of``. ``distinct`` because the + chain fans a control out across each variant it links to.""" + return ( + stmt.join(ClinvarAlleleLink, ClinvarAlleleLink.clinvar_control_id == ClinvarControl.id) + .join(Allele, Allele.id == ClinvarAlleleLink.allele_id) + .join(MappingRecordAllele, MappingRecordAllele.allele_id == Allele.id) + .join(MappingRecord, MappingRecord.id == MappingRecordAllele.mapping_record_id) + .join(Variant, Variant.id == MappingRecord.variant_id) + .where(ClinvarAlleleLink.live_at(as_of)) + .where(MappingRecordAllele.live_at(as_of)) + .where(MappingRecord.live_at(as_of)) + .where(Variant.score_set_id == score_set_id) + .distinct() + ) + + +def get_clinical_controls_with_variant_urns( + db: Session, + score_set_id: int, + *, + as_of: Optional[datetime] = None, + db_name: Optional[str] = None, + db_version: Optional[str] = None, +) -> list[tuple[ClinvarControl, list[str]]]: + """The live ClinVar controls for a score set, each paired with the URNs of the score-set variants + that link to it. Optionally narrowed to one control DB (``db_name``) and/or release (``db_version``). + Controls come back in first-seen order; each control's variant URNs preserve query order.""" + stmt = _scope_to_live_score_set_controls( + select(ClinvarControl, Variant.urn.label("variant_urn")), score_set_id, as_of + ) + if db_name is not None: + stmt = stmt.where(ClinvarControl.db_name == db_name) + if db_version is not None: + stmt = stmt.where(ClinvarControl.db_version == db_version) + + controls: dict[int, ClinvarControl] = {} + urns_by_control: dict[int, list[str]] = {} + for ctrl, variant_urn in db.execute(stmt).tuples(): + # A persisted variant reached through the join always carries a URN (unique, set at creation); + # narrow the nullable column so the URN list matches the view model's required `str`. + if variant_urn is None: + continue + if ctrl.id not in controls: + controls[ctrl.id] = ctrl + urns_by_control[ctrl.id] = [] + urns_by_control[ctrl.id].append(variant_urn) + + return [(ctrl, urns_by_control[ctrl.id]) for ctrl in controls.values()] + + +def get_clinical_control_options( + db: Session, score_set_id: int, *, as_of: Optional[datetime] = None +) -> list[tuple[str, list[str]]]: + """The options facet: each control DB live-linked to the score set, paired with its available + versions newest-first. Options are returned even for pairwise groupings that may have no variants + when ultimately requested together (the list endpoint 404s that case).""" + stmt = _scope_to_live_score_set_controls( + select(ClinvarControl.db_name, ClinvarControl.db_version), score_set_id, as_of + ) + options: dict[str, list[str]] = {} + for db_name, db_version in db.execute(stmt): + options.setdefault(db_name, []).append(db_version) + + return [ + (db_name, sorted(versions, key=clinvar_version_sort_key, reverse=True)) for db_name, versions in options.items() + ] + + +def clinvar_version_sort_key(version: str) -> tuple[int, int]: + """Sort key for a ClinVar release string in ``MM_YYYY`` form, ordering by ``(year, month)``. + Unparseable versions sort to the bottom as ``(0, 0)`` rather than raising.""" + try: + month, year = version.split("_") + return int(year), int(month) + except (ValueError, AttributeError): + return (0, 0) diff --git a/src/mavedb/routers/score_sets.py b/src/mavedb/routers/score_sets.py index 9d150fed0..413d64a07 100644 --- a/src/mavedb/routers/score_sets.py +++ b/src/mavedb/routers/score_sets.py @@ -47,6 +47,7 @@ logging_context, save_to_logging_context, ) +from mavedb.lib.clinical_controls import get_clinical_control_options, get_clinical_controls_with_variant_urns from mavedb.lib.permissions import Action, assert_permission, has_permission from mavedb.lib.score_calibrations import create_score_calibration from mavedb.lib.score_set_variants import get_lean_score_set_variants @@ -74,17 +75,12 @@ generate_score_set_urn, ) from mavedb.lib.workflow.pipeline_factory import PipelineFactory -from mavedb.models.allele import Allele -from mavedb.models.clinical_control import ClinvarControl -from mavedb.models.clinvar_allele_link import ClinvarAlleleLink from mavedb.models.contributor import Contributor from mavedb.models.enums.processing_state import ProcessingState from mavedb.models.experiment import Experiment from mavedb.models.gnomad_variant import GnomADVariant from mavedb.models.license import License from mavedb.models.mapped_variant import MappedVariant -from mavedb.models.mapping_record import MappingRecord -from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.pipeline import Pipeline from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.score_set import ScoreSet @@ -2475,6 +2471,10 @@ async def get_clinical_controls_for_score_set( # Rename user facing kwargs for consistency with code base naming conventions. db_name = db db_version = version + if db_name is not None: + save_to_logging_context({"db_name": db_name}) + if db_version is not None: + save_to_logging_context({"db_version": db_version}) item: Optional[ScoreSet] = _db.scalars(select(ScoreSet).where(ScoreSet.urn == urn)).one_or_none() if not item: @@ -2486,38 +2486,10 @@ async def get_clinical_controls_for_score_set( assert_permission(user_data, item, Action.READ) - clinical_controls_query = ( - select(ClinvarControl, Variant.urn.label("variant_urn")) - .join(ClinvarAlleleLink, ClinvarAlleleLink.clinvar_control_id == ClinvarControl.id) - .join(Allele, Allele.id == ClinvarAlleleLink.allele_id) - .join(MappingRecordAllele, MappingRecordAllele.allele_id == Allele.id) - .join(MappingRecord, MappingRecord.id == MappingRecordAllele.mapping_record_id) - .join(Variant, Variant.id == MappingRecord.variant_id) - .where(ClinvarAlleleLink.live_at(as_of)) - .where(MappingRecordAllele.live_at(as_of)) - .where(MappingRecord.live_at(as_of)) - .where(Variant.score_set_id == item.id) - .distinct() + controls = get_clinical_controls_with_variant_urns( + _db, item.id, as_of=as_of, db_name=db_name, db_version=db_version ) - if db_name is not None: - save_to_logging_context({"db_name": db_name}) - clinical_controls_query = clinical_controls_query.where(ClinvarControl.db_name == db_name) - - if db_version is not None: - save_to_logging_context({"db_version": db_version}) - clinical_controls_query = clinical_controls_query.where(ClinvarControl.db_version == db_version) - - rows = _db.execute(clinical_controls_query).tuples().all() - - controls: dict[int, ClinvarControl] = {} - urns_by_control: dict[int, list[str]] = {} - for ctrl, variant_urn in rows: - if ctrl.id not in controls: - controls[ctrl.id] = ctrl - urns_by_control[ctrl.id] = [] - urns_by_control[ctrl.id].append(variant_urn) - if not controls: logger.info( msg="No clinical control variants matching the provided filters are associated with the requested score set.", @@ -2542,10 +2514,10 @@ async def get_clinical_controls_for_score_set( "db_name": ctrl.db_name, "modification_date": ctrl.modification_date, "creation_date": ctrl.creation_date, - "mapped_variants": [{"variant_urn": v_urn} for v_urn in urns_by_control[ctrl.id]], + "mapped_variants": [{"variant_urn": v_urn} for v_urn in variant_urns], } ) - for ctrl in controls.values() + for ctrl, variant_urns in controls ] @@ -2592,29 +2564,9 @@ async def get_clinical_controls_options_for_score_set( assert_permission(user_data, item, Action.READ) - clinical_controls_query = ( - select(ClinvarControl.db_name, ClinvarControl.db_version) - .join(ClinvarAlleleLink, ClinvarAlleleLink.clinvar_control_id == ClinvarControl.id) - .join(Allele, Allele.id == ClinvarAlleleLink.allele_id) - .join(MappingRecordAllele, MappingRecordAllele.allele_id == Allele.id) - .join(MappingRecord, MappingRecord.id == MappingRecordAllele.mapping_record_id) - .join(Variant, Variant.id == MappingRecord.variant_id) - .where(ClinvarAlleleLink.live_at(as_of)) - .where(MappingRecordAllele.live_at(as_of)) - .where(MappingRecord.live_at(as_of)) - .where(Variant.score_set_id == item.id) - .distinct() - ) - - clinical_controls_for_item = db.execute(clinical_controls_query) + options = get_clinical_control_options(db, item.id, as_of=as_of) - # NOTE: We return options even for pairwise groupings which may have no associated mapped variants - # and 404 when ultimately requested together. - clinical_control_options: dict[str, list[str]] = {} - for db_name, db_version in clinical_controls_for_item: - clinical_control_options.setdefault(db_name, []).append(db_version) - - if not clinical_control_options: + if not options: logger.info( msg="Failed to fetch clinical control options for score set; No clinical control variants are associated with this score set.", extra=logging_context(), @@ -2624,17 +2576,7 @@ async def get_clinical_controls_options_for_score_set( detail=f"no clinical control variants associated with score set URN {urn} were found", ) - def _version_sort_key(v: str) -> tuple[int, int]: - try: - month, year = v.split("_") - return int(year), int(month) - except (ValueError, AttributeError): - return (0, 0) - - return [ - {"db_name": db_name, "available_versions": sorted(db_versions, key=_version_sort_key, reverse=True)} - for db_name, db_versions in clinical_control_options.items() - ] + return [{"db_name": db_name, "available_versions": available_versions} for db_name, available_versions in options] @router.get( diff --git a/tests/lib/test_clinical_controls.py b/tests/lib/test_clinical_controls.py new file mode 100644 index 000000000..194ff68bd --- /dev/null +++ b/tests/lib/test_clinical_controls.py @@ -0,0 +1,32 @@ +"""Unit tests for the clinical-controls lib helpers (``lib/clinical_controls.py``). + +The live-link query paths are exercised end-to-end by the score-set router tests; these pin the pure +``clinvar_version_sort_key``, which is the ``MM_YYYY`` release-ordering contract the UI parses in +parallel (``lib/clinical-controls.ts``) and so must stay stable. +""" + +from mavedb.lib.clinical_controls import clinvar_version_sort_key + + +def test_version_sort_key_orders_by_year_then_month(): + versions = ["01_2024", "12_2023", "06_2024", "03_2025"] + assert sorted(versions, key=clinvar_version_sort_key, reverse=True) == [ + "03_2025", + "06_2024", + "01_2024", + "12_2023", + ] + + +def test_version_sort_key_parses_month_and_year(): + # (year, month) — a later month within the same year sorts higher. + assert clinvar_version_sort_key("06_2024") == (2024, 6) + assert clinvar_version_sort_key("11_2024") > clinvar_version_sort_key("06_2024") + + +def test_version_sort_key_unparseable_sorts_to_bottom(): + # Malformed versions collapse to (0, 0) rather than raising, so they sink under any real release. + assert clinvar_version_sort_key("garbage") == (0, 0) + assert clinvar_version_sort_key("2024") == (0, 0) + assert clinvar_version_sort_key("13_20_2024") == (0, 0) + assert clinvar_version_sort_key("01_2020") > clinvar_version_sort_key("garbage") From 3717f5d823f779b0ace865853df17977e069d7f1 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 7 Jul 2026 14:10:54 -0700 Subject: [PATCH 065/252] refactor(variants): rename supersededBy to supersededByScoreSet on variant detail The field carries the superseding *score set*'s URN, not a variant URN, so the old name read as if a consumer could jump straight to a superseding variant. Supersession is versioned at the score-set level and a newer version may add, drop, or renumber variants, so no stable superseding-variant pointer exists; document that alongside the rename. Also modernize get_variant's fetch to a 2.0-style select (preserving the MultipleResultsFound handling). --- src/mavedb/lib/variant_detail.py | 14 +++++++++----- src/mavedb/routers/variants.py | 4 ++-- src/mavedb/view_models/variant_detail.py | 8 ++++++-- tests/lib/test_variant_detail.py | 6 +++--- tests/routers/test_variant.py | 4 ++-- 5 files changed, 22 insertions(+), 14 deletions(-) diff --git a/src/mavedb/lib/variant_detail.py b/src/mavedb/lib/variant_detail.py index 09e48a072..3ea10d000 100644 --- a/src/mavedb/lib/variant_detail.py +++ b/src/mavedb/lib/variant_detail.py @@ -5,8 +5,8 @@ on the fly by :mod:`lib.cat_vrs`) and, riding alongside it keyed by VRS digest, the MaveDB layer — per-member relations and the digest-keyed external-annotation map (:mod:`lib.annotations`). Also carries the per-calibration functional ``classifications`` the variant falls into, and its version -standing (``is_current`` / ``superseded_by``) so a superseded variant self-describes rather than -reading as current (design §9.2). +standing (``is_current`` / ``superseded_by_score_set``) so a superseded variant self-describes rather +than reading as current (design §9.2). Temporal scope of ``as_of``: only the **molecular** layer is versioned — Cat-VRS membership and the VEP/gnomAD/ClinVar annotations reconstruct at the past instant. Scores are immutable (``Variant`` is @@ -77,7 +77,11 @@ class VariantDetail: # Version standing — self-descriptive for a superseded variant. is_current: bool - superseded_by: Optional[str] # URN of the superseding variant's score set, if any (and readable) + # URN of the score-set version that supersedes this variant's, if any (and readable). This is a + # *score set* URN, not a variant URN — supersession is versioned at the score-set level, and a newer + # version may add/drop/renumber variants, so there is no stable superseding-variant to point at; + # consumers resolve the current measurement by looking this variant up within that score set. + superseded_by_score_set: Optional[str] def _classifications_for_variant( @@ -126,7 +130,7 @@ def get_variant_detail( ``superseding_score_set`` is the newer version the caller has already resolved for visibility (blanked when the user cannot read it, mirroring ``fetch_score_set_by_urn``); its presence drives - ``is_current``/``superseded_by``. ``visible_calibration_ids`` restricts classifications to + ``is_current``/``superseded_by_score_set``. ``visible_calibration_ids`` restricts classifications to readable calibrations. ``as_of`` reconstructs the molecular layer (Cat-VRS membership + annotations); scores and classifications are as-of-invariant. See the module docstring. """ @@ -176,5 +180,5 @@ def get_variant_detail( member_relations=member_relations, annotations=annotations, is_current=superseding_score_set is None, - superseded_by=superseding_score_set.urn if superseding_score_set is not None else None, + superseded_by_score_set=superseding_score_set.urn if superseding_score_set is not None else None, ) diff --git a/src/mavedb/routers/variants.py b/src/mavedb/routers/variants.py index 97352a8ec..66f302df6 100644 --- a/src/mavedb/routers/variants.py +++ b/src/mavedb/routers/variants.py @@ -465,12 +465,12 @@ def get_variant( Flat assay-level fields for the common UI case plus the spec-pure GA4GH CategoricalVariant and a digest-keyed annotation map for machine/standard consumers. A superseded variant is served (it is - the citable unit) but self-describes via isCurrent/supersededBy rather than reading as current. + the citable unit) but self-describes via isCurrent/supersededByScoreSet rather than reading as current. """ save_to_logging_context({"requested_resource": urn, "as_of": as_of}) response.headers["X-As-Of"] = as_of.isoformat() if as_of is not None else "current" try: - variant = db.query(Variant).filter(Variant.urn == urn).one_or_none() + variant = db.scalars(select(Variant).where(Variant.urn == urn)).one_or_none() except MultipleResultsFound: logger.info(msg="Could not fetch the requested variant; Multiple such variants exist.", extra=logging_context()) raise HTTPException(status_code=500, detail=f"multiple variants with URN '{urn}' were found") diff --git a/src/mavedb/view_models/variant_detail.py b/src/mavedb/view_models/variant_detail.py index 18a78f844..2c242f756 100644 --- a/src/mavedb/view_models/variant_detail.py +++ b/src/mavedb/view_models/variant_detail.py @@ -36,7 +36,11 @@ class VariantDetail(BaseModel): is a client-side toggle, no refetch) plus the spec-pure GA4GH ``molecularRepresentation`` (``CategoricalVariant``, no MaveDB fields inside). The MaveDB layer rides alongside keyed by VRS digest: ``memberRelations`` (member→defining relation) and the ``annotations`` map. ``isCurrent`` - /``supersededBy`` let a superseded variant self-describe. Absent fields are omitted. + /``supersededByScoreSet`` let a superseded variant self-describe: ``supersededByScoreSet`` is the + superseding *score set*'s URN, not a variant URN. Supersession is versioned at the score-set level, + and a newer version may add, drop, or renumber variants — so there is no stable + superseding-*variant* pointer to hand back; a consumer resolves the current measurement by looking + this variant up within that score set. Absent fields are omitted. """ urn: str @@ -57,7 +61,7 @@ class VariantDetail(BaseModel): annotations: dict[str, AlleleAnnotations] = {} is_current: bool - superseded_by: Optional[str] = None + superseded_by_score_set: Optional[str] = None class Config: from_attributes = True diff --git a/tests/lib/test_variant_detail.py b/tests/lib/test_variant_detail.py index 61594e4f0..6cea44858 100644 --- a/tests/lib/test_variant_detail.py +++ b/tests/lib/test_variant_detail.py @@ -5,7 +5,7 @@ off the live record + authoritative allele; the spec-pure Cat-VRS + MaveDB mode/member-relations off the on-the-fly builder; the digest-keyed annotation map; the per-calibration classifications (primary first, filterable by visibility); scores/counts passthrough; and the ``is_current`` / -``superseded_by`` version standing. Cat-VRS/annotation internals are covered in their own suites — here +``superseded_by_score_set`` version standing. Cat-VRS/annotation internals are covered in their own suites — here we assert they are wired in and keyed correctly. """ @@ -159,7 +159,7 @@ def test_full_envelope_for_a_coding_assay(session, setup_lib_db_with_score_set): assert detail.annotations["cdna-digest"].vep is not None assert detail.annotations["cdna-digest"].vep.consequence == "missense_variant" assert detail.is_current is True - assert detail.superseded_by is None + assert detail.superseded_by_score_set is None @pytest.mark.integration @@ -245,4 +245,4 @@ def test_superseded_variant_self_describes(session, setup_lib_db_with_score_set) detail = get_variant_detail(session, variant, superseding_score_set=newer) assert detail.is_current is False - assert detail.superseded_by == "urn:mavedb:00000001-a-2" + assert detail.superseded_by_score_set == "urn:mavedb:00000001-a-2" diff --git a/tests/routers/test_variant.py b/tests/routers/test_variant.py index 68b48cecc..365fe9ca7 100644 --- a/tests/routers/test_variant.py +++ b/tests/routers/test_variant.py @@ -109,7 +109,7 @@ def test_get_variant_detail_envelope(client, session, data_provider, data_files, assert body["memberRelations"] == {"prot-digest": "translation_of"} assert body["annotations"]["cdna-digest"]["vep"]["consequence"] == "missense_variant" assert body["isCurrent"] is True - assert "supersededBy" not in body # dropped by exclude_none when current + assert "supersededByScoreSet" not in body # dropped by exclude_none when current assert response.headers["X-As-Of"] == "current" @@ -154,7 +154,7 @@ def test_superseded_variant_self_describes(client, session, data_provider, data_ assert response.status_code == 200 body = response.json() assert body["isCurrent"] is False - assert body["supersededBy"] == newer["urn"] + assert body["supersededByScoreSet"] == newer["urn"] def test_get_variant_detail_anonymous_cannot_read_private( From ac8f4052d03fc4b456aaba8884c7f8fabd439d95 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 7 Jul 2026 14:11:03 -0700 Subject: [PATCH 066/252] fix(mapping): enforce one live authoritative allele link per record The lean whole-set view joins the authoritative MappingRecordAllele expecting it 1:1 with the variant. The existing (mapping_record, allele) unique index cannot stop two different alleles both being flagged authoritative for one record, so a second live authoritative link would silently duplicate the variant and double-count its score rather than raise. Add a partial unique index on (mapping_record_id) WHERE is_authoritative AND valid_to IS NULL so the mistake fails loud in the mapping job at write time. Detection query is clean on dev data. --- ...ique_live_authoritative_link_per_record.py | 46 +++++++++++++++++++ src/mavedb/models/mapping_record_allele.py | 14 ++++++ 2 files changed, 60 insertions(+) create mode 100644 alembic/versions/f1a3c7e9b2d5_unique_live_authoritative_link_per_record.py diff --git a/alembic/versions/f1a3c7e9b2d5_unique_live_authoritative_link_per_record.py b/alembic/versions/f1a3c7e9b2d5_unique_live_authoritative_link_per_record.py new file mode 100644 index 000000000..6d7b6c989 --- /dev/null +++ b/alembic/versions/f1a3c7e9b2d5_unique_live_authoritative_link_per_record.py @@ -0,0 +1,46 @@ +"""enforce one live authoritative link per mapping record + +Revision ID: f1a3c7e9b2d5 +Revises: c9b2a4f8e1d3 +Create Date: 2026-07-07 + +Adds a partial unique index enforcing at most one live authoritative link per mapping record. The +existing uq_mapping_record_alleles_live keys on (mapping_record, allele) and so cannot prevent two +different alleles both being flagged is_authoritative for the same record. The serving layer depends +on there being exactly one: the lean whole-set view (lib/score_set_variants) joins the authoritative +link expecting it 1:1 with the variant, so a second live authoritative link would silently duplicate +the variant and double-count its score in the histogram rather than raise. This index moves that +failure to write time in the mapping job, where the bug actually lives. + +Assumes no pre-existing record with two live authoritative links — the invariant is upheld today by +the mapping job's write logic. If this ever runs against data that violates it, the unique index +creation fails and the offending duplicates must be retired first. To check before applying: + + SELECT mapping_record_id, count(*) + FROM mapping_record_alleles + WHERE is_authoritative AND valid_to IS NULL + GROUP BY mapping_record_id HAVING count(*) > 1; +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "f1a3c7e9b2d5" +down_revision = "c9b2a4f8e1d3" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_index( + "uq_mapping_record_alleles_live_authoritative", + "mapping_record_alleles", + ["mapping_record_id"], + unique=True, + postgresql_where=sa.text("is_authoritative AND valid_to IS NULL"), + ) + + +def downgrade() -> None: + op.drop_index("uq_mapping_record_alleles_live_authoritative", table_name="mapping_record_alleles") diff --git a/src/mavedb/models/mapping_record_allele.py b/src/mavedb/models/mapping_record_allele.py index cda8bf0f3..7bfbd7885 100644 --- a/src/mavedb/models/mapping_record_allele.py +++ b/src/mavedb/models/mapping_record_allele.py @@ -57,4 +57,18 @@ class MappingRecordAllele(ValidTime, Base): unique=True, postgresql_where=text("valid_to IS NULL"), ), + # At most one live *authoritative* link per mapping record. The index above keys on + # (record, allele) and so cannot stop two different alleles both being flagged authoritative + # for one record; this one enforces the "one authoritative (measured) allele per record" + # invariant directly. It is the backstop the serving layer relies on: the lean whole-set view + # (lib/score_set_variants) inner-joins the authoritative link expecting it 1:1 with the + # variant, so a second live authoritative link would silently duplicate the variant (and + # double-count its score) rather than raise. With this index, that mistake fails loud in the + # mapping job at write time instead. + Index( + "uq_mapping_record_alleles_live_authoritative", + "mapping_record_id", + unique=True, + postgresql_where=text("is_authoritative AND valid_to IS NULL"), + ), ) From 7b92d6936fe1551bf20e0a34603efc595dfd49fc Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 7 Jul 2026 22:33:40 -0700 Subject: [PATCH 067/252] refactor(calibrations): extract shared calibration-preference and evidence helpers Calibration selection had diverged across the serving layer (variant_detail ordered by primary/id; the new allele-measurements list used a fuller cascade). Add two pure helpers to lib/score_calibrations: - calibration_preference_key: the UI's default-calibration cascade (primary -> investigator_provided -> non-research-use-only), so the API and UI agree on the default calibration for a score set. - classification_evidence_strength: the (magnitude, direction) evidence primitive (ACMG points -> |ln(oddspath)| -> functional call). Repoint variant_detail's classification ordering onto the shared key so the variant page's first-listed classification matches the measurement card and the UI default. Annotate ScoreCalibration.id/primary as Mapped for the typed helpers. --- src/mavedb/lib/score_calibrations.py | 51 ++++++++++++++++++++++++++ src/mavedb/lib/variant_detail.py | 23 ++++++++---- src/mavedb/models/score_calibration.py | 4 +- 3 files changed, 68 insertions(+), 10 deletions(-) diff --git a/src/mavedb/lib/score_calibrations.py b/src/mavedb/lib/score_calibrations.py index f7b84580f..25608df7c 100644 --- a/src/mavedb/lib/score_calibrations.py +++ b/src/mavedb/lib/score_calibrations.py @@ -18,6 +18,7 @@ ) from mavedb.lib.validation.utilities import inf_or_float from mavedb.lib.variants import score_from_variant_data +from mavedb.models.enums.functional_classification import FunctionalClassification from mavedb.models.enums.score_calibration_relation import ScoreCalibrationRelation from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification @@ -28,6 +29,56 @@ from mavedb.view_models import score_calibration +def calibration_preference_key(calibration: ScoreCalibration) -> tuple[int, int, int]: + """The UI's default-calibration preference cascade as a sort key (lower sorts first = more preferred): + ``primary``, then ``investigator_provided``, then non-``research_use_only``. Mirrors the UI's + ``activeCalibrationOptions`` default selection, so the API and UI agree on which calibration is "the + default" for a score set. + + Intentionally stops at the cascade tiers and carries **no** evidence strength (a variant-specific + concern) and **no** id — callers append their own deterministic tiebreak (calibration id or urn), and, where + a within-tier strongest-evidence tiebreak is wanted, :func:`classification_evidence_strength`. + """ + return ( + 0 if calibration.primary else 1, + 0 if calibration.investigator_provided else 1, + 0 if not calibration.research_use_only else 1, + ) + + +def classification_evidence_strength( + classification: Optional[ScoreCalibrationFunctionalClassification], +) -> tuple[float, int]: + """``(magnitude, direction)`` of a functional classification's evidence, for ranking. + + ``magnitude`` is the evidence strength (larger = stronger); ``direction`` biases toward pathogenic + (0 = pathogenic, 1 = benign, 2 = unknown). ACMG points are the primary signal (signed: + positive = pathogenic), falling back to ``|ln(oddspaths_ratio)|`` then the functional call + (abnormal → pathogenic, normal → benign) for direction only. + """ + if classification is None: + return (0.0, 2) + + acmg = classification.acmg_classification + if acmg is not None and acmg.points is not None: + points = float(acmg.points) + return (abs(points), 0 if points > 0 else 1 if points < 0 else 2) + + # OddsPath is a ratio centered at 1 (pathogenic >1, benign <1); ln maps it to a symmetric log-odds + # strength around 0 (x and 1/x are equal-and-opposite), comparable across direction and commensurate + # with the already-log-scaled ACMG points. + ratio = classification.oddspaths_ratio + if ratio is not None and ratio > 0: + return (abs(math.log(ratio)), 0 if ratio > 1 else 1 if ratio < 1 else 2) + + if classification.functional_classification == FunctionalClassification.abnormal: + return (0.0, 0) + elif classification.functional_classification == FunctionalClassification.normal: + return (0.0, 1) + + return (0.0, 2) + + def create_functional_classification( db: Session, functional_range_create: Union[ diff --git a/src/mavedb/lib/variant_detail.py b/src/mavedb/lib/variant_detail.py index 3ea10d000..0331c7d7a 100644 --- a/src/mavedb/lib/variant_detail.py +++ b/src/mavedb/lib/variant_detail.py @@ -25,6 +25,7 @@ from mavedb.lib.allele_annotations import AlleleAnnotations, get_allele_annotations from mavedb.lib.alleles import get_live_record_allele_links from mavedb.lib.cat_vrs import categorical_variant_for_variant +from mavedb.lib.score_calibrations import calibration_preference_key from mavedb.models.enums.annotation_layer import AnnotationLayer from mavedb.models.mapping_record import MappingRecord from mavedb.models.score_calibration import ScoreCalibration @@ -88,31 +89,37 @@ def _classifications_for_variant( db: Session, variant: Variant, *, visible_calibration_ids: Optional[set[int]] ) -> list[VariantClassificationRecord]: """The functional classifications the variant belongs to (membership is materialized on the - calibration→classification→variant m2m). One row per calibration that classifies the variant; - the primary calibration sorts first. ``visible_calibration_ids`` restricts to calibrations the - caller resolved as readable (``None`` = no restriction, for lib-level use).""" + calibration→classification→variant m2m). One row per calibration that classifies the variant, ordered + by the shared calibration preference cascade (``calibration_preference_key`` + id) so the first entry + is the same default the UI and the allele-measurements card surface. ``visible_calibration_ids`` + restricts to calibrations the caller resolved as readable (``None`` = no restriction, for lib-level use).""" statement = ( - select(ScoreCalibrationFunctionalClassification, ScoreCalibration.primary, ScoreCalibration.id) + select(ScoreCalibrationFunctionalClassification, ScoreCalibration) .join( classification_variants, classification_variants.c.functional_classification_id == ScoreCalibrationFunctionalClassification.id, ) .join(ScoreCalibration, ScoreCalibration.id == ScoreCalibrationFunctionalClassification.calibration_id) .where(classification_variants.c.variant_id == variant.id) - .order_by(ScoreCalibration.primary.desc(), ScoreCalibration.id) ) if visible_calibration_ids is not None: statement = statement.where(ScoreCalibration.id.in_(visible_calibration_ids)) + rows = sorted( + db.execute(statement).tuples().all(), key=lambda row: (*calibration_preference_key(row[1]), row[1].id) + ) return [ - VariantClassificationRecord(calibration_id=calibration_id, primary=primary, classification=classification) - for classification, primary, calibration_id in db.execute(statement) + VariantClassificationRecord( + calibration_id=calibration.id, primary=calibration.primary, classification=classification + ) + for classification, calibration in rows ] def _submitted_assay_level_hgvs(variant: Variant, assay_level: Optional[str]) -> Optional[str]: """The depositor-submitted HGVS in the variant's assay frame: protein for a protein assay, - otherwise the nucleotide expression (genomic or coding share ``hgvs_nt``).""" + otherwise the nucleotide expression (genomic or coding share ``hgvs_nt`` and ``hgvs_splice`` is + never an index column).""" if assay_level == AnnotationLayer.protein.value: return variant.hgvs_pro return variant.hgvs_nt diff --git a/src/mavedb/models/score_calibration.py b/src/mavedb/models/score_calibration.py index 38ce1f286..1941cb863 100644 --- a/src/mavedb/models/score_calibration.py +++ b/src/mavedb/models/score_calibration.py @@ -25,7 +25,7 @@ class ScoreCalibration(Base): __tablename__ = "score_calibrations" # TODO#544: Add a partial unique index to enforce only one primary calibration per score set. - id = Column(Integer, primary_key=True) + id: Mapped[int] = Column(Integer, primary_key=True) urn = Column(String(64), nullable=True, default=generate_calibration_urn, unique=True, index=True) score_set_id = Column(Integer, ForeignKey("scoresets.id"), nullable=False) @@ -33,7 +33,7 @@ class ScoreCalibration(Base): title = Column(String, nullable=False) research_use_only = Column(Boolean, nullable=False, default=False) - primary = Column(Boolean, nullable=False, default=False) + primary: Mapped[bool] = Column(Boolean, nullable=False, default=False) investigator_provided: Mapped[bool] = Column(Boolean, nullable=False, default=False) private = Column(Boolean, nullable=False, default=True) notes = Column(String, nullable=True) From e47791c0d52922e3c9696f3271193b31f71677b8 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 7 Jul 2026 22:34:12 -0700 Subject: [PATCH 068/252] feat(clingen-alleles): add equivalence-class measurements endpoint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GET /clingen-alleles/{caid}/measurements — the ClinGen-allele-centric variant page's entrypoint. Resolves a CA/PA to its anchor alleles and lists every measurement whose cross-layer equivalence class (MappingRecordAllele co-membership) touches it, not just the exact allele. Each card is labeled by its assayed level and its relationship to the query (direct / protein_consequence / nucleotide_encoding); the split falls out of which allele anchors the co-membership. Carries the primary readable classification inline, orders direct-first then by strongest pathogenic-biased evidence, and opts into superseded measurements and as_of. Authorization is has_permission in the lib (score-set READ gates inclusion, calibration READ gates the inline classification). --- src/mavedb/lib/allele_measurements.py | 234 ++++++++++++ src/mavedb/routers/clingen_alleles.py | 85 +++++ src/mavedb/server_main.py | 3 + src/mavedb/view_models/allele_measurement.py | 41 ++ tests/lib/test_allele_measurements.py | 371 +++++++++++++++++++ tests/routers/test_clingen_allele.py | 122 ++++++ 6 files changed, 856 insertions(+) create mode 100644 src/mavedb/lib/allele_measurements.py create mode 100644 src/mavedb/routers/clingen_alleles.py create mode 100644 src/mavedb/view_models/allele_measurement.py create mode 100644 tests/lib/test_allele_measurements.py create mode 100644 tests/routers/test_clingen_allele.py diff --git a/src/mavedb/lib/allele_measurements.py b/src/mavedb/lib/allele_measurements.py new file mode 100644 index 000000000..eabb414f5 --- /dev/null +++ b/src/mavedb/lib/allele_measurements.py @@ -0,0 +1,234 @@ +"""The ClinGen-allele-centric variant page's measurements list — ``GET /clingen-alleles/{caid}/measurements``. + +Given a ClinGen allele id (a nucleotide ``CA`` or a protein ``PA``), list every measurement whose +**cross-layer equivalence class** touches that allele — not just the exact allele. The equivalence +relation is co-membership in a ``MappingRecord``'s allele set (the mapping + reverse-translation graph, +:mod:`lib.alleles`): a ``CA`` anchors on its nucleotide alleles (genomic + coding share the CA), so the +list gathers the nt measurements of that change (**direct**) *and* the protein measurements of its +consequence (**related**); a ``PA`` anchors on the protein allele, gathering the protein measurements of +that change (**direct**) *and* every nt measurement that encodes it (**related**). The asymmetry falls +out of *which* allele anchors the co-membership — a sibling nt change that also encodes the same protein +is not pulled onto a CA page, because its record links the protein but not the anchor nt allele. See +design §7.5. + +Distinct from :func:`lib.variant_detail.get_variant_detail`, which is the *record-scoped*, all-levels +detail of one *selected* measurement. This list is the *cross-record* union — who else measured +something in the equivalence class. + +Authorization uses ``has_permission`` directly (matching ``lib/score_sets.py``): ``user_data`` is checked +per entity — score-set READ gates whether a measurement appears at all (a private score set's measurement +never leaks), and calibration READ gates the inline classification (withheld while the measurement still +shows). The two-gate split is deliberate: a readable measurement can carry an unreadable classification. +``as_of`` reconstructs the molecular layer (which records/links are live) at a past instant; scores are +immutable and calibrations carry no valid-time, so both are as-of-invariant. +""" + +from dataclasses import dataclass +from datetime import date, datetime +from enum import Enum +from typing import Optional + +from sqlalchemy import and_, select +from sqlalchemy.orm import Session, aliased, joinedload + +from mavedb.lib.permissions import Action, has_permission +from mavedb.lib.score_calibrations import calibration_preference_key, classification_evidence_strength +from mavedb.lib.types.authentication import UserData +from mavedb.lib.variants import variant_score +from mavedb.models.allele import Allele +from mavedb.models.enums.annotation_layer import AnnotationLayer +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.score_calibration import ScoreCalibration +from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification +from mavedb.models.score_calibration_functional_classification_variant_association import ( + score_calibration_functional_classification_variants_association_table as classification_variants, +) +from mavedb.models.variant import Variant + + +class MeasurementRelationship(str, Enum): + """The relationship of a measurement to the queried ClinGen allele. ``direct`` = the measurement was + assayed *at* this allele; the other two are the RT-related measurements, named for how they relate + to the query. + """ + + direct = "direct" # the measurement was assayed *at* this allele + protein_consequence = "protein_consequence" # CA query, protein measurement of consequence(N) + nucleotide_encoding = "nucleotide_encoding" # PA query, nt measurement encoding P + + +@dataclass(frozen=True) +class AlleleMeasurement: + """One measurement in the queried allele's equivalence class (transit; serialized by + ``view_models.allele_measurement``). + + ``assay_level`` is the level at which *this* measurement was assayed (protein/cdna/genomic) — the + clinically load-bearing fact, always shown. ``relationship`` says how it relates to the queried + ClinGen id. ``primary_classification`` is the primary readable functional classification (``None`` + when the score set has none or the caller cannot read the calibration). + """ + + variant_urn: str + score: Optional[float] + assay_level: Optional[str] + relationship: str + assay_level_hgvs: Optional[str] + submitted_hgvs: Optional[str] + score_set_urn: str + score_set_title: str + primary_classification: Optional[ScoreCalibrationFunctionalClassification] + is_current: bool + superseded_by_score_set: Optional[str] + + +def _preferred_classification( + db: Session, variant: Variant, *, user_data: Optional[UserData] +) -> Optional[ScoreCalibrationFunctionalClassification]: + """The variant's preferred *readable* functional classification, or ``None``. + + A variant falls into one classification per calibration (non-overlapping ranges); we surface the one + the UI would default to, mirroring its calibration preference cascade: ``primary`` first, then + ``investigator_provided``, then non-``research_use_only``. Within a tier we take the strongest + evidence (the VA-Spec posture — the strongest calibration is the evidence calibration), then ``id`` + only for determinism. The UI's "any with ranges / any calibration" tail steps collapse here: this + query only surfaces calibrations that already classify the variant. Only calibrations the caller can + read (calibration READ) are considered, so a private calibration is skipped rather than blanking a + readable call. Kept in sync with the UI's `activeCalibrationOptions` default selection. + """ + candidates = [ + (classification, calibration) + for classification, calibration in db.execute( + select(ScoreCalibrationFunctionalClassification, ScoreCalibration) + .join( + classification_variants, + classification_variants.c.functional_classification_id == ScoreCalibrationFunctionalClassification.id, + ) + .join(ScoreCalibration, ScoreCalibration.id == ScoreCalibrationFunctionalClassification.calibration_id) + .where(classification_variants.c.variant_id == variant.id) + ).all() + if has_permission(user_data, calibration, Action.READ).permitted + ] + if not candidates: + return None + + def preference(pair: tuple[ScoreCalibrationFunctionalClassification, ScoreCalibration]) -> tuple: + classification, calibration = pair + magnitude, _ = classification_evidence_strength(classification) + return ( + *calibration_preference_key(calibration), + -magnitude, + calibration.id, + ) + + return min(candidates, key=preference)[0] + + +def _ordering_key(measurement: AlleleMeasurement, published_date: Optional[date]) -> tuple: + """Sort order: + 1. Direct measurements, then related (protein_consequence / nucleotide_encoding) + 2. Within each, the strongest evidence first (pathogenic wins ties) + 3. Within each, the newest-published first + 4. Within each, the URN for a stable tiebreak + """ + magnitude, direction = classification_evidence_strength(measurement.primary_classification) + return ( + 0 if measurement.relationship == MeasurementRelationship.direct else 1, + 0 if measurement.primary_classification is not None else 1, + -magnitude, + direction, + -(published_date.toordinal()) if published_date is not None else 0, + measurement.variant_urn, + ) + + +def get_allele_measurements( + db: Session, + clingen_allele_id: str, + *, + user_data: Optional[UserData], + include_superseded: bool = False, + as_of: Optional[datetime] = None, +) -> list[AlleleMeasurement]: + """List the measurements in ``clingen_allele_id``'s cross-layer equivalence class. Returns + ``[]`` when the id resolves to no allele or no live record. + """ + anchor = db.execute(select(Allele.id, Allele.level).where(Allele.clingen_allele_id == clingen_allele_id)).all() + if not anchor: + return [] + + anchor_ids = [row.id for row in anchor] + entry_is_protein = any(row.level == AnnotationLayer.protein.value for row in anchor) + + record_ids = db.scalars( + select(MappingRecordAllele.mapping_record_id) + .where(MappingRecordAllele.allele_id.in_(anchor_ids)) + .where(MappingRecordAllele.live_at(as_of)) + .distinct() + ).all() + if not record_ids: + return [] + + # Each record's authoritative (measured) allele fixes the assayed level and the direct/related call. + authoritative = aliased(MappingRecordAllele) + rows = ( + db.execute( + select(MappingRecord, Variant, Allele) + .join(Variant, Variant.id == MappingRecord.variant_id) + .join( + authoritative, + and_( + authoritative.mapping_record_id == MappingRecord.id, + authoritative.is_authoritative.is_(True), + authoritative.live_at(as_of), + ), + ) + .join(Allele, Allele.id == authoritative.allele_id) + .where(MappingRecord.id.in_(record_ids)) + .where(MappingRecord.live_at(as_of)) + .options(joinedload(Variant.score_set)) + ) + .tuples() + .all() + ) + + measurements: list[tuple[AlleleMeasurement, Optional[date]]] = [] + for record, variant, measured_allele in rows: + score_set = variant.score_set + if not has_permission(user_data, score_set, Action.READ).permitted: + continue + + # Don't leak unpermitted resources. + superseding = score_set.superseding_score_set + if superseding is not None and not has_permission(user_data, superseding, Action.READ).permitted: + superseding = None + + is_current = superseding is None + if not is_current and not include_superseded: + continue + + if measured_allele.clingen_allele_id == clingen_allele_id: + relationship = MeasurementRelationship.direct + elif entry_is_protein: + relationship = MeasurementRelationship.nucleotide_encoding + else: + relationship = MeasurementRelationship.protein_consequence + + assay_level = record.assay_level + measurement = AlleleMeasurement( + variant_urn=variant.urn or "", + score=variant_score(variant), + assay_level=assay_level, + relationship=relationship, + assay_level_hgvs=record.hgvs_assay_level, + submitted_hgvs=variant.hgvs_pro if assay_level == AnnotationLayer.protein.value else variant.hgvs_nt, + score_set_urn=score_set.urn or "", + score_set_title=score_set.title or "", + primary_classification=_preferred_classification(db, variant, user_data=user_data), + is_current=is_current, + superseded_by_score_set=superseding.urn if superseding is not None else None, + ) + measurements.append((measurement, score_set.published_date)) + + measurements.sort(key=lambda pair: _ordering_key(pair[0], pair[1])) + return [measurement for measurement, _ in measurements] diff --git a/src/mavedb/routers/clingen_alleles.py b/src/mavedb/routers/clingen_alleles.py new file mode 100644 index 000000000..b3b626c02 --- /dev/null +++ b/src/mavedb/routers/clingen_alleles.py @@ -0,0 +1,85 @@ +import logging +from datetime import datetime +from typing import Optional + +from fastapi import APIRouter, Depends, Query, Response +from sqlalchemy.orm import Session + +from mavedb import deps +from mavedb.lib.allele_measurements import get_allele_measurements +from mavedb.lib.authentication import get_current_user +from mavedb.lib.logging import LoggedRoute +from mavedb.lib.logging.context import save_to_logging_context +from mavedb.lib.types.authentication import UserData +from mavedb.routers.shared import ACCESS_CONTROL_ERROR_RESPONSES, PUBLIC_ERROR_RESPONSES, ROUTER_BASE_PREFIX +from mavedb.view_models.allele_measurement import AlleleMeasurement + +TAG_NAME = "ClinGen alleles" + +logger = logging.getLogger(__name__) + +router = APIRouter( + prefix=f"{ROUTER_BASE_PREFIX}/clingen-alleles", + tags=[TAG_NAME], + responses={**PUBLIC_ERROR_RESPONSES}, + route_class=LoggedRoute, +) + +metadata = { + "name": TAG_NAME, + "description": "The ClinGen-allele-centric variant page: measurements across a variant's equivalence class.", +} + + +@router.get( + "/{clingen_allele_id}/measurements", + status_code=200, + response_model=list[AlleleMeasurement], + responses={**ACCESS_CONTROL_ERROR_RESPONSES}, + response_model_exclude_none=True, + summary="List measurements for a ClinGen allele's equivalence class", +) +def get_clingen_allele_measurements( + *, + clingen_allele_id: str, + response: Response, + include_superseded: bool = Query( + default=False, + description=( + "Include measurements from superseded score-set versions. Default false — superseded " + "measurements are a deliberate power-user / citation path, never surfaced by discovery." + ), + ), + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the equivalence class (which mapping records / allele links are live) as it " + "stood at this instant. ISO 8601, ideally timezone-aware. Scores and classifications are " + "as-of-invariant. Defaults to current." + ), + ), + db: Session = Depends(deps.get_db), + user_data: Optional[UserData] = Depends(get_current_user), +): + """List every measurement whose cross-layer equivalence class touches this ClinGen allele (a ``CA`` or + ``PA``) — the direct measurements assayed at this change plus the reverse-translation-related ones, + each labeled by its assayed level and relationship. This is the ClinGen-allele-centric variant page's + entrypoint. A private score set's measurement is never included; its inline classification is withheld + where the calibration is unreadable while the measurement still shows. + """ + save_to_logging_context( + { + "requested_resource": clingen_allele_id, + "as_of": as_of, + "include_superseded": include_superseded, + } + ) + response.headers["X-As-Of"] = as_of.isoformat() if as_of is not None else "current" + + return get_allele_measurements( + db, + clingen_allele_id, + user_data=user_data, + include_superseded=include_superseded, + as_of=as_of, + ) diff --git a/src/mavedb/server_main.py b/src/mavedb/server_main.py index 150714883..e79819999 100644 --- a/src/mavedb/server_main.py +++ b/src/mavedb/server_main.py @@ -41,6 +41,7 @@ access_keys, alphafold, api_information, + clingen_alleles, collections, controlled_keywords, doi_identifiers, @@ -93,6 +94,7 @@ app.add_middleware(GZipMiddleware, minimum_size=1000, compresslevel=5) app.include_router(access_keys.router) app.include_router(api_information.router) +app.include_router(clingen_alleles.router) app.include_router(collections.router) app.include_router(controlled_keywords.router) app.include_router(doi_identifiers.router) @@ -241,6 +243,7 @@ def customize_openapi_schema(): openapi_schema["tags"] = [ access_keys.metadata, api_information.metadata, + clingen_alleles.metadata, collections.metadata, controlled_keywords.metadata, doi_identifiers.metadata, diff --git a/src/mavedb/view_models/allele_measurement.py b/src/mavedb/view_models/allele_measurement.py new file mode 100644 index 000000000..d61197524 --- /dev/null +++ b/src/mavedb/view_models/allele_measurement.py @@ -0,0 +1,41 @@ +"""Response view model for the variant page's measurements list +(``GET /clingen-alleles/{caid}/measurements``, design §7.5). + +The pydantic serialization boundary over the ``lib.allele_measurements`` transit dataclass +(``from_attributes`` coerces it directly); aliases camelize per the shared base config. +""" + +from typing import Optional + +from mavedb.lib.allele_measurements import MeasurementRelationship +from mavedb.view_models.base.base import BaseModel +from mavedb.view_models.score_calibration import SavedFunctionalClassification + + +class AlleleMeasurement(BaseModel): + """One measurement in the queried ClinGen allele's cross-layer equivalence class. + + ``assayLevel`` is the level at which this measurement was actually assayed (``protein`` / ``cdna`` / + ``genomic``) — always shown, since the measured level is the clinically load-bearing fact. + ``relationship`` says how the measurement relates to the queried ClinGen id: ``direct`` (assayed at + this allele), ``protein_consequence`` (a protein measurement of a nt query's consequence), or + ``nucleotide_encoding`` (a nt measurement encoding a protein query). ``primaryClassification`` is the + primary readable functional classification, omitted when absent or gated. ``isCurrent`` / + ``supersededByScoreSet`` let a superseded measurement (surfaced only under ``include_superseded``) + self-describe; ``supersededByScoreSet`` is the superseding *score set*'s URN. + """ + + variant_urn: str + score: Optional[float] = None + assay_level: Optional[str] = None + relationship: MeasurementRelationship + assay_level_hgvs: Optional[str] = None + submitted_hgvs: Optional[str] = None + score_set_urn: str + score_set_title: str + primary_classification: Optional[SavedFunctionalClassification] = None + is_current: bool + superseded_by_score_set: Optional[str] = None + + class Config: + from_attributes = True diff --git a/tests/lib/test_allele_measurements.py b/tests/lib/test_allele_measurements.py new file mode 100644 index 000000000..6c49f6713 --- /dev/null +++ b/tests/lib/test_allele_measurements.py @@ -0,0 +1,371 @@ +# ruff: noqa: E402 +"""Integration tests for the ClinGen-allele measurements list (``lib/allele_measurements.py``). + +These pin the model that the first pass got wrong: measurements aggregate the **cross-layer equivalence +class** (co-membership in the mapping/RT link graph), not one exact allele, and *which* allele anchors it +determines the direct/related split (a CA anchors on the nt alleles → nt measurements are direct, the +protein consequence is related; a PA anchors on the protein → protein measurements are direct, the nt +encodings related). Also: the narrowness (a sibling nt change is not pulled onto a CA page), the two +authorization gates (score-set READ hides the measurement; calibration READ withholds only the inline +classification) exercised through the real ``has_permission``, superseded opt-in, and the default ordering +(direct-first, strongest evidence, pathogenic-biased). +""" + +import pytest + +pytest.importorskip("psycopg2") + +from mavedb.lib.allele_measurements import get_allele_measurements +from mavedb.lib.types.authentication import UserData +from mavedb.models.allele import Allele +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.score_calibration import ScoreCalibration +from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification +from mavedb.models.score_set import ScoreSet +from mavedb.models.user import User +from mavedb.models.variant import Variant +from tests.helpers.constants import EXTRA_USER, TEST_MINIMAL_VARIANT, TEST_SEQ_SCORESET, TEST_USER + + +def _user(session, username): + return session.query(User).filter(User.username == username).one() + + +def _user_data(session, username=TEST_USER["username"]): + """A ``UserData`` for a seeded user — the currency ``has_permission`` reads. ``None`` is anonymous.""" + return UserData(user=_user(session, username), active_roles=[]) + + +def _variant(session, score_set, suffix, *, data=None, hgvs_nt=None, hgvs_pro=None): + variant = Variant( + urn=f"{score_set.urn}#{suffix}", + score_set_id=score_set.id, + data=data if data is not None else TEST_MINIMAL_VARIANT["data"], + hgvs_nt=hgvs_nt, + hgvs_pro=hgvs_pro, + creation_date=TEST_MINIMAL_VARIANT["creation_date"], + modification_date=TEST_MINIMAL_VARIANT["modification_date"], + ) + session.add(variant) + session.commit() + return variant + + +def _allele(session, digest, *, level="cdna", clingen_allele_id=None): + allele = Allele(vrs_digest=digest, level=level, clingen_allele_id=clingen_allele_id) + session.add(allele) + session.commit() + return allele + + +def _record(session, variant, *, assay_level="cdna", hgvs_assay_level=None): + record = MappingRecord( + variant_id=variant.id, + assay_level=assay_level, + hgvs_assay_level=hgvs_assay_level, + mapping_api_version="test.0.0", + ) + session.add(record) + session.commit() + return record + + +def _link(session, record, allele, *, is_authoritative=False): + session.add( + MappingRecordAllele(mapping_record_id=record.id, allele_id=allele.id, is_authoritative=is_authoritative) + ) + session.commit() + + +def _calibration( + session, + score_set, + *, + variants, + functional="abnormal", + oddspaths_ratio=None, + primary=True, + private=False, + investigator_provided=False, + research_use_only=False, +): + user = _user(session, TEST_USER["username"]) + calibration = ScoreCalibration( + score_set_id=score_set.id, + title="cal", + primary=primary, + private=private, + investigator_provided=investigator_provided, + research_use_only=research_use_only, + created_by_id=user.id, + modified_by_id=user.id, + ) + session.add(calibration) + session.commit() + fc = ScoreCalibrationFunctionalClassification( + calibration_id=calibration.id, + label=functional, + functional_classification=functional, + oddspaths_ratio=oddspaths_ratio, + ) + fc.variants = variants + session.add(fc) + session.commit() + return calibration + + +def _second_score_set( + session, base, urn, *, owner_username=TEST_USER["username"], private=True, superseded_score_set_id=None +): + """A sibling score set in the same experiment — for cross-score-set, private, and supersession tests. + ``superseded_score_set_id`` set on this (newer) row makes it the superseding version of that older + set (``older.superseding_score_set`` resolves back here).""" + owner = _user(session, owner_username) + scaffold = TEST_SEQ_SCORESET.copy() + scaffold.pop("target_genes", None) + score_set = ScoreSet( + **scaffold, + urn=urn, + experiment_id=base.experiment_id, + licence_id=base.licence_id, + superseded_score_set_id=superseded_score_set_id, + ) + score_set.private = private + score_set.created_by = owner + score_set.modified_by = owner + session.add(score_set) + session.commit() + session.refresh(score_set) + return score_set + + +def _seed_two_level_pair(session, score_set): + """One nt change (CA123) and its protein consequence (PA9), each measured in its own assay, with the + other level riding as a member — the minimal shape that exercises the direct/related asymmetry.""" + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + prot = _allele(session, "prot-P", level="protein", clingen_allele_id="PA9") + + nt_measured = _variant(session, score_set, 1, data={"score_data": {"score": -2.0}}, hgvs_nt="c.1A>T") + nt_record = _record(session, nt_measured, assay_level="cdna", hgvs_assay_level="NM_0:c.1A>T") + _link(session, nt_record, nt, is_authoritative=True) + _link(session, nt_record, prot) + + protein_measured = _variant(session, score_set, 2, data={"score_data": {"score": 1.0}}, hgvs_pro="p.Met1Leu") + protein_record = _record(session, protein_measured, assay_level="protein", hgvs_assay_level="NP_0:p.Met1Leu") + _link(session, protein_record, prot, is_authoritative=True) + _link(session, protein_record, nt) + + return nt_measured, protein_measured + + +@pytest.mark.integration +def test_ca_entry_aggregates_direct_and_related(session, setup_lib_db_with_score_set): + """A CA page: the nt measurement is direct (assayed at this change); the protein measurement of its + consequence is related, labeled ``protein_consequence``, each carrying its own measured level.""" + score_set = setup_lib_db_with_score_set + nt_measured, protein_measured = _seed_two_level_pair(session, score_set) + + result = get_allele_measurements(session, "CA123", user_data=_user_data(session)) + + by_urn = {m.variant_urn: m for m in result} + assert set(by_urn) == {nt_measured.urn, protein_measured.urn} + assert (by_urn[nt_measured.urn].relationship, by_urn[nt_measured.urn].assay_level) == ("direct", "cdna") + assert (by_urn[protein_measured.urn].relationship, by_urn[protein_measured.urn].assay_level) == ( + "protein_consequence", + "protein", + ) + # Assay HGVS pair rides through (mapped reference + submitted target). + assert by_urn[nt_measured.urn].assay_level_hgvs == "NM_0:c.1A>T" + assert by_urn[nt_measured.urn].submitted_hgvs == "c.1A>T" + assert by_urn[protein_measured.urn].submitted_hgvs == "p.Met1Leu" + + +@pytest.mark.integration +def test_pa_entry_swaps_direct_and_related(session, setup_lib_db_with_score_set): + """The same data anchored on the PA: the protein measurement is now direct and the nt measurement is + the related ``nucleotide_encoding`` — the asymmetry is purely which allele anchors the co-membership.""" + score_set = setup_lib_db_with_score_set + nt_measured, protein_measured = _seed_two_level_pair(session, score_set) + + result = get_allele_measurements(session, "PA9", user_data=_user_data(session)) + + by_urn = {m.variant_urn: m for m in result} + assert set(by_urn) == {nt_measured.urn, protein_measured.urn} + assert by_urn[protein_measured.urn].relationship == "direct" + assert by_urn[nt_measured.urn].relationship == "nucleotide_encoding" + + +@pytest.mark.integration +def test_sibling_nt_not_pulled_onto_ca_page(session, setup_lib_db_with_score_set): + """A sibling nt change that encodes the *same* protein is not pulled onto a CA page (its record links + the protein but not this nt allele) — but both nt changes show on the protein's PA page as encodings.""" + score_set = setup_lib_db_with_score_set + nt1 = _allele(session, "nt-1", level="cdna", clingen_allele_id="CA111") + nt2 = _allele(session, "nt-2", level="cdna", clingen_allele_id="CA222") + prot = _allele(session, "prot-P", level="protein", clingen_allele_id="PA9") + + a = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}) + ra = _record(session, a, assay_level="cdna") + _link(session, ra, nt1, is_authoritative=True) + _link(session, ra, prot) + + c = _variant(session, score_set, 2, data={"score_data": {"score": 1.0}}) + rc = _record(session, c, assay_level="cdna") + _link(session, rc, nt2, is_authoritative=True) + _link(session, rc, prot) + + ca_page = get_allele_measurements(session, "CA111", user_data=_user_data(session)) + assert {m.variant_urn for m in ca_page} == {a.urn} + + pa_page = get_allele_measurements(session, "PA9", user_data=_user_data(session)) + assert {m.variant_urn for m in pa_page} == {a.urn, c.urn} + assert all(m.relationship == "nucleotide_encoding" for m in pa_page) + + +@pytest.mark.integration +def test_private_score_set_measurement_excluded(session, setup_lib_db_with_score_set): + """Score-set READ gates inclusion: a measurement in a score set the caller cannot read never leaks, + even though it links the same (public, content-addressed) allele.""" + score_set = setup_lib_db_with_score_set # private, owned by TEST_USER + other = _second_score_set(session, score_set, "urn:mavedb:00000002-a-1", owner_username=EXTRA_USER["username"]) + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + + visible = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}) + _link(session, _record(session, visible, assay_level="cdna"), nt, is_authoritative=True) + hidden = _variant(session, other, 1, data={"score_data": {"score": 1.0}}) + _link(session, _record(session, hidden, assay_level="cdna"), nt, is_authoritative=True) + + result = get_allele_measurements(session, "CA123", user_data=_user_data(session)) # TEST_USER + assert {m.variant_urn for m in result} == {visible.urn} + + +@pytest.mark.integration +def test_calibration_gate_withholds_classification(session, setup_lib_db_with_score_set): + """Calibration READ gates only the inline classification: on a public score set with a private + calibration, an anonymous caller still sees the measurement but its classification is withheld, while + the owner sees both.""" + score_set = setup_lib_db_with_score_set + public = _second_score_set(session, score_set, "urn:mavedb:00000002-a-1", private=False) + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + variant = _variant(session, public, 1, data={"score_data": {"score": 1.0}}) + _link(session, _record(session, variant, assay_level="cdna"), nt, is_authoritative=True) + _calibration(session, public, variants=[variant], functional="abnormal", private=True) + + anonymous = get_allele_measurements(session, "CA123", user_data=None) + assert len(anonymous) == 1 + assert anonymous[0].primary_classification is None + + owner = get_allele_measurements(session, "CA123", user_data=_user_data(session)) + assert owner[0].primary_classification is not None + assert owner[0].primary_classification.functional_classification.value == "abnormal" + + +@pytest.mark.integration +def test_primary_classification_cascade_beats_strength(session, setup_lib_db_with_score_set): + """With no primary calibration, the preference cascade dominates evidence strength: an + investigator-provided calibration wins over a stronger non-RUO one and a stronger-still RUO one.""" + score_set = setup_lib_db_with_score_set + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + variant = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}) + _link(session, _record(session, variant, assay_level="cdna"), nt, is_authoritative=True) + + _calibration(session, score_set, variants=[variant], primary=False, oddspaths_ratio=1000.0, research_use_only=True) + _calibration(session, score_set, variants=[variant], primary=False, oddspaths_ratio=100.0) # plain, non-RUO + _calibration(session, score_set, variants=[variant], primary=False, oddspaths_ratio=2.0, investigator_provided=True) + + result = get_allele_measurements(session, "CA123", user_data=_user_data(session)) + + # Investigator-provided (weakest) is chosen over the stronger non-RUO and RUO calibrations. + assert result[0].primary_classification.oddspaths_ratio == 2.0 + + +@pytest.mark.integration +def test_primary_classification_strongest_within_tier(session, setup_lib_db_with_score_set): + """Within one cascade tier (here two plain non-RUO calibrations), the strongest evidence wins.""" + score_set = setup_lib_db_with_score_set + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + variant = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}) + _link(session, _record(session, variant, assay_level="cdna"), nt, is_authoritative=True) + + _calibration(session, score_set, variants=[variant], primary=False, oddspaths_ratio=2.0) + _calibration(session, score_set, variants=[variant], primary=False, oddspaths_ratio=100.0) + + result = get_allele_measurements(session, "CA123", user_data=_user_data(session)) + + assert result[0].primary_classification.oddspaths_ratio == 100.0 + + +@pytest.mark.integration +def test_superseded_excluded_by_default_included_opt_in(session, setup_lib_db_with_score_set): + """Superseded measurements are current-tail-only by default; ``include_superseded`` opts in and they + self-describe (``is_current`` false + the superseding score set's URN).""" + score_set = setup_lib_db_with_score_set + _second_score_set(session, score_set, "urn:mavedb:00000001-a-2", superseded_score_set_id=score_set.id) + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + variant = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}) + _link(session, _record(session, variant, assay_level="cdna"), nt, is_authoritative=True) + + assert get_allele_measurements(session, "CA123", user_data=_user_data(session)) == [] + + opted = get_allele_measurements(session, "CA123", user_data=_user_data(session), include_superseded=True) + assert len(opted) == 1 + assert opted[0].is_current is False + assert opted[0].superseded_by_score_set == "urn:mavedb:00000001-a-2" + + +@pytest.mark.integration +def test_unreadable_superseding_reads_as_current(session, setup_lib_db_with_score_set): + """A superseding version the caller cannot read is not leaked — the measurement reads as current and + is included by default.""" + score_set = setup_lib_db_with_score_set # owned by TEST_USER + _second_score_set( + session, + score_set, + "urn:mavedb:00000001-a-2", + owner_username=EXTRA_USER["username"], + superseded_score_set_id=score_set.id, + ) + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + variant = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}) + _link(session, _record(session, variant, assay_level="cdna"), nt, is_authoritative=True) + + result = get_allele_measurements(session, "CA123", user_data=_user_data(session)) # TEST_USER + assert len(result) == 1 + assert result[0].is_current is True + assert result[0].superseded_by_score_set is None + + +@pytest.mark.integration +def test_ordering_direct_first_then_strongest_pathogenic(session, setup_lib_db_with_score_set): + """Default order: direct before related; within a group strongest evidence first, pathogenic before + benign at equal magnitude; the unclassified related measurement sorts last.""" + score_set = setup_lib_db_with_score_set + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + prot = _allele(session, "prot-P", level="protein", clingen_allele_id="PA9") + + strong_path = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}) + _link(session, _record(session, strong_path, assay_level="cdna"), nt, is_authoritative=True) + _calibration(session, score_set, variants=[strong_path], functional="abnormal", oddspaths_ratio=100.0) + + strong_benign = _variant(session, score_set, 2, data={"score_data": {"score": 1.0}}) + _link(session, _record(session, strong_benign, assay_level="cdna"), nt, is_authoritative=True) + _calibration(session, score_set, variants=[strong_benign], functional="normal", oddspaths_ratio=0.01) + + weak_path = _variant(session, score_set, 3, data={"score_data": {"score": 1.0}}) + _link(session, _record(session, weak_path, assay_level="cdna"), nt, is_authoritative=True) + _calibration(session, score_set, variants=[weak_path], functional="abnormal", oddspaths_ratio=2.0) + + related = _variant(session, score_set, 4, data={"score_data": {"score": 1.0}}) + related_record = _record(session, related, assay_level="protein") + _link(session, related_record, prot, is_authoritative=True) + _link(session, related_record, nt) + + result = get_allele_measurements(session, "CA123", user_data=_user_data(session)) + + assert [m.variant_urn for m in result] == [strong_path.urn, strong_benign.urn, weak_path.urn, related.urn] + + +@pytest.mark.integration +def test_unknown_clingen_id_returns_empty(session, setup_lib_db_with_score_set): + assert get_allele_measurements(session, "CA000", user_data=_user_data(session)) == [] diff --git a/tests/routers/test_clingen_allele.py b/tests/routers/test_clingen_allele.py new file mode 100644 index 000000000..7d75739db --- /dev/null +++ b/tests/routers/test_clingen_allele.py @@ -0,0 +1,122 @@ +# ruff: noqa: E402 +"""Router tests for the ClinGen-allele measurements endpoint +(``GET /clingen-alleles/{caid}/measurements``, design §7.5). + +Exercise the HTTP surface: the equivalence-class list serializes and validates, the ``as_of`` +content-time is echoed in ``X-As-Of``, an unknown id is an empty list (not a 404), superseded +measurements are opt-in, and a private score set's measurement never leaks. +""" + +import pytest + +arq = pytest.importorskip("arq") +cdot = pytest.importorskip("cdot") +fastapi = pytest.importorskip("fastapi") + +from sqlalchemy import select + +from mavedb.models.allele import Allele +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.score_set import ScoreSet as ScoreSetDbModel +from mavedb.models.variant import Variant as VariantDbModel +from mavedb.view_models.allele_measurement import AlleleMeasurement +from tests.helpers.dependency_overrider import DependencyOverrider +from tests.helpers.util.experiment import create_experiment +from tests.helpers.util.score_set import create_seq_score_set_with_variants +from tests.helpers.util.user import change_ownership + + +def _seed_cdna_measurement(session, variant_urn, *, clingen_allele_id="CA123"): + """Give a variant a live coding-measured mapping record whose authoritative allele carries the + ClinGen id — the minimal shape for a direct measurement on that CA's page.""" + variant = session.scalar(select(VariantDbModel).where(VariantDbModel.urn == variant_urn)) + record = MappingRecord( + variant_id=variant.id, + assay_level="cdna", + hgvs_assay_level="NM_000546.6:c.1216G>A", + mapping_api_version="test.0.0", + ) + session.add(record) + session.commit() + + measured = Allele(vrs_digest="cdna-digest", level="cdna", clingen_allele_id=clingen_allele_id) + session.add(measured) + session.commit() + session.add(MappingRecordAllele(mapping_record_id=record.id, allele_id=measured.id, is_authoritative=True)) + session.commit() + + +def test_measurements_serialize(client, session, data_provider, data_files, setup_router_db): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + urn = f"{score_set['urn']}#1" + _seed_cdna_measurement(session, urn) + + response = client.get("/api/v1/clingen-alleles/CA123/measurements") + + assert response.status_code == 200 + body = response.json() + assert len(body) == 1 + AlleleMeasurement.model_validate(body[0]) + assert body[0]["variantUrn"] == urn + assert body[0]["relationship"] == "direct" + assert body[0]["assayLevel"] == "cdna" + assert body[0]["assayLevelHgvs"] == "NM_000546.6:c.1216G>A" + assert body[0]["submittedHgvs"] == "c.1A>T" + assert body[0]["scoreSetUrn"] == score_set["urn"] + assert body[0]["isCurrent"] is True + assert "supersededByScoreSet" not in body[0] # dropped by exclude_none when current + assert response.headers["X-As-Of"] == "current" + + +def test_unknown_clingen_id_returns_empty_list(client, setup_router_db): + response = client.get("/api/v1/clingen-alleles/CA000/measurements") + assert response.status_code == 200 + assert response.json() == [] + + +def test_superseded_measurement_is_opt_in(client, session, data_provider, data_files, setup_router_db): + experiment = create_experiment(client) + older = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + newer = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + older_db = session.scalar(select(ScoreSetDbModel).where(ScoreSetDbModel.urn == older["urn"])) + newer_db = session.scalar(select(ScoreSetDbModel).where(ScoreSetDbModel.urn == newer["urn"])) + newer_db.superseded_score_set_id = older_db.id + session.commit() + _seed_cdna_measurement(session, f"{older['urn']}#1") + + default = client.get("/api/v1/clingen-alleles/CA123/measurements") + assert default.status_code == 200 + assert default.json() == [] + + opted = client.get("/api/v1/clingen-alleles/CA123/measurements", params={"include_superseded": True}) + assert opted.status_code == 200 + body = opted.json() + assert len(body) == 1 + assert body[0]["isCurrent"] is False + assert body[0]["supersededByScoreSet"] == newer["urn"] + + +def test_anonymous_cannot_see_private_measurement( + client, session, data_provider, data_files, setup_router_db, anonymous_app_overrides +): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + _seed_cdna_measurement(session, f"{score_set['urn']}#1") + change_ownership(session, score_set["urn"], ScoreSetDbModel) + + with DependencyOverrider(anonymous_app_overrides): + response = client.get("/api/v1/clingen-alleles/CA123/measurements") + + # A content-addressed ClinGen id is public, but a private score set's measurement is filtered out. + assert response.status_code == 200 + assert response.json() == [] From 41ada42c8a260d14f32af28365d510cfb439361e Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 7 Jul 2026 22:58:42 -0700 Subject: [PATCH 069/252] feat(variants): add per-allele identity sidecar to variant detail envelope Replace the UI-unused memberRelations map on GET /variants/{urn} with an alleles sidecar keyed by VRS digest: one entry per linked allele (the record-scoped, all-levels set, sharing keys with annotations) carrying level, reference-frame HGVS, ClinGen id, and its relation to the measured allele. This lets the selection panel label annotations at every level by level + HGVS rather than by digest, without per-digest /alleles/{digest} fetches. --- src/mavedb/lib/variant_detail.py | 57 ++++++++++++++++++++---- src/mavedb/view_models/variant_detail.py | 35 +++++++++++---- tests/lib/test_variant_detail.py | 37 ++++++++++++--- tests/routers/test_variant.py | 22 +++++++-- 4 files changed, 125 insertions(+), 26 deletions(-) diff --git a/src/mavedb/lib/variant_detail.py b/src/mavedb/lib/variant_detail.py index 0331c7d7a..30f3fbc3b 100644 --- a/src/mavedb/lib/variant_detail.py +++ b/src/mavedb/lib/variant_detail.py @@ -3,7 +3,8 @@ Assembles the two-tier envelope for a single variant: flat, UI-ergonomic assay fields (the assay-level HGVS pair, digest, ClinGen id) plus the spec-pure GA4GH ``CategoricalVariant`` (built on the fly by :mod:`lib.cat_vrs`) and, riding alongside it keyed by VRS digest, the MaveDB layer — -per-member relations and the digest-keyed external-annotation map (:mod:`lib.annotations`). Also +the per-allele identity sidecar (level / HGVS / ClinGen id / member→defining relation) and the +digest-keyed external-annotation map (:mod:`lib.allele_annotations`). Also carries the per-calibration functional ``classifications`` the variant falls into, and its version standing (``is_current`` / ``superseded_by_score_set``) so a superseded variant self-describes rather than reading as current (design §9.2). @@ -52,6 +53,24 @@ class VariantClassificationRecord: classification: ScoreCalibrationFunctionalClassification +@dataclass(frozen=True) +class AlleleIdentity: + """The MaveDB molecular-identity facts for one of the variant's linked alleles. + + Rides alongside the spec-pure Cat-VRS keyed by VRS digest (the ``alleles`` map). ``level`` and + ``hgvs`` (the reference-frame HGVS, exactly one of the allele's genomic/coding/protein columns) are + what the UI labels the per-level annotation panel by — *never* the digest. ``relation`` is this + allele's relation to the measured (defining) allele; ``None`` when it *is* the measured allele, or + when the allele is not a Cat-VRS member. (The Cat-VRS relation is a *member*→defining code, hence + absent for the defining allele — see ``cat_vrs.CategoricalVariantTransit.member_relations``.) + """ + + level: Optional[str] + hgvs: Optional[str] + clingen_allele_id: Optional[str] + relation: Optional[str] + + @dataclass(frozen=True) class VariantDetail: """The assembled variant-detail envelope (transit; serialized by ``view_models.variant``).""" @@ -68,12 +87,14 @@ class VariantDetail: assay_level_digest: Optional[str] clingen_allele_id: Optional[str] - # Spec-pure GA4GH Cat-VRS (no MaveDB fields inside), plus the MaveDB layer riding alongside. + # Spec-pure GA4GH Cat-VRS (no MaveDB fields inside), plus the MaveDB layer riding alongside: the + # per-allele identity sidecar, keyed by VRS digest — one entry per linked allele (the record-scoped, + # all-levels set, sharing keys with `annotations`), carrying level + HGVS + ClinGen id + relation. molecular_representation: Optional[dict[str, Any]] mode: Optional[str] # CatVrsMode value — projection | reverse_translation - member_relations: dict[str, str] # member vrs_digest -> relation (member -> defining) + alleles: dict[str, AlleleIdentity] - # External annotations, keyed by VRS digest, joined to the Cat-VRS members. + # External annotations, keyed by VRS digest, joined to the Cat-VRS members / the alleles sidecar. annotations: dict[str, AlleleAnnotations] # Version standing — self-descriptive for a superseded variant. @@ -161,16 +182,36 @@ def get_variant_detail( annotations = get_allele_annotations(db, [link.allele for link in links], as_of=as_of) - # Spec-pure Cat-VRS built on the fly, plus the MaveDB layer (mode + per-member relations). + # Spec-pure Cat-VRS built on the fly, plus the MaveDB layer (mode + per-member relations). The + # relations key the sidecar below; the defining allele is deliberately absent from them, so it + # gets relation=None. transit = categorical_variant_for_variant(db, variant.id, name=variant.urn or "", as_of=as_of) if transit is not None: molecular_representation = transit.categorical_variant.model_dump(mode="json", exclude_none=True) mode: Optional[str] = transit.mode.value - member_relations = {digest: relation.value for digest, relation in transit.member_relations.items()} + relations = transit.member_relations else: molecular_representation = None mode = None - member_relations = {} + relations = {} + + # The per-allele identity sidecar: one entry per linked allele, keyed by VRS digest (the same + # record-scoped, all-levels link set that seeds `annotations`, so the two maps share keys). Carries + # the molecular-identity facts the UI labels annotations by — level, reference-frame HGVS (exactly one + # of hgvs_g/c/p is populated per allele), ClinGen id — and the member→defining relation. + alleles: dict[str, AlleleIdentity] = {} + for link in links: + allele = link.allele + if allele.vrs_digest is None: + continue + + relation = relations.get(allele.vrs_digest) + alleles[allele.vrs_digest] = AlleleIdentity( + level=allele.level, + hgvs=allele.hgvs_g or allele.hgvs_c or allele.hgvs_p, + clingen_allele_id=allele.clingen_allele_id, + relation=relation.value if relation is not None else None, + ) return VariantDetail( urn=variant.urn or "", @@ -184,7 +225,7 @@ def get_variant_detail( clingen_allele_id=clingen_allele_id, molecular_representation=molecular_representation, mode=mode, - member_relations=member_relations, + alleles=alleles, annotations=annotations, is_current=superseding_score_set is None, superseded_by_score_set=superseding_score_set.urn if superseding_score_set is not None else None, diff --git a/src/mavedb/view_models/variant_detail.py b/src/mavedb/view_models/variant_detail.py index 2c242f756..b4c5fa5b2 100644 --- a/src/mavedb/view_models/variant_detail.py +++ b/src/mavedb/view_models/variant_detail.py @@ -29,18 +29,37 @@ class Config: from_attributes = True +class AlleleIdentity(BaseModel): + """The MaveDB molecular-identity facts for one of the variant's linked alleles. + + An entry of the ``alleles`` sidecar, keyed by VRS digest. ``level`` + ``hgvs`` (the reference-frame + HGVS) are what the UI labels the per-level annotation panel by — never the digest (design §7.5). + ``relation`` is this allele's relation to the measured (defining) allele; ``null`` when it *is* the + measured allele, or when the allele is not a Cat-VRS member. + """ + + level: Optional[str] = None + hgvs: Optional[str] = None + clingen_allele_id: Optional[str] = None + relation: Optional[str] = None + + class Config: + from_attributes = True + + class VariantDetail(BaseModel): """The assayed variant-detail envelope (``GET /variants/{urn}``, design §7.1). Two tiers: flat, UI-ergonomic assay fields (the ``targetHgvs``/``referenceHgvs`` coordinate pair is a client-side toggle, no refetch) plus the spec-pure GA4GH ``molecularRepresentation`` - (``CategoricalVariant``, no MaveDB fields inside). The MaveDB layer rides alongside keyed by VRS - digest: ``memberRelations`` (member→defining relation) and the ``annotations`` map. ``isCurrent`` - /``supersededByScoreSet`` let a superseded variant self-describe: ``supersededByScoreSet`` is the - superseding *score set*'s URN, not a variant URN. Supersession is versioned at the score-set level, - and a newer version may add, drop, or renumber variants — so there is no stable - superseding-*variant* pointer to hand back; a consumer resolves the current measurement by looking - this variant up within that score set. Absent fields are omitted. + (``CategoricalVariant``, no MaveDB fields inside). The MaveDB layer rides alongside, keyed by VRS + digest: the ``alleles`` identity sidecar (per-allele ``level`` / ``hgvs`` / ``clingenAlleleId`` / + ``relation`` — one entry per linked allele, sharing keys with ``annotations``) and the + ``annotations`` map. ``isCurrent``/``supersededByScoreSet`` let a superseded variant self-describe: + ``supersededByScoreSet`` is the superseding *score set*'s URN, not a variant URN. Supersession is + versioned at the score-set level, and a newer version may add, drop, or renumber variants — so there + is no stable superseding-*variant* pointer to hand back; a consumer resolves the current measurement + by looking this variant up within that score set. Absent fields are omitted. """ urn: str @@ -56,7 +75,7 @@ class VariantDetail(BaseModel): molecular_representation: Optional[dict[str, Any]] = None mode: Optional[str] = None - member_relations: dict[str, str] = {} + alleles: dict[str, AlleleIdentity] = {} annotations: dict[str, AlleleAnnotations] = {} diff --git a/tests/lib/test_variant_detail.py b/tests/lib/test_variant_detail.py index 6cea44858..fcf98c31b 100644 --- a/tests/lib/test_variant_detail.py +++ b/tests/lib/test_variant_detail.py @@ -66,8 +66,16 @@ def _variant(session, score_set, suffix, *, data=None, hgvs_nt=None, hgvs_pro=No return variant -def _allele(session, digest, *, level="cdna", clingen_allele_id=None): - allele = Allele(vrs_digest=digest, level=level, post_mapped=_post_mapped(), clingen_allele_id=clingen_allele_id) +def _allele(session, digest, *, level="cdna", clingen_allele_id=None, hgvs_g=None, hgvs_c=None, hgvs_p=None): + allele = Allele( + vrs_digest=digest, + level=level, + post_mapped=_post_mapped(), + clingen_allele_id=clingen_allele_id, + hgvs_g=hgvs_g, + hgvs_c=hgvs_c, + hgvs_p=hgvs_p, + ) session.add(allele) session.commit() return allele @@ -134,8 +142,8 @@ def test_full_envelope_for_a_coding_assay(session, setup_lib_db_with_score_set): session, score_set, 1, data={"score_data": {"score": -2.3}}, hgvs_nt="c.1216G>A", hgvs_pro="p.Ala406Thr" ) record = _record(session, variant, assay_level="cdna", hgvs_assay_level="NM_000546.6:c.1216G>A") - measured = _allele(session, "cdna-digest", level="cdna", clingen_allele_id="CA123") - protein = _allele(session, "prot-digest", level="protein") + measured = _allele(session, "cdna-digest", level="cdna", clingen_allele_id="CA123", hgvs_c="NM_000546.6:c.1216G>A") + protein = _allele(session, "prot-digest", level="protein", hgvs_p="NP_000537.3:p.Ala406Thr") _link(session, record, measured, is_authoritative=True) _link(session, record, protein) _vep(session, measured, "missense_variant") @@ -152,8 +160,19 @@ def test_full_envelope_for_a_coding_assay(session, setup_lib_db_with_score_set): assert detail.mode == "projection" assert detail.molecular_representation is not None assert detail.molecular_representation["type"] == "CategoricalVariant" + # The alleles sidecar carries one identity per linked allele, keyed by digest: level + + # reference-frame HGVS (coalesced from hgvs_g/c/p) + ClinGen id + member->defining relation. + assert set(detail.alleles) == {"cdna-digest", "prot-digest"} + measured_identity = detail.alleles["cdna-digest"] + assert measured_identity.level == "cdna" + assert measured_identity.hgvs == "NM_000546.6:c.1216G>A" # coalesced from hgvs_c + assert measured_identity.clingen_allele_id == "CA123" + assert measured_identity.relation is None # the defining allele has no relation to itself + protein_identity = detail.alleles["prot-digest"] + assert protein_identity.level == "protein" + assert protein_identity.hgvs == "NP_000537.3:p.Ala406Thr" # coalesced from hgvs_p # The protein member is a translation of the measured coding allele (member -> defining). - assert detail.member_relations == {"prot-digest": "translation_of"} + assert protein_identity.relation == "translation_of" # Annotations keyed by digest, covering both linked alleles; VEP rode in on the measured allele. assert set(detail.annotations) == {"cdna-digest", "prot-digest"} assert detail.annotations["cdna-digest"].vep is not None @@ -181,7 +200,11 @@ def test_protein_assay_targets_the_protein_frame(session, setup_lib_db_with_scor assert detail.reference_hgvs == "NP_000537.3:p.Ala406Thr" assert detail.assay_level_digest == "prot-digest" assert detail.mode == "reverse_translation" - assert detail.member_relations == {"cdna-digest": "encodes"} + # Defining protein allele has no relation to itself; the coding member encodes it. + assert detail.alleles["prot-digest"].level == "protein" + assert detail.alleles["prot-digest"].relation is None + assert detail.alleles["cdna-digest"].level == "cdna" + assert detail.alleles["cdna-digest"].relation == "encodes" @pytest.mark.integration @@ -201,7 +224,7 @@ def test_unmapped_variant_has_null_molecular_layer(session, setup_lib_db_with_sc assert detail.clingen_allele_id is None assert detail.molecular_representation is None assert detail.mode is None - assert detail.member_relations == {} + assert detail.alleles == {} assert detail.annotations == {} assert detail.is_current is True diff --git a/tests/routers/test_variant.py b/tests/routers/test_variant.py index 365fe9ca7..8bb1c3503 100644 --- a/tests/routers/test_variant.py +++ b/tests/routers/test_variant.py @@ -64,8 +64,16 @@ def _seed_mapping(session, variant_urn): session.add(record) session.commit() - measured = Allele(vrs_digest="cdna-digest", level="cdna", post_mapped=_post_mapped(), clingen_allele_id="CA123") - protein = Allele(vrs_digest="prot-digest", level="protein", post_mapped=_post_mapped()) + measured = Allele( + vrs_digest="cdna-digest", + level="cdna", + post_mapped=_post_mapped(), + clingen_allele_id="CA123", + hgvs_c="NM_000546.6:c.1216G>A", + ) + protein = Allele( + vrs_digest="prot-digest", level="protein", post_mapped=_post_mapped(), hgvs_p="NP_000537.3:p.Ala406Thr" + ) session.add_all([measured, protein]) session.commit() @@ -106,7 +114,15 @@ def test_get_variant_detail_envelope(client, session, data_provider, data_files, assert body["mode"] == "projection" # Spec-pure Cat-VRS carries its own field names (no camelization of the nested GA4GH object). assert body["molecularRepresentation"]["type"] == "CategoricalVariant" - assert body["memberRelations"] == {"prot-digest": "translation_of"} + # The alleles identity sidecar serializes camelCase, keyed by digest; the protein member is a + # translation of the defining coding allele (which itself has no relation to itself). + assert body["alleles"]["cdna-digest"]["level"] == "cdna" + assert body["alleles"]["cdna-digest"]["hgvs"] == "NM_000546.6:c.1216G>A" + assert body["alleles"]["cdna-digest"]["clingenAlleleId"] == "CA123" + assert "relation" not in body["alleles"]["cdna-digest"] # null relation dropped by exclude_none + assert body["alleles"]["prot-digest"]["level"] == "protein" + assert body["alleles"]["prot-digest"]["hgvs"] == "NP_000537.3:p.Ala406Thr" + assert body["alleles"]["prot-digest"]["relation"] == "translation_of" assert body["annotations"]["cdna-digest"]["vep"]["consequence"] == "missense_variant" assert body["isCurrent"] is True assert "supersededByScoreSet" not in body # dropped by exclude_none when current From 031eaab3cf2fc29e7b1ef1054d24d2d24ae93f77 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 8 Jul 2026 12:34:24 -0700 Subject: [PATCH 070/252] =?UTF-8?q?perf(score-set-variants):=20split=20pro?= =?UTF-8?q?tein=20projection=20into=20separate=20query=20to=20avoid=20O(N?= =?UTF-8?q?=C2=B2)=20planner=20trap?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The protein HGVS subquery uses DISTINCT ON, whose output cardinality is opaque to the planner (~18 estimated vs ~11.5k actual). Joined in, the planner buried it on the inner side of a nested loop and rescanned it in full once per variant — accounting for ~97% of the endpoint's total time. MATERIALIZED in a CTE fixed construction but not the per-variant rescan. Pulling the protein projection out as a standalone query and stitching it back by mapping_record_id in Python keeps both queries O(N) — the same batched-join pattern SQLAlchemy's selectin loader uses internally for the same class of cardinality estimation problem. --- src/mavedb/lib/score_set_variants.py | 93 ++++++++++++++-------------- 1 file changed, 48 insertions(+), 45 deletions(-) diff --git a/src/mavedb/lib/score_set_variants.py b/src/mavedb/lib/score_set_variants.py index 134a4eae2..c6bd8e382 100644 --- a/src/mavedb/lib/score_set_variants.py +++ b/src/mavedb/lib/score_set_variants.py @@ -1,8 +1,10 @@ """The lean whole-set view backing the score-set page. The score-set table, heatmap, and score/effect histograms all compose from one pre-chewed per-variant -dataset. This module assembles that dataset for an entire score set in a single bulk query and returns -one record per variant. +dataset. This module assembles that dataset for an entire score set in two O(N) bulk queries — a base +per-variant projection plus a standalone protein-HGVS projection stitched back in Python — and returns +one record per variant. (The protein projection is deliberately *not* a join: folded in, its opaque +cardinality drives the SQL planner into an O(N^2) per-variant rescan; see ``get_lean_score_set_variants``.) Each record carries the HGVS in both frames the heatmap toggles between — the depositor-**submitted** strings (``hgvs_nt``/``hgvs_pro``/``hgvs_splice``, the target frame) and the **mapped** assay-level @@ -92,50 +94,18 @@ def get_lean_score_set_variants( histogram still see them. ``as_of`` reconstructs the annotation layer at a past instant (submitted HGVS and scores are immutable and unaffected); it defaults to the currently-live rows. """ - # Per record, the forward-translated protein HGVS (hgvs_p) of its protein-level allele. - prot_link = aliased(MappingRecordAllele) - prot_allele = aliased(Allele) - prot_record = aliased(MappingRecord) - prot_variant = aliased(Variant) - protein_allele = ( - select( - prot_link.mapping_record_id.label("mapping_record_id"), - prot_allele.hgvs_p.label("protein_hgvs"), - ) - # DISTINCT ON -> at most one protein level row per record. - .distinct(prot_link.mapping_record_id) - # link -> allele: level and hgvs_p live on the *allele*. - .join(prot_allele, prot_allele.id == prot_link.allele_id) - # link -> record -> variant: the bridge to scoreset_id, to scope the subquery (next). - .join(prot_record, prot_record.id == prot_link.mapping_record_id) - .join(prot_variant, prot_variant.id == prot_record.variant_id) - # Scope to this score set so the subquery's cost scales with the response. - .where(prot_variant.score_set_id == score_set.id) - # Live links only. A live link implies a live parent record (ValidTime invariant), so this also - # covers record-liveness — no prot_record.live_at needed; the MR join is just the scoping bridge. - .where(prot_link.live_at(as_of)) - # Only the protein-level allele. - .where(prot_allele.level == AnnotationLayer.protein.value) - # DISTINCT ON requires ORDER BY to lead with its column; allele.id then picks which row survives - # per record — inert today (only one protein allele exists). - .order_by(prot_link.mapping_record_id, prot_allele.id) - # MATERIALIZED forces this to run exactly once. The planner badly under-counted the DISTINCT ON - # cardinality (est. ~5 vs ~2.8k actual) and, left as a plain subquery, buries it on the inner - # side of a nested loop, re-running the whole projection once per variant. That is O(N^2) in the - # score set size. Materializing collapses it back to O(N). - .cte("protein_allele") - .prefix_with("MATERIALIZED") - ) - - statement = ( + # The base per-variant projection: the four annotation joins that stay strictly 1:1 with the variant, + # so this whole query is a stream of O(N) index-scan lookups. MappingRecord.id rides along as the key + # the protein projection (a *separate* query, below) is stitched back on. + base_statement = ( select( Variant.urn, Variant.data, Variant.hgvs_nt, Variant.hgvs_pro, Variant.hgvs_splice, + MappingRecord.id.label("mapping_record_id"), MappingRecord.hgvs_assay_level, - protein_allele.c.protein_hgvs.label("protein_hgvs"), Allele.vrs_digest, Allele.clingen_allele_id, VepAlleleConsequence.functional_consequence, @@ -162,16 +132,49 @@ def get_lean_score_set_variants( .outerjoin( VepAlleleConsequence, and_(VepAlleleConsequence.allele_id == Allele.id, VepAlleleConsequence.live_at(as_of)) ) - # The protein projection (materialized above), matched back by record id (a globally-unique PK -> - # the right protein row, no cross-set risk). LEFT: some rows have no protein projection - # (UTR/intronic) -> null. - .outerjoin(protein_allele, protein_allele.c.mapping_record_id == MappingRecord.id) # The anchor: just this score set's variants. .where(Variant.score_set_id == score_set.id) # Natural table order: variant number (the integer after '#'), id breaks ties stably. .order_by(cast(func.split_part(Variant.urn, "#", 2), Integer), Variant.id) ) + # The protein projection runs as its own query, NOT a join into the base statement. Joined in, it forces + # an O(N^2) plan: the DISTINCT ON cardinality is opaque to the planner (est. ~18 vs ~11.5k actual), so it + # buried the projection on the inner side of a nested loop and rescanned it in full once per variant — + # ~97% of the endpoint's total time. MATERIALIZED in the query above fixes the *construction* but not the + # per-variant *rescan*. Pulling out and stitching these queries by mapping_record_id in Python enables both + # queries to run in O(N) time, making this function (and the endpoint it serves) considerably faster. + prot_link = aliased(MappingRecordAllele) + prot_allele = aliased(Allele) + prot_record = aliased(MappingRecord) + prot_variant = aliased(Variant) + protein_statement = ( + select( + prot_link.mapping_record_id.label("mapping_record_id"), + prot_allele.hgvs_p.label("protein_hgvs"), + ) + # DISTINCT ON -> at most one protein level row per record. + .distinct(prot_link.mapping_record_id) + # link -> allele: level and hgvs_p live on the *allele*. + .join(prot_allele, prot_allele.id == prot_link.allele_id) + # link -> record -> variant: the bridge to scoreset_id, to scope the query. + .join(prot_record, prot_record.id == prot_link.mapping_record_id) + .join(prot_variant, prot_variant.id == prot_record.variant_id) + # Scope to this score set so the cost scales with the response. + .where(prot_variant.score_set_id == score_set.id) + # Live links only. A live link implies a live parent record (ValidTime invariant), so this also + # covers record-liveness — no prot_record.live_at needed; the MR join is just the scoping bridge. + .where(prot_link.live_at(as_of)) + # Only the protein-level allele. + .where(prot_allele.level == AnnotationLayer.protein.value) + # DISTINCT ON requires ORDER BY to lead with its column; allele.id then picks which row survives + # per record — inert today (only one protein allele exists). + .order_by(prot_link.mapping_record_id, prot_allele.id) + ) + # Keyed by record id (a globally-unique PK -> the right protein row, no cross-set risk). Some records + # have no protein projection (UTR/intronic) -> absent from the map -> null protein field below. + protein_hgvs_by_record = {row.mapping_record_id: row.protein_hgvs for row in db.execute(protein_statement)} + return [ LeanVariantRecord( variant_urn=row.urn, @@ -183,7 +186,7 @@ def get_lean_score_set_variants( hgvs_pro=_hgvs_field_for_str(row.hgvs_pro), hgvs_splice=_hgvs_field_for_str(row.hgvs_splice), assay_level_hgvs=_hgvs_field_for_str(row.hgvs_assay_level), - protein_level_hgvs=_hgvs_field_for_str(row.protein_hgvs), + protein_level_hgvs=_hgvs_field_for_str(protein_hgvs_by_record.get(row.mapping_record_id)), ) - for row in db.execute(statement) + for row in db.execute(base_statement) ] From aafe524f2aa8b70057e62da4a85c1cb64473b0d2 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 8 Jul 2026 22:08:55 -0700 Subject: [PATCH 071/252] feat(annotation): pair c/g projections via projection_group Add a nullable projection_group column to mapping_record_alleles and thread the coding/genomic pairing it records through the reverse- translation worker and the serving layer. - migration e7b2c9a1f4d3: add nullable projection_group (no index, no backfill; existing rows stay NULL and re-group on next re-map) - RT worker: consume the library's ProjectionPair list, stamp both links of a pair with a shared per-record group id, leave the protein apex ungrouped, and fold a pair member that equals the record's authoritative allele onto the existing link instead of duplicating it - lean variant view: replace assayLevelHgvs/proteinLevelHgvs with an assayLevel pointer plus a mapped MappedTriple (genomic/cdna/protein), filling the other nucleotide slot from the authoritative link's projection_group sibling via one indexed 1:1 join - variant detail: add derivation (authoritative/projection/candidate) and projection_of (the sibling's VRS digest) to AlleleIdentity Unpopulated projection_group (pre-RT data, protein apex) degrades gracefully to a null sibling / empty slot. --- ...jection_group_to_mapping_record_alleles.py | 37 ++ src/mavedb/lib/score_set_variants.py | 150 +++++-- src/mavedb/lib/variant_detail.py | 83 +++- src/mavedb/models/mapping_record_allele.py | 16 +- src/mavedb/view_models/lean_variant.py | 36 +- src/mavedb/view_models/variant_detail.py | 13 +- .../variant_processing/reverse_translation.py | 84 ++-- tests/helpers/util/annotation.py | 6 +- tests/lib/test_score_set_variants.py | 121 +++-- tests/lib/test_variant_detail.py | 97 +++- tests/routers/test_score_set.py | 30 +- tests/routers/test_variant.py | 25 +- .../test_reverse_translation.py | 417 ++++++++++++++---- 13 files changed, 916 insertions(+), 199 deletions(-) create mode 100644 alembic/versions/e7b2c9a1f4d3_add_projection_group_to_mapping_record_alleles.py diff --git a/alembic/versions/e7b2c9a1f4d3_add_projection_group_to_mapping_record_alleles.py b/alembic/versions/e7b2c9a1f4d3_add_projection_group_to_mapping_record_alleles.py new file mode 100644 index 000000000..6d0d0ff7a --- /dev/null +++ b/alembic/versions/e7b2c9a1f4d3_add_projection_group_to_mapping_record_alleles.py @@ -0,0 +1,37 @@ +"""add projection_group to mapping_record_alleles + +Revision ID: e7b2c9a1f4d3 +Revises: f1a3c7e9b2d5 +Create Date: 2026-07-08 + +Adds a nullable ``projection_group`` integer column to ``mapping_record_alleles`` — a within-record +grouping key that pairs the coding and genomic links of one reverse-translation projection pair. +The two members of a pair share a value; the shared protein apex is NULL, as is every row written +before reverse translation runs. See the model and the reverse_translation worker for the full +semantics (per-record index, authoritative fold-in). + +No index is added: the per-URN serving fetch is already covered by +``ix_mapping_record_alleles_mapping_record_id`` and a record's link set is small. No backfill: +existing rows stay NULL and re-group lazily on the next re-map / reverse-translation run. +""" + +import sqlalchemy as sa + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "e7b2c9a1f4d3" +down_revision = "f1a3c7e9b2d5" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column( + "mapping_record_alleles", + sa.Column("projection_group", sa.Integer(), nullable=True), + ) + + +def downgrade() -> None: + op.drop_column("mapping_record_alleles", "projection_group") diff --git a/src/mavedb/lib/score_set_variants.py b/src/mavedb/lib/score_set_variants.py index c6bd8e382..b2a2e060a 100644 --- a/src/mavedb/lib/score_set_variants.py +++ b/src/mavedb/lib/score_set_variants.py @@ -6,18 +6,25 @@ one record per variant. (The protein projection is deliberately *not* a join: folded in, its opaque cardinality drives the SQL planner into an O(N^2) per-variant rescan; see ``get_lean_score_set_variants``.) -Each record carries the HGVS in both frames the heatmap toggles between — the depositor-**submitted** -strings (``hgvs_nt``/``hgvs_pro``/``hgvs_splice``, the target frame) and the **mapped** assay-level -string (reference frame) — plus, riding alongside each string, the parsed ``position``/``ref``/``alt`` -block when the expression is a placeable simple substitution (the heatmap grid; ``None`` for -splice/indels/multivariants, which keep the string alone). The string is the canonical, lossless field; -the block is a convenience the client would otherwise derive by parsing. - -Because every mapped string now comes from the ``MappingRecord`` and the only allele we need is the -**authoritative** (measured) one — for its digest, ClinGen id, and VEP consequence — the query joins -the authoritative link only and so returns **one row per variant** (no allele fan-out to regroup). It -does not assemble Cat-VRS. ``as_of`` reconstructs the annotation layer at a past instant over -the variant's immutable scores/submitted HGVS; it defaults to the currently-live rows. +Each record carries the **submitted** HGVS the heatmap's frame toggle needs (``hgvs_nt``/``hgvs_pro``/ +``hgvs_splice``, the depositor's target frame) plus the **mapped** (reference frame) representation as a +``MappedTriple`` — one canonical HGVS per level (``genomic`` / ``cdna`` / ``protein``) and an +``assay_level`` pointer naming which slot is the measured one. ``mapped[assay_level]`` is the measured +representation. Only the *canonical* projection is served here — never the reverse-translation +fan-out (that lives in the detail endpoint), so a nucleotide assay fills all three slots while a protein +assay fills only ``protein`` (``cdna``/``genomic`` stay ``None`` — the c/g fan-out is ambiguous, so no +pick is fabricated). Each HGVS rides as an ``HgvsField``: the string (canonical, lossless) plus a parsed +``position``/``ref``/``alt`` block when it is a placeable simple substitution (the heatmap grid; ``None`` +for splice/indels/multivariants). + +The canonical nucleotide pair comes from the **authoritative** (measured) allele — its digest, ClinGen +id, and VEP consequence — plus one indexed join to its ``projection_group`` sibling at the other +nucleotide level (still **one row per variant**: a group has ≤2 members, so the authoritative has ≤1 +sibling). The protein slot is the separate ``DISTINCT ON`` subquery. Where ``projection_group`` is +unpopulated (pre-reverse-translation data) the sibling join returns null → the other nucleotide slot +stays ``None`` and improves as data is re-processed. This view does not assemble Cat-VRS. ``as_of`` +reconstructs the annotation layer at a past instant over the variant's immutable scores/submitted HGVS; +it defaults to the currently-live rows. """ from dataclasses import dataclass @@ -53,15 +60,34 @@ class HgvsField: alt: Optional[str] = None -# TODO#784: Add a canonical nucleotide level projection in parallel with the canonical protein level projection. +@dataclass(frozen=True) +class MappedTriple: + """The mapped (reference-frame) HGVS at each level — the canonical projection of the measured change. + + One slot per level, each an :class:`HgvsField` or ``None``. A **nucleotide** assay populates all + three (``mapped[assay_level]`` is the measured slot, the other nucleotide slot is the authoritative + allele's ``projection_group`` sibling, ``protein`` is the apex). A **protein** assay populates only + ``protein`` — the c/g fan-out is ambiguous, so ``cdna``/``genomic`` stay ``None`` rather than + fabricate a canonical pick. A slot is also ``None`` when its source is simply absent (an unmapped + variant; a projection that failed; pre-reverse-translation data with no sibling recorded). + + ``cdna`` is the level-invariant search key — populated even when ``assay_level`` is ``genomic``. + """ + + genomic: Optional[HgvsField] = None + cdna: Optional[HgvsField] = None + protein: Optional[HgvsField] = None + + @dataclass(frozen=True) class LeanVariantRecord: """One pre-chewed per-variant record: the selection key (``variant_urn``), the baseline ``score``, a representative (lossy) VEP ``consequence``, the bridge identifiers into the annotation dimensions - (``clingen_allele_id``, ``assay_level_digest``), the submitted HGVS at each level, and the two mapped - representations: ``assay_level_hgvs`` (the measured level) and ``protein_level_hgvs`` (the protein level). - For a protein assay the two mapped fields coincide. Any field is ``None`` when its source is absent - (unmapped variant → null mapped fields; a level the variant does not carry → null slot).""" + (``clingen_allele_id``, ``assay_level_digest``), the submitted HGVS at each level, and the mapped + (reference-frame) representation as an ``assay_level`` pointer (an ``AnnotationLayer`` value naming + the measured/canonical slot) plus a ``mapped`` :class:`MappedTriple`. ``mapped[assay_level]`` is the + measured representation; ``mapped.cdna`` is the level-invariant search key. Any field is ``None`` + when its source is absent (unmapped variant → null ``assay_level`` + empty triple).""" variant_urn: str score: Optional[float] @@ -71,8 +97,8 @@ class LeanVariantRecord: hgvs_nt: Optional[HgvsField] hgvs_pro: Optional[HgvsField] hgvs_splice: Optional[HgvsField] - assay_level_hgvs: Optional[HgvsField] - protein_level_hgvs: Optional[HgvsField] + assay_level: Optional[str] + mapped: MappedTriple def _hgvs_field_for_str(hgvs: Optional[str]) -> Optional[HgvsField]: @@ -86,6 +112,43 @@ def _hgvs_field_for_str(hgvs: Optional[str]) -> Optional[HgvsField]: return HgvsField(hgvs=hgvs, position=block.position, ref=block.ref, alt=block.alt) +def _mapped_triple( + *, + assay_level: Optional[str], + assay_level_hgvs: Optional[str], + sibling_level: Optional[str], + sibling_hgvs: Optional[str], + protein_hgvs: Optional[str], +) -> MappedTriple: + """Assemble the canonical :class:`MappedTriple` for one variant from its per-row query fields. + + - Measured slot (``mapped[assay_level]``) ← ``assay_level_hgvs`` (the record's mapped assay HGVS). + - Other nucleotide slot ← the authoritative link's ``projection_group`` sibling (``sibling_level`` / + ``sibling_hgvs``), populated only on a nucleotide assay and only where the pairing is recorded. + - ``protein`` slot ← the separate protein-apex subquery. + + A **protein** assay fills only ``protein`` (the measured slot is protein itself); its nucleotide + fan-out is ambiguous, so ``cdna``/``genomic`` are deliberately left ``None``. An unmapped variant + (``assay_level is None``) yields an empty triple. + """ + slots: dict[str, Optional[HgvsField]] = {} + # Protein assay: the measured slot *is* protein. No canonical c/g — do not fabricate one. + if assay_level == AnnotationLayer.protein.value: + slots[AnnotationLayer.protein.value] = _hgvs_field_for_str(assay_level_hgvs) + elif assay_level in (AnnotationLayer.cdna.value, AnnotationLayer.genomic.value): + slots[assay_level] = _hgvs_field_for_str(assay_level_hgvs) + # The other nucleotide level, from the projection_group sibling (null where unpopulated). + if sibling_level is not None: + slots[sibling_level] = _hgvs_field_for_str(sibling_hgvs) + slots[AnnotationLayer.protein.value] = _hgvs_field_for_str(protein_hgvs) + + return MappedTriple( + genomic=slots.get(AnnotationLayer.genomic.value), + cdna=slots.get(AnnotationLayer.cdna.value), + protein=slots.get(AnnotationLayer.protein.value), + ) + + def get_lean_score_set_variants( db: Session, score_set: ScoreSet, *, as_of: Optional[datetime] = None ) -> list[LeanVariantRecord]: @@ -94,9 +157,15 @@ def get_lean_score_set_variants( histogram still see them. ``as_of`` reconstructs the annotation layer at a past instant (submitted HGVS and scores are immutable and unaffected); it defaults to the currently-live rows. """ - # The base per-variant projection: the four annotation joins that stay strictly 1:1 with the variant, - # so this whole query is a stream of O(N) index-scan lookups. MappingRecord.id rides along as the key - # the protein projection (a *separate* query, below) is stitched back on. + # The other-nucleotide sibling of the authoritative link: same mapping record + same + # projection_group, but NOT authoritative (so it is the *other* member of the ≤2-member group). Its + # allele carries the canonical HGVS at the level the assay was not measured at. See the join below. + sibling_link = aliased(MappingRecordAllele) + sibling_allele = aliased(Allele) + + # The base per-variant projection: 1:1 annotation joins, so this whole query is a stream of O(N) + # index-scan lookups. MappingRecord.id rides along as the key the protein projection (a *separate* + # query, below) is stitched back on. base_statement = ( select( Variant.urn, @@ -105,10 +174,15 @@ def get_lean_score_set_variants( Variant.hgvs_pro, Variant.hgvs_splice, MappingRecord.id.label("mapping_record_id"), + MappingRecord.assay_level, MappingRecord.hgvs_assay_level, Allele.vrs_digest, Allele.clingen_allele_id, VepAlleleConsequence.functional_consequence, + sibling_allele.level.label("sibling_level"), + # Exactly one of the sibling's hgvs_g/hgvs_c is populated (it is a nucleotide allele), so + # coalesce yields its canonical string regardless of which nucleotide level it sits at. + func.coalesce(sibling_allele.hgvs_g, sibling_allele.hgvs_c).label("sibling_hgvs"), ) # Variant -> its live mapping record. LEFT join so unmapped variants stay (with null mapped fields) # for the table + score histogram. live_at rides in the ON clause, never a WHERE: a WHERE would @@ -132,6 +206,24 @@ def get_lean_score_set_variants( .outerjoin( VepAlleleConsequence, and_(VepAlleleConsequence.allele_id == Allele.id, VepAlleleConsequence.live_at(as_of)) ) + # The authoritative link's projection_group sibling — the canonical projection at the *other* + # nucleotide level. UNLIKE the protein subquery, this stays a direct 1:1 outer join: a group has + # ≤2 members and the authoritative link is one of them, so at most one non-authoritative sibling + # shares its (mapping_record_id, projection_group). The mapping_record_id predicate rides the + # existing ix_mapping_record_alleles_mapping_record_id index; within a record only a handful of + # links are scanned, so no O(N^2) rescan (contrast the opaque-cardinality protein subquery below). + # A NULL authoritative group (protein assay, or pre-reverse-translation data) matches nothing — + # SQL equality on NULL is never true — so the sibling stays null and the slot degrades gracefully. + .outerjoin( + sibling_link, + and_( + sibling_link.mapping_record_id == MappingRecord.id, + sibling_link.projection_group == MappingRecordAllele.projection_group, + sibling_link.is_authoritative.is_(False), + sibling_link.live_at(as_of), + ), + ) + .outerjoin(sibling_allele, sibling_allele.id == sibling_link.allele_id) # The anchor: just this score set's variants. .where(Variant.score_set_id == score_set.id) # Natural table order: variant number (the integer after '#'), id breaks ties stably. @@ -173,7 +265,9 @@ def get_lean_score_set_variants( ) # Keyed by record id (a globally-unique PK -> the right protein row, no cross-set risk). Some records # have no protein projection (UTR/intronic) -> absent from the map -> null protein field below. - protein_hgvs_by_record = {row.mapping_record_id: row.protein_hgvs for row in db.execute(protein_statement)} + protein_hgvs_by_record: dict[int, str] = { + row.mapping_record_id: row.protein_hgvs for row in db.execute(protein_statement) + } return [ LeanVariantRecord( @@ -185,8 +279,14 @@ def get_lean_score_set_variants( hgvs_nt=_hgvs_field_for_str(row.hgvs_nt), hgvs_pro=_hgvs_field_for_str(row.hgvs_pro), hgvs_splice=_hgvs_field_for_str(row.hgvs_splice), - assay_level_hgvs=_hgvs_field_for_str(row.hgvs_assay_level), - protein_level_hgvs=_hgvs_field_for_str(protein_hgvs_by_record.get(row.mapping_record_id)), + assay_level=row.assay_level, + mapped=_mapped_triple( + assay_level=row.assay_level, + assay_level_hgvs=row.hgvs_assay_level, + sibling_level=row.sibling_level, + sibling_hgvs=row.sibling_hgvs, + protein_hgvs=protein_hgvs_by_record.get(row.mapping_record_id), + ), ) for row in db.execute(base_statement) ] diff --git a/src/mavedb/lib/variant_detail.py b/src/mavedb/lib/variant_detail.py index 30f3fbc3b..e833545a8 100644 --- a/src/mavedb/lib/variant_detail.py +++ b/src/mavedb/lib/variant_detail.py @@ -18,6 +18,7 @@ from dataclasses import dataclass from datetime import datetime +from enum import Enum from typing import Any, Optional from sqlalchemy import select @@ -53,22 +54,54 @@ class VariantClassificationRecord: classification: ScoreCalibrationFunctionalClassification +class AlleleDerivation(str, Enum): + """How an allele's representation was arrived at — its confidence/provenance axis. + + Distinct from (orthogonal to) the Cat-VRS ``relation`` axis: ``relation`` is *structural* (which + level relates to which, member→defining), whereas ``derivation`` is *epistemic* (how much to trust + the representation). See the design's "Semantics note — two axes, keep them separate". Derived at + serialization time from ``is_authoritative`` + the assay level — no stored column. + """ + + # The assay's actual measurement (the authoritative link). Precise by definition. + AUTHORITATIVE = "authoritative" + # Deterministic and precise. Every allele derived from a *nucleotide* measurement is a projection: + # nucleotide↔nucleotide and nucleotide→protein are both deterministic given (assembly, transcript). + PROJECTION = "projection" + # Reverse-translation output of a *protein* measurement — genuinely ambiguous (many synonymous + # codons). One member of the fanned-out equivalence class, not a precise coordinate. + CANDIDATE = "candidate" + + @dataclass(frozen=True) class AlleleIdentity: """The MaveDB molecular-identity facts for one of the variant's linked alleles. Rides alongside the spec-pure Cat-VRS keyed by VRS digest (the ``alleles`` map). ``level`` and ``hgvs`` (the reference-frame HGVS, exactly one of the allele's genomic/coding/protein columns) are - what the UI labels the per-level annotation panel by — *never* the digest. ``relation`` is this - allele's relation to the measured (defining) allele; ``None`` when it *is* the measured allele, or - when the allele is not a Cat-VRS member. (The Cat-VRS relation is a *member*→defining code, hence - absent for the defining allele — see ``cat_vrs.CategoricalVariantTransit.member_relations``.) + what the UI labels the per-level annotation panel by — *never* the digest. + + Three axes ride here, deliberately independent: + + - ``relation`` (Cat-VRS, structural): this allele's member→defining relation + (``is_protein_of`` / ``coordinate_representation_of`` / …). ``None`` when it *is* the measured + allele, or when the allele is not a Cat-VRS member. Sourced from + ``cat_vrs.CategoricalVariantTransit.member_relations``. + - ``derivation`` (provenance): :class:`AlleleDerivation` — authoritative / projection / + candidate. Orthogonal to ``relation``; never conflate the two (a protein member of a nucleotide + assay has ``relation=translation_of`` but ``derivation=projection``, whereas the same-shaped + member of a protein assay is ``derivation=candidate``). + - ``projection_of`` (provenance): the VRS digest of this allele's projection sibling — the ≤1 *other* + member of its ``projection_group`` (a c↔g pair). ``None`` for the protein apex (group ``NULL``), + for pre-reverse-translation data, and where a level's projection failed. """ level: Optional[str] hgvs: Optional[str] clingen_allele_id: Optional[str] relation: Optional[str] + derivation: Optional[str] + projection_of: Optional[str] @dataclass(frozen=True) @@ -137,6 +170,29 @@ def _classifications_for_variant( ] +def _derivation_for(*, is_authoritative: bool, assay_level: Optional[str]) -> AlleleDerivation: + """The provenance of a linked allele's representation, from ``is_authoritative`` + the assay level. + + The measured allele is ``authoritative``. Every *other* allele's confidence is set by the *only* + ambiguous boundary in the stack — protein → codon: + + - **nucleotide assay** (``assay_level`` genomic/cdna) → ``projection`` for all derived alleles. + nucleotide↔nucleotide and nucleotide→protein are deterministic, so the whole equivalence class + is precise. + - **protein assay** (``assay_level`` protein) → ``candidate`` for the derived (nucleotide) fan-out. + Reverse translation is genuinely ambiguous. + + Only ``assay_level`` (not the allele's own level) distinguishes the two, because on a nucleotide + assay every derived level is deterministic and on a protein assay every derived allele is part of + the ambiguous nucleotide fan-out. + """ + if is_authoritative: + return AlleleDerivation.AUTHORITATIVE + if assay_level == AnnotationLayer.protein.value: + return AlleleDerivation.CANDIDATE + return AlleleDerivation.PROJECTION + + def _submitted_assay_level_hgvs(variant: Variant, assay_level: Optional[str]) -> Optional[str]: """The depositor-submitted HGVS in the variant's assay frame: protein for a protein assay, otherwise the nucleotide expression (genomic or coding share ``hgvs_nt`` and ``hgvs_splice`` is @@ -199,18 +255,37 @@ def get_variant_detail( # record-scoped, all-levels link set that seeds `annotations`, so the two maps share keys). Carries # the molecular-identity facts the UI labels annotations by — level, reference-frame HGVS (exactly one # of hgvs_g/c/p is populated per allele), ClinGen id — and the member→defining relation. + # Within-record projection grouping: projection_group -> the digests sharing it. A link's projection + # sibling is the ≤1 *other* digest in its group (the c↔g pair); the protein apex and any pre-RT link + # carry a NULL group and so appear in no bucket (projection_of stays None for them). + group_digests: dict[int, list[str]] = {} + for link in links: + if link.projection_group is not None and link.allele.vrs_digest is not None: + group_digests.setdefault(link.projection_group, []).append(link.allele.vrs_digest) + alleles: dict[str, AlleleIdentity] = {} for link in links: allele = link.allele if allele.vrs_digest is None: continue + # The projection sibling: the other digest in this link's projection_group, if any. A group has + # ≤2 members, so there is at most one sibling — this allele's projection partner. + projection_of: Optional[str] = None + if link.projection_group is not None: + projection_of = next( + (digest for digest in group_digests.get(link.projection_group, []) if digest != allele.vrs_digest), + None, + ) + relation = relations.get(allele.vrs_digest) alleles[allele.vrs_digest] = AlleleIdentity( level=allele.level, hgvs=allele.hgvs_g or allele.hgvs_c or allele.hgvs_p, clingen_allele_id=allele.clingen_allele_id, relation=relation.value if relation is not None else None, + derivation=_derivation_for(is_authoritative=link.is_authoritative, assay_level=assay_level).value, + projection_of=projection_of, ) return VariantDetail( diff --git a/src/mavedb/models/mapping_record_allele.py b/src/mavedb/models/mapping_record_allele.py index 7bfbd7885..5179877c2 100644 --- a/src/mavedb/models/mapping_record_allele.py +++ b/src/mavedb/models/mapping_record_allele.py @@ -31,7 +31,21 @@ class MappingRecordAllele(ValidTime, Base): # synonymous translator. The flag lives on the link rather than on # ``Allele`` because the same VRS allele can be authoritative for one # mapping record and translator-derived for another. - is_authoritative = Column(Boolean, nullable=False, default=False) + is_authoritative: Mapped[bool] = Column(Boolean, nullable=False, default=False) + # Within-record projection grouping. The reverse-translation job fans a variant's protein + # consequence out into projection pairs, each a coding/genomic pair (the same change expressed + # at two levels). The two links of one pair share a ``projection_group`` value + # (an integer index, 0..N-1, local to this mapping record), so we can reconstruct projections + # onto transcripts for more precise provenance in consumers. ``NULL`` for links that belong to no + # pair: the shared protein apex, and any link written before reverse translation has run. + # + # It is a grouping key, NOT a cross-record identity — regenerated whenever a record is + # superseded and re-linked. Identity lives in the allele VRS digests; two records that happen to + # share a transcript and codon re-encode the pairing independently. Serving resolves the + # canonical projection by pairing this with ``is_authoritative``: the authoritative (measured) + # allele's group gathers its sibling links, yielding its c/g projection. See the RT job for the + # group-assignment and authoritative fold-in logic. + projection_group = Column(Integer, nullable=True) mapping_record: Mapped["MappingRecord"] = relationship("MappingRecord", back_populates="allele_links") allele: Mapped["Allele"] = relationship("Allele", back_populates="mapping_record_links") diff --git a/src/mavedb/view_models/lean_variant.py b/src/mavedb/view_models/lean_variant.py index 029a217a6..31f163524 100644 --- a/src/mavedb/view_models/lean_variant.py +++ b/src/mavedb/view_models/lean_variant.py @@ -3,8 +3,9 @@ The pydantic serialization boundary over the ``lib.score_set_variants`` transit dataclasses. ``from_attributes`` lets the route return the ``LeanVariantRecord`` dataclasses directly and have FastAPI's ``response_model`` coerce them (field names line up); aliases camelize per the shared base config, so clients see -``variantUrn`` / ``assayLevelHgvs`` etc. ``response_model_exclude_none`` drops absent fields, so a slot -with no parsed block serializes as just ``{"hgvs": "..."}``. +``variantUrn`` / ``assayLevel`` / ``mapped`` etc. ``response_model_exclude_none`` drops absent fields, so a +slot with no parsed block serializes as just ``{"hgvs": "..."}`` and a null mapped level drops out of the +``mapped`` object entirely. """ from typing import Optional @@ -28,15 +29,32 @@ class Config: from_attributes = True +class MappedTriple(BaseModel): + """The mapped (reference-frame) HGVS keyed by level — the canonical projection of the measured change. + + One slot per level. A nucleotide assay populates all three (``mapped[assayLevel]`` is the measured + slot; ``cdna`` is the level-invariant search key, present even when ``assayLevel`` is ``genomic``); a + protein assay populates only ``protein`` (the ambiguous c/g fan-out is not fabricated). Null slots are + omitted under ``response_model_exclude_none``. + """ + + genomic: Optional[HgvsField] = None + cdna: Optional[HgvsField] = None + protein: Optional[HgvsField] = None + + class Config: + from_attributes = True + + class LeanVariant(BaseModel): """One pre-chewed per-variant record feeding the score-set table, heatmap, and histograms. ``variantUrn`` is the universal selection key; ``assayLevelDigest`` bridges into the digest-keyed - annotation dimensions. The submitted HGVS (``hgvsNt``/``hgvsPro``/``hgvsSplice``, target frame) and - the mapped ``assayLevelHgvs`` (reference frame) carry both sides of the heatmap's frame toggle, each - with an optional parsed block for the level toggle. ``proteinLevelHgvs`` is the mapped protein - representation (distinct from the *submitted* ``hgvsPro``); for a protein assay it coincides with - ``assayLevelHgvs``. Fields are omitted when null. + annotation dimensions. The submitted HGVS (``hgvsNt``/``hgvsPro``/``hgvsSplice``, target frame) carry + the depositor's frame for the heatmap's raw↔mapped toggle. The mapped (reference) frame is the + ``mapped`` :class:`MappedTriple` plus the ``assayLevel`` pointer (an ``AnnotationLayer`` value) naming + the measured/canonical slot: ``mapped[assayLevel]`` is the measured representation and ``mapped.cdna`` + the level-invariant search key. Fields are omitted when null. """ variant_urn: str @@ -47,8 +65,8 @@ class LeanVariant(BaseModel): hgvs_nt: Optional[HgvsField] = None hgvs_pro: Optional[HgvsField] = None hgvs_splice: Optional[HgvsField] = None - assay_level_hgvs: Optional[HgvsField] = None - protein_level_hgvs: Optional[HgvsField] = None + assay_level: Optional[str] = None + mapped: MappedTriple = MappedTriple() class Config: from_attributes = True diff --git a/src/mavedb/view_models/variant_detail.py b/src/mavedb/view_models/variant_detail.py index b4c5fa5b2..1ae03def8 100644 --- a/src/mavedb/view_models/variant_detail.py +++ b/src/mavedb/view_models/variant_detail.py @@ -34,14 +34,23 @@ class AlleleIdentity(BaseModel): An entry of the ``alleles`` sidecar, keyed by VRS digest. ``level`` + ``hgvs`` (the reference-frame HGVS) are what the UI labels the per-level annotation panel by — never the digest (design §7.5). - ``relation`` is this allele's relation to the measured (defining) allele; ``null`` when it *is* the - measured allele, or when the allele is not a Cat-VRS member. + + Three independent axes: + - ``relation`` (Cat-VRS, structural): member→defining relation; ``null`` when it *is* the measured + allele, or when the allele is not a Cat-VRS member. + - ``derivation`` (provenance): ``authoritative`` (measured) / ``projection`` (deterministic, + precise) / ``candidate`` (reverse-translation, ambiguous). Orthogonal to ``relation`` — never + conflate them. + - ``projectionOf`` (provenance): the VRS digest of this allele's projection sibling (the paired c↔g member of + its projection pair group); ``null`` for the protein apex and pre-reverse-translation data. """ level: Optional[str] = None hgvs: Optional[str] = None clingen_allele_id: Optional[str] = None relation: Optional[str] = None + derivation: Optional[str] = None + projection_of: Optional[str] = None class Config: from_attributes = True diff --git a/src/mavedb/worker/jobs/variant_processing/reverse_translation.py b/src/mavedb/worker/jobs/variant_processing/reverse_translation.py index d7f17e919..8d861270a 100644 --- a/src/mavedb/worker/jobs/variant_processing/reverse_translation.py +++ b/src/mavedb/worker/jobs/variant_processing/reverse_translation.py @@ -1,10 +1,19 @@ """Reverse translation worker job — builds cross-level HGVS equivalence classes. For each mapped variant in a score set, calls construct_equivalent_variants from -the variant-annotation library to produce all coding and genomic HGVS candidates -encoding the same protein consequence, plus that protein consequence itself. The -candidates (coding, genomic, and the protein) are written as non-authoritative -Allele rows linked to the existing MappingRecord via MappingRecordAllele. +the variant-annotation library to produce the coding/genomic HGVS candidates +encoding the same protein consequence, plus that protein consequence itself. Each +candidate is written as a non-authoritative Allele row linked to the existing +MappingRecord via MappingRecordAllele. + +The library returns the equivalence class as a list of ProjectionPair objects, +each one projection pair (a coding candidate and its deterministic genomic projection +— the same change at two levels). The job preserves that pairing by stamping both +links of a projection pair with a shared per-record ``projection_group`` id; the protein +apex is shared across all projection pairs and carries no group. Where a projection pair member +equals the record's authoritative (measured) allele, its group is folded onto the +existing authoritative link rather than duplicated, so the canonical c/g projection +is resolvable from the authoritative allele's group downstream. """ import asyncio @@ -355,19 +364,22 @@ async def reverse_translate_variants_for_score_set( # superseded atomically — retire and insert share a timestamp with no gap. new_links: list[MappingRecordAllele] = [] - # The live authoritative (record, allele) pairs. A derived candidate that equals a record's - # authoritative allele is not linked again. - authoritative_pairs: set[tuple[int, int]] = { - (record_id, allele_id) - for record_id, allele_id in job_manager.db.execute( - select(MappingRecordAllele.mapping_record_id, MappingRecordAllele.allele_id) + # The live authoritative links for the current records, keyed by (record_id, allele_id). + # Held as ORM objects (not a bare pair set) because the authoritative fold-in updates them in + # place: when an RT candidate's allele equals a record's authoritative (measured) allele, that + # link already exists (is_authoritative=True, projection_group NULL, written by the mapping job) + # and uq_mapping_record_alleles_live forbids a derived duplicate — so the candidate's group is + # stamped onto this existing link instead of inserting a new one. Modifying the loaded object + # emits an UPDATE on flush; supersede_live_where at the end only retires derived links, leaving + # the authoritative link (and its freshly stamped group) intact. + authoritative_links: dict[tuple[int, int], MappingRecordAllele] = { + (link.mapping_record_id, link.allele_id): link + for link in job_manager.db.scalars( + select(MappingRecordAllele) .where(MappingRecordAllele.is_authoritative.is_(True)) .where(MappingRecordAllele.current) .where(MappingRecordAllele.mapping_record_id.in_(current_record_ids)) - ) - .tuples() - .all() - if record_id is not None and allele_id is not None + ).all() } for result in results: @@ -378,17 +390,28 @@ async def reverse_translate_variants_for_score_set( # Dedup by vrs_digest per mapping record to avoid duplicate links. seen_digests: set[str] = set() failed_candidates: list[dict[str, str]] = [] - candidates: list[tuple[str, AnnotationLayer, str]] = [ - (hgvs_g, AnnotationLayer.genomic, "hgvs_g") for hgvs_g in result.hgvs_g_candidates - ] + [(hgvs_c, AnnotationLayer.cdna, "hgvs_c") for hgvs_c in result.hgvs_c_candidates] - # Emit the protein consequence as the protein-level member of the equivalence set. - # Prediction parens (p.(Ala222Val)) are stripped before translation and storage. - # None for protein-assay inputs where the protein is already the authoritative allele. + # Flatten the projection pairs into a work list of (hgvs, level, field, projection_group) members, + # preserving the coding↔genomic pairing the library emits. Each ProjectionPair's coding and + # genomic members share the pair's per-record group id — its index in the equivalence class + # (0..N-1). hgvs_c is always present (the pair key); hgvs_g is None when the c→g projection + # failed, yielding a well-formed one-member (coding only) group rather than a desync. The + # group id is assigned once per pair here, so the two members are guaranteed to carry the + # same id even though they translate and link independently below. + members: list[tuple[str, AnnotationLayer, str, int | None]] = [] + for group_id, pair in enumerate(result.projection_pairs): + members.append((pair.hgvs_c, AnnotationLayer.cdna, "hgvs_c", group_id)) + if pair.hgvs_g is not None: + members.append((pair.hgvs_g, AnnotationLayer.genomic, "hgvs_g", group_id)) + + # The protein consequence is the apex of the equivalence set — shared across every + # pair, a member of none — so it carries no group (None). Prediction parens + # (p.(Ala222Val)) are stripped before translation and storage. None for protein-assay + # inputs, where the protein is already the authoritative allele. if result.hgvs_p: - candidates.append((strip_protein_prediction_parens(result.hgvs_p), AnnotationLayer.protein, "hgvs_p")) + members.append((strip_protein_prediction_parens(result.hgvs_p), AnnotationLayer.protein, "hgvs_p", None)) - for hgvs, level, hgvs_field in candidates: + for hgvs, level, hgvs_field, projection_group in members: # A candidate may not be translatable (intronic projection, malformed expression). # Track per-candidate failures; a variant fails only if *all* candidates fail. try: @@ -415,14 +438,20 @@ async def reverse_translate_variants_for_score_set( allele = get_or_create_allele(job_manager.db, draft_allele) job_manager.db.flush() - if (rec.id, allele.id) in authoritative_pairs: - continue # already linked + authoritative_link = authoritative_links.get((rec.id, allele.id)) + if authoritative_link is not None: + # Authoritative fold-in: this member is the record's measured allele, already + # linked authoritatively. Stamp it's projection group to preserve the c↔g pairing, + # and skip creating a new derived link. + authoritative_link.projection_group = projection_group + continue new_links.append( MappingRecordAllele( mapping_record_id=rec.id, allele_id=allele.id, is_authoritative=False, + projection_group=projection_group, ) ) candidate_count += 1 @@ -430,8 +459,11 @@ async def reverse_translate_variants_for_score_set( annotation_counts["alleles_created"] += candidate_count annotation_metadata = { "hgvs_input": result.input.hgvs, - "hgvs_c_candidates": result.hgvs_c_candidates, - "hgvs_g_candidates": result.hgvs_g_candidates, + # Serializable projection of the projection pairs (the pairing the links now encode). + "candidates": [ + {"hgvs_c": pair.hgvs_c, "hgvs_g": pair.hgvs_g, "variant_type": pair.variant_type} + for pair in result.projection_pairs + ], "hgvs_p": result.hgvs_p, "alleles_created": candidate_count, "failed_candidates": failed_candidates, diff --git a/tests/helpers/util/annotation.py b/tests/helpers/util/annotation.py index 7e80a7c73..f845cdc1c 100644 --- a/tests/helpers/util/annotation.py +++ b/tests/helpers/util/annotation.py @@ -47,6 +47,7 @@ class AlleleSpec: hgvs_g: Optional[str] = None post_mapped: Optional[dict] = None vep_consequence: Optional[str] = None + projection_group: Optional[int] = None clinvar_control_ids: Sequence[int] = field(default_factory=tuple) gnomad_variant_ids: Sequence[int] = field(default_factory=tuple) @@ -99,7 +100,10 @@ def seed_mapping_record( links: list[Any] = [ MappingRecordAllele( - mapping_record_id=record.id, allele_id=allele.id, is_authoritative=spec.is_authoritative + mapping_record_id=record.id, + allele_id=allele.id, + is_authoritative=spec.is_authoritative, + projection_group=spec.projection_group, ) ] if spec.vep_consequence is not None: diff --git a/tests/lib/test_score_set_variants.py b/tests/lib/test_score_set_variants.py index 22f4f43d0..4c0cbb5b0 100644 --- a/tests/lib/test_score_set_variants.py +++ b/tests/lib/test_score_set_variants.py @@ -15,7 +15,7 @@ pytest.importorskip("psycopg2") -from mavedb.lib.score_set_variants import HgvsField, get_lean_score_set_variants +from mavedb.lib.score_set_variants import HgvsField, MappedTriple, get_lean_score_set_variants from mavedb.models.allele import Allele from mavedb.models.mapping_record import MappingRecord from mavedb.models.mapping_record_allele import MappingRecordAllele @@ -46,12 +46,14 @@ def _variant(session, score_set, suffix, *, score=None, hgvs_nt=None, hgvs_pro=N return variant -def _allele(session, digest, *, level="cdna", clingen_allele_id=None, hgvs_p=None): +def _allele(session, digest, *, level="cdna", clingen_allele_id=None, hgvs_c=None, hgvs_g=None, hgvs_p=None): allele = Allele( vrs_digest=digest, level=level, post_mapped={"type": "Allele"}, clingen_allele_id=clingen_allele_id, + hgvs_c=hgvs_c, + hgvs_g=hgvs_g, hgvs_p=hgvs_p, ) session.add(allele) @@ -72,9 +74,13 @@ def _record(session, variant, *, assay_level="cdna", hgvs_assay_level=None, vali return record -def _link(session, record, allele, *, is_authoritative=True, valid_from=None): +def _link(session, record, allele, *, is_authoritative=True, projection_group=None, valid_from=None): link = MappingRecordAllele( - mapping_record_id=record.id, allele_id=allele.id, is_authoritative=is_authoritative, valid_from=valid_from + mapping_record_id=record.id, + allele_id=allele.id, + is_authoritative=is_authoritative, + projection_group=projection_group, + valid_from=valid_from, ) session.add(link) session.commit() @@ -91,15 +97,20 @@ def _consequence(session, allele, value): @pytest.mark.integration -def test_nucleotide_assay(session, setup_lib_db_with_score_set): - """Coding assay: submitted nt+pro from the variant, mapped coding string from the record, and the - digest/ClinGen id/consequence off the authoritative allele. Each parseable string gets its block.""" +def test_nucleotide_assay_measured_at_cdna(session, setup_lib_db_with_score_set): + """Coding assay: submitted nt+pro from the variant; the mapped triple carries the measured cdna slot + (from the record), its projection_group genomic sibling, and the protein apex; digest/ClinGen + id/consequence off the authoritative allele. Each parseable string gets its block.""" score_set = setup_lib_db_with_score_set variant = _variant(session, score_set, 1, score=-2.3, hgvs_nt="c.1A>T", hgvs_pro="p.Thr1Ser") record = _record(session, variant, assay_level="cdna", hgvs_assay_level="NM_000546.6:c.1216G>A") measured = _allele(session, "cdna-digest", level="cdna", clingen_allele_id="CA123") + genomic = _allele(session, "gen-digest", level="genomic", hgvs_g="NC_000017.11:g.7676154C>T") protein = _allele(session, "prot-digest", level="protein", hgvs_p="NP_000537.3:p.Ala406Thr") - _link(session, record, measured, is_authoritative=True) + # The measured cdna link and its genomic projection share a projection_group (the RT fold-in); the + # protein apex is a member of no pair (group None). + _link(session, record, measured, is_authoritative=True, projection_group=0) + _link(session, record, genomic, is_authoritative=False, projection_group=0) _link(session, record, protein, is_authoritative=False) _consequence(session, measured, "missense_variant") @@ -113,17 +124,45 @@ def test_nucleotide_assay(session, setup_lib_db_with_score_set): assert record_out.hgvs_nt == HgvsField(hgvs="c.1A>T", position=1, ref="A", alt="T") assert record_out.hgvs_pro == HgvsField(hgvs="p.Thr1Ser", position=1, ref="Thr", alt="Ser") assert record_out.hgvs_splice is None - assert record_out.assay_level_hgvs == HgvsField(hgvs="NM_000546.6:c.1216G>A", position=1216, ref="G", alt="A") - # The mapped protein comes off the protein-level member allele's hgvs_p, not the measured coding allele. - assert record_out.protein_level_hgvs == HgvsField( - hgvs="NP_000537.3:p.Ala406Thr", position=406, ref="Ala", alt="Thr" + # assayLevel names the measured slot; the triple fills all three levels. + assert record_out.assay_level == "cdna" + assert record_out.mapped == MappedTriple( + cdna=HgvsField(hgvs="NM_000546.6:c.1216G>A", position=1216, ref="G", alt="A"), + genomic=HgvsField(hgvs="NC_000017.11:g.7676154C>T", position=7676154, ref="C", alt="T"), + protein=HgvsField(hgvs="NP_000537.3:p.Ala406Thr", position=406, ref="Ala", alt="Thr"), ) @pytest.mark.integration -def test_protein_assay_maps_assay_level_to_a_protein_block(session, setup_lib_db_with_score_set): - """Protein assay: the mapped assay-level string is a p. expression, parsed into a protein block; the - digest comes off the authoritative protein allele.""" +def test_nucleotide_assay_measured_at_genomic(session, setup_lib_db_with_score_set): + """Genomic assay: the measured slot is genomic (from the record); the cdna slot — the level-invariant + search key — comes off the projection_group sibling even though the assay was measured at genomic.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, score=0.1, hgvs_nt="g.7676154C>T") + record = _record(session, variant, assay_level="genomic", hgvs_assay_level="NC_000017.11:g.7676154C>T") + measured = _allele(session, "gen-digest", level="genomic", clingen_allele_id="CA9") + cdna = _allele(session, "cdna-digest", level="cdna", hgvs_c="NM_000546.6:c.1216G>A") + protein = _allele(session, "prot-digest", level="protein", hgvs_p="NP_000537.3:p.Ala406Thr") + _link(session, record, measured, is_authoritative=True, projection_group=0) + _link(session, record, cdna, is_authoritative=False, projection_group=0) + _link(session, record, protein, is_authoritative=False) + + [record_out] = get_lean_score_set_variants(session, score_set) + + assert record_out.assay_level == "genomic" + assert record_out.assay_level_digest == "gen-digest" + # Measured slot is genomic (from the record); cdna (search key) comes off the sibling allele. + assert record_out.mapped == MappedTriple( + genomic=HgvsField(hgvs="NC_000017.11:g.7676154C>T", position=7676154, ref="C", alt="T"), + cdna=HgvsField(hgvs="NM_000546.6:c.1216G>A", position=1216, ref="G", alt="A"), + protein=HgvsField(hgvs="NP_000537.3:p.Ala406Thr", position=406, ref="Ala", alt="Thr"), + ) + + +@pytest.mark.integration +def test_protein_assay_serves_only_the_protein_slot(session, setup_lib_db_with_score_set): + """Protein assay: the measured slot *is* protein, so only the protein slot is populated; the cdna and + genomic slots stay null (the c/g fan-out is ambiguous — no canonical pick is fabricated).""" score_set = setup_lib_db_with_score_set variant = _variant(session, score_set, 1, score=1.0, hgvs_pro="p.Ala406Thr") record = _record(session, variant, assay_level="protein", hgvs_assay_level="NP_000537.3:p.Ala406Thr") @@ -131,15 +170,43 @@ def test_protein_assay_maps_assay_level_to_a_protein_block(session, setup_lib_db session, "prot-digest", level="protein", clingen_allele_id="PA9", hgvs_p="NP_000537.3:p.Ala406Thr" ) _link(session, record, measured, is_authoritative=True) + # A candidate fan-out member (cdna) with its own projection pair group — the protein authoritative link is + # in no group, so this candidate must NOT be pulled into the canonical triple. + candidate = _allele(session, "cand-digest", level="cdna", hgvs_c="NM_000546.6:c.1216G>A") + _link(session, record, candidate, is_authoritative=False, projection_group=0) [record_out] = get_lean_score_set_variants(session, score_set) + assert record_out.assay_level == "protein" assert record_out.assay_level_digest == "prot-digest" assert record_out.clingen_allele_id == "PA9" - block = HgvsField(hgvs="NP_000537.3:p.Ala406Thr", position=406, ref="Ala", alt="Thr") - # For a protein assay the measured allele *is* the protein one, so the two mapped fields coincide. - assert record_out.assay_level_hgvs == block - assert record_out.protein_level_hgvs == block + # Only the protein slot; cdna/genomic stay null despite the candidate fan-out being linked. + assert record_out.mapped == MappedTriple( + protein=HgvsField(hgvs="NP_000537.3:p.Ala406Thr", position=406, ref="Ala", alt="Thr") + ) + + +@pytest.mark.integration +def test_unpopulated_projection_group_degrades_to_a_null_sibling(session, setup_lib_db_with_score_set): + """Pre-reverse-translation data (no projection_group on the authoritative link): the sibling join + returns nothing, so the other nucleotide slot stays null — today's behavior — while the measured and + protein slots still populate. Still one row per variant.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, score=-2.3, hgvs_nt="c.1A>T") + record = _record(session, variant, assay_level="cdna", hgvs_assay_level="NM_000546.6:c.1216G>A") + measured = _allele(session, "cdna-digest", level="cdna", clingen_allele_id="CA123") + protein = _allele(session, "prot-digest", level="protein", hgvs_p="NP_000537.3:p.Ala406Thr") + _link(session, record, measured, is_authoritative=True) # projection_group defaults to None + _link(session, record, protein, is_authoritative=False) + + [record_out] = get_lean_score_set_variants(session, score_set) + + assert record_out.assay_level == "cdna" + # Measured cdna + protein apex populate; genomic is null (no recorded sibling — graceful degradation). + assert record_out.mapped == MappedTriple( + cdna=HgvsField(hgvs="NM_000546.6:c.1216G>A", position=1216, ref="G", alt="A"), + protein=HgvsField(hgvs="NP_000537.3:p.Ala406Thr", position=406, ref="Ala", alt="Thr"), + ) @pytest.mark.integration @@ -157,7 +224,8 @@ def test_hgvs_string_rides_even_when_unparseable(session, setup_lib_db_with_scor assert record_out.hgvs_nt == HgvsField(hgvs="NM_000546.6:c.[197A>G;472T>C]") assert record_out.hgvs_nt.position is None assert record_out.hgvs_splice == HgvsField(hgvs="c.122-6T>A") - assert record_out.assay_level_hgvs == HgvsField(hgvs="c.76_78del") + # The measured cdna slot carries the (unparseable) indel string alone. + assert record_out.mapped.cdna == HgvsField(hgvs="c.76_78del") @pytest.mark.integration @@ -174,8 +242,8 @@ def test_unmapped_variant_has_null_mapped_fields(session, setup_lib_db_with_scor assert record_out.consequence is None assert record_out.clingen_allele_id is None assert record_out.assay_level_digest is None - assert record_out.assay_level_hgvs is None - assert record_out.protein_level_hgvs is None + assert record_out.assay_level is None + assert record_out.mapped == MappedTriple() # empty triple — no slots populated @pytest.mark.integration @@ -191,7 +259,7 @@ def test_no_vep_consequence_is_none(session, setup_lib_db_with_score_set): assert record_out.consequence is None assert record_out.assay_level_digest == "cdna-digest" - assert record_out.assay_level_hgvs == HgvsField(hgvs="NM_000546.6:c.1216G>A", position=1216, ref="G", alt="A") + assert record_out.mapped.cdna == HgvsField(hgvs="NM_000546.6:c.1216G>A", position=1216, ref="G", alt="A") @pytest.mark.integration @@ -225,11 +293,11 @@ def test_as_of_reconstructs_the_historical_annotation_layer(session, setup_lib_d [current] = get_lean_score_set_variants(session, score_set) assert current.assay_level_digest == "new-digest" - assert current.assay_level_hgvs == HgvsField(hgvs="NM_000546.6:c.500A>T", position=500, ref="A", alt="T") + assert current.mapped.cdna == HgvsField(hgvs="NM_000546.6:c.500A>T", position=500, ref="A", alt="T") [historical] = get_lean_score_set_variants(session, score_set, as_of=T1 - timedelta(days=1)) assert historical.assay_level_digest == "old-digest" - assert historical.assay_level_hgvs == HgvsField(hgvs="NM_000546.6:c.1216G>A", position=1216, ref="G", alt="A") + assert historical.mapped.cdna == HgvsField(hgvs="NM_000546.6:c.1216G>A", position=1216, ref="G", alt="A") # Submitted HGVS and score are immutable — unaffected by as_of. assert historical.hgvs_nt == HgvsField(hgvs="c.1A>T", position=1, ref="A", alt="T") assert historical.score == -2.3 @@ -237,4 +305,5 @@ def test_as_of_reconstructs_the_historical_annotation_layer(session, setup_lib_d # Before anything was mapped, the variant still appears with null mapped fields. [pre_mapping] = get_lean_score_set_variants(session, score_set, as_of=T0 - timedelta(days=1)) assert pre_mapping.assay_level_digest is None - assert pre_mapping.assay_level_hgvs is None + assert pre_mapping.assay_level is None + assert pre_mapping.mapped == MappedTriple() diff --git a/tests/lib/test_variant_detail.py b/tests/lib/test_variant_detail.py index fcf98c31b..fa193b53d 100644 --- a/tests/lib/test_variant_detail.py +++ b/tests/lib/test_variant_detail.py @@ -93,8 +93,13 @@ def _record(session, variant, *, assay_level="cdna", hgvs_assay_level=None): return record -def _link(session, record, allele, *, is_authoritative=False): - link = MappingRecordAllele(mapping_record_id=record.id, allele_id=allele.id, is_authoritative=is_authoritative) +def _link(session, record, allele, *, is_authoritative=False, projection_group=None): + link = MappingRecordAllele( + mapping_record_id=record.id, + allele_id=allele.id, + is_authoritative=is_authoritative, + projection_group=projection_group, + ) session.add(link) session.commit() return link @@ -143,8 +148,12 @@ def test_full_envelope_for_a_coding_assay(session, setup_lib_db_with_score_set): ) record = _record(session, variant, assay_level="cdna", hgvs_assay_level="NM_000546.6:c.1216G>A") measured = _allele(session, "cdna-digest", level="cdna", clingen_allele_id="CA123", hgvs_c="NM_000546.6:c.1216G>A") + genomic = _allele(session, "gen-digest", level="genomic", hgvs_g="NC_000017.11:g.7676154C>T") protein = _allele(session, "prot-digest", level="protein", hgvs_p="NP_000537.3:p.Ala406Thr") - _link(session, record, measured, is_authoritative=True) + # The measured cdna link and its genomic projection share a projection_group (the RT fold-in); the + # protein apex is in no pair (group None). + _link(session, record, measured, is_authoritative=True, projection_group=0) + _link(session, record, genomic, projection_group=0) _link(session, record, protein) _vep(session, measured, "missense_variant") @@ -161,20 +170,33 @@ def test_full_envelope_for_a_coding_assay(session, setup_lib_db_with_score_set): assert detail.molecular_representation is not None assert detail.molecular_representation["type"] == "CategoricalVariant" # The alleles sidecar carries one identity per linked allele, keyed by digest: level + - # reference-frame HGVS (coalesced from hgvs_g/c/p) + ClinGen id + member->defining relation. - assert set(detail.alleles) == {"cdna-digest", "prot-digest"} + # reference-frame HGVS (coalesced from hgvs_g/c/p) + ClinGen id + member->defining relation + + # derivation + projection_of. + assert set(detail.alleles) == {"cdna-digest", "gen-digest", "prot-digest"} measured_identity = detail.alleles["cdna-digest"] assert measured_identity.level == "cdna" assert measured_identity.hgvs == "NM_000546.6:c.1216G>A" # coalesced from hgvs_c assert measured_identity.clingen_allele_id == "CA123" assert measured_identity.relation is None # the defining allele has no relation to itself + # The measured allele is authoritative and pairs with its genomic projection sibling. + assert measured_identity.derivation == "authoritative" + assert measured_identity.projection_of == "gen-digest" + genomic_identity = detail.alleles["gen-digest"] + # Nucleotide assay: the genomic sibling is a precise projection, paired back to the measured cdna. + assert genomic_identity.level == "genomic" + assert genomic_identity.derivation == "projection" + assert genomic_identity.projection_of == "cdna-digest" protein_identity = detail.alleles["prot-digest"] assert protein_identity.level == "protein" assert protein_identity.hgvs == "NP_000537.3:p.Ala406Thr" # coalesced from hgvs_p - # The protein member is a translation of the measured coding allele (member -> defining). + # The protein member is a translation of the measured coding allele (member -> defining)... assert protein_identity.relation == "translation_of" - # Annotations keyed by digest, covering both linked alleles; VEP rode in on the measured allele. - assert set(detail.annotations) == {"cdna-digest", "prot-digest"} + # ...and, on a nucleotide assay, is itself a deterministic projection (axes are independent) with no + # projection sibling (the apex is in no c/g pair). + assert protein_identity.derivation == "projection" + assert protein_identity.projection_of is None + # Annotations keyed by digest, covering all linked alleles; VEP rode in on the measured allele. + assert set(detail.annotations) == {"cdna-digest", "gen-digest", "prot-digest"} assert detail.annotations["cdna-digest"].vep is not None assert detail.annotations["cdna-digest"].vep.consequence == "missense_variant" assert detail.is_current is True @@ -183,15 +205,22 @@ def test_full_envelope_for_a_coding_assay(session, setup_lib_db_with_score_set): @pytest.mark.integration def test_protein_assay_targets_the_protein_frame(session, setup_lib_db_with_score_set): - """Mode 2 (protein measured): assay level is protein, target HGVS is the submitted p. string, and - the nt member `encodes` the defining protein allele.""" + """Mode 2 (protein measured): assay level is protein, target HGVS is the submitted p. string, the nt + members `encode` the defining protein allele, and the fanned-out c/g candidates are labelled + `candidate` (ambiguous) while still pairing to each other via projection_of. The protein apex is in + no pair (group None). The `relation` (structural) and `derivation` (provenance) axes are independent: + the coding candidate is `encodes` + `candidate` here, versus `translation_of` + `projection` for the + protein member of a nucleotide assay.""" score_set = setup_lib_db_with_score_set variant = _variant(session, score_set, 1, hgvs_nt="c.1216G>A", hgvs_pro="p.Ala406Thr") record = _record(session, variant, assay_level="protein", hgvs_assay_level="NP_000537.3:p.Ala406Thr") measured = _allele(session, "prot-digest", level="protein", clingen_allele_id="PA9") - coding = _allele(session, "cdna-digest", level="cdna") - _link(session, record, measured, is_authoritative=True) - _link(session, record, coding) + coding = _allele(session, "cdna-digest", level="cdna", hgvs_c="NM_000546.6:c.1216G>A") + genomic = _allele(session, "gen-digest", level="genomic", hgvs_g="NC_000017.11:g.7676154C>T") + _link(session, record, measured, is_authoritative=True) # protein apex — group None + # One reverse-translation projection pair: a coding candidate paired with its genomic projection. + _link(session, record, coding, projection_group=0) + _link(session, record, genomic, projection_group=0) detail = get_variant_detail(session, variant) @@ -200,11 +229,43 @@ def test_protein_assay_targets_the_protein_frame(session, setup_lib_db_with_scor assert detail.reference_hgvs == "NP_000537.3:p.Ala406Thr" assert detail.assay_level_digest == "prot-digest" assert detail.mode == "reverse_translation" - # Defining protein allele has no relation to itself; the coding member encodes it. - assert detail.alleles["prot-digest"].level == "protein" - assert detail.alleles["prot-digest"].relation is None - assert detail.alleles["cdna-digest"].level == "cdna" - assert detail.alleles["cdna-digest"].relation == "encodes" + # Defining protein allele has no relation to itself, is authoritative, and is in no c/g pair. + prot = detail.alleles["prot-digest"] + assert prot.level == "protein" + assert prot.relation is None + assert prot.derivation == "authoritative" + assert prot.projection_of is None + # The coding member `encodes` the protein (structural relation) but is an ambiguous `candidate` + # (provenance) — the two axes disagree by design — and pairs to its genomic projection. + coding_identity = detail.alleles["cdna-digest"] + assert coding_identity.relation == "encodes" + assert coding_identity.derivation == "candidate" + assert coding_identity.projection_of == "gen-digest" + genomic_identity = detail.alleles["gen-digest"] + assert genomic_identity.relation == "encodes" + assert genomic_identity.derivation == "candidate" + assert genomic_identity.projection_of == "cdna-digest" + + +@pytest.mark.integration +def test_pre_reverse_translation_data_degrades_projection_of_to_null(session, setup_lib_db_with_score_set): + """Links written before reverse translation ran carry a NULL projection_group: derivation is still + computed (authoritative / projection from is_authoritative + assay level), but projection_of degrades + to null — nothing pairs until the data is re-processed.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, hgvs_nt="c.1216G>A") + record = _record(session, variant, assay_level="cdna", hgvs_assay_level="NM_000546.6:c.1216G>A") + measured = _allele(session, "cdna-digest", level="cdna", hgvs_c="NM_000546.6:c.1216G>A") + protein = _allele(session, "prot-digest", level="protein", hgvs_p="NP_000537.3:p.Ala406Thr") + _link(session, record, measured, is_authoritative=True) # no projection_group (pre-RT) + _link(session, record, protein) + + detail = get_variant_detail(session, variant) + + assert detail.alleles["cdna-digest"].derivation == "authoritative" + assert detail.alleles["cdna-digest"].projection_of is None # no group -> no sibling + assert detail.alleles["prot-digest"].derivation == "projection" + assert detail.alleles["prot-digest"].projection_of is None @pytest.mark.integration diff --git a/tests/routers/test_score_set.py b/tests/routers/test_score_set.py index 30e5af017..632c0fa15 100644 --- a/tests/routers/test_score_set.py +++ b/tests/routers/test_score_set.py @@ -4562,7 +4562,8 @@ def test_cannot_fetch_gnomad_variants_for_score_set_when_none_exist( def _seed_lean_mapping(session, variant_urn): """Give one variant a live coding-measured mapping record (with an assay-level HGVS) whose - authoritative allele carries a digest + ClinGen id + a live VEP consequence.""" + authoritative allele carries a digest + ClinGen id + a live VEP consequence, plus its canonical + genomic projection sibling (shared ``projection_group``) and the protein apex — the full triple.""" seed_mapping_record( session, variant_urn, @@ -4575,6 +4576,13 @@ def _seed_lean_mapping(session, variant_urn): is_authoritative=True, clingen_allele_id="CA123", vep_consequence="missense_variant", + projection_group=0, + ), + AlleleSpec( + digest="gen-digest", + level="genomic", + hgvs_g="NC_000017.11:g.7676154C>T", + projection_group=0, ), AlleleSpec(digest="prot-digest", level="protein", hgvs_p="NP_000537.3:p.Ala406Thr"), ], @@ -4605,21 +4613,31 @@ def test_get_lean_variants(client, session, data_provider, data_files, setup_rou # Submitted HGVS (from the uploaded CSV), each with its parsed block riding alongside. assert mapped["hgvsNt"] == {"hgvs": "c.1A>T", "position": 1, "ref": "A", "alt": "T"} assert mapped["hgvsPro"] == {"hgvs": "p.Thr1Ser", "position": 1, "ref": "Thr", "alt": "Ser"} - # Mapped assay-level HGVS (reference frame) and the mapped protein representation. - assert mapped["assayLevelHgvs"] == {"hgvs": "NM_000546.6:c.1216G>A", "position": 1216, "ref": "G", "alt": "A"} - assert mapped["proteinLevelHgvs"] == { + # The mapped triple (reference frame): the measured slot named by assayLevel, its projection_group + # genomic sibling, and the protein apex. mapped.cdna is the search key even here (measured at cdna). + assert mapped["assayLevel"] == "cdna" + assert mapped["mapped"]["cdna"] == {"hgvs": "NM_000546.6:c.1216G>A", "position": 1216, "ref": "G", "alt": "A"} + assert mapped["mapped"]["genomic"] == { + "hgvs": "NC_000017.11:g.7676154C>T", + "position": 7676154, + "ref": "C", + "alt": "T", + } + assert mapped["mapped"]["protein"] == { "hgvs": "NP_000537.3:p.Ala406Thr", "position": 406, "ref": "Ala", "alt": "Thr", } - # An unmapped variant keeps its submitted HGVS + score; the mapped fields are dropped (exclude_none). + # An unmapped variant keeps its submitted HGVS + score; the mapped fields are dropped (exclude_none), + # and the mapped triple serializes empty (no slots) since none is populated. unmapped = records[1] assert unmapped["score"] == 1.0 assert unmapped["hgvsNt"]["hgvs"] == "c.2C>T" - for omitted in ("consequence", "clingenAlleleId", "assayLevelDigest", "assayLevelHgvs"): + for omitted in ("consequence", "clingenAlleleId", "assayLevelDigest", "assayLevel"): assert omitted not in unmapped + assert unmapped["mapped"] == {} def test_get_lean_variants_unknown_score_set_is_404(client, setup_router_db): diff --git a/tests/routers/test_variant.py b/tests/routers/test_variant.py index 8bb1c3503..ea1c01086 100644 --- a/tests/routers/test_variant.py +++ b/tests/routers/test_variant.py @@ -53,7 +53,8 @@ def _post_mapped() -> dict: def _seed_mapping(session, variant_urn): """Give a variant a live coding-measured mapping record whose authoritative allele carries a - digest + ClinGen id + a live VEP consequence, plus a protein member — enough to build Cat-VRS.""" + digest + ClinGen id + a live VEP consequence, its genomic projection sibling (shared + projection_group), and a protein apex — enough to build Cat-VRS and exercise the projection axes.""" variant = session.scalar(select(VariantDbModel).where(VariantDbModel.urn == variant_urn)) record = MappingRecord( variant_id=variant.id, @@ -71,15 +72,25 @@ def _seed_mapping(session, variant_urn): clingen_allele_id="CA123", hgvs_c="NM_000546.6:c.1216G>A", ) + genomic = Allele( + vrs_digest="gen-digest", level="genomic", post_mapped=_post_mapped(), hgvs_g="NC_000017.11:g.7676154C>T" + ) protein = Allele( vrs_digest="prot-digest", level="protein", post_mapped=_post_mapped(), hgvs_p="NP_000537.3:p.Ala406Thr" ) - session.add_all([measured, protein]) + session.add_all([measured, genomic, protein]) session.commit() session.add_all( [ - MappingRecordAllele(mapping_record_id=record.id, allele_id=measured.id, is_authoritative=True), + # The measured cdna link and its genomic projection share a projection_group; the protein + # apex is in no pair (group None). + MappingRecordAllele( + mapping_record_id=record.id, allele_id=measured.id, is_authoritative=True, projection_group=0 + ), + MappingRecordAllele( + mapping_record_id=record.id, allele_id=genomic.id, is_authoritative=False, projection_group=0 + ), MappingRecordAllele(mapping_record_id=record.id, allele_id=protein.id, is_authoritative=False), VepAlleleConsequence( allele_id=measured.id, @@ -120,9 +131,17 @@ def test_get_variant_detail_envelope(client, session, data_provider, data_files, assert body["alleles"]["cdna-digest"]["hgvs"] == "NM_000546.6:c.1216G>A" assert body["alleles"]["cdna-digest"]["clingenAlleleId"] == "CA123" assert "relation" not in body["alleles"]["cdna-digest"] # null relation dropped by exclude_none + # Provenance axis (derivation) + the projection pairing (projectionOf), camelCased. + assert body["alleles"]["cdna-digest"]["derivation"] == "authoritative" + assert body["alleles"]["cdna-digest"]["projectionOf"] == "gen-digest" + assert body["alleles"]["gen-digest"]["derivation"] == "projection" + assert body["alleles"]["gen-digest"]["projectionOf"] == "cdna-digest" assert body["alleles"]["prot-digest"]["level"] == "protein" assert body["alleles"]["prot-digest"]["hgvs"] == "NP_000537.3:p.Ala406Thr" assert body["alleles"]["prot-digest"]["relation"] == "translation_of" + # The apex is a deterministic projection here but pairs with nothing (projectionOf dropped as null). + assert body["alleles"]["prot-digest"]["derivation"] == "projection" + assert "projectionOf" not in body["alleles"]["prot-digest"] assert body["annotations"]["cdna-digest"]["vep"]["consequence"] == "missense_variant" assert body["isCurrent"] is True assert "supersededByScoreSet" not in body # dropped by exclude_none when current diff --git a/tests/worker/jobs/variant_processing/test_reverse_translation.py b/tests/worker/jobs/variant_processing/test_reverse_translation.py index 1a77e8e86..7bc33cf99 100644 --- a/tests/worker/jobs/variant_processing/test_reverse_translation.py +++ b/tests/worker/jobs/variant_processing/test_reverse_translation.py @@ -9,7 +9,12 @@ from datetime import timedelta from unittest.mock import MagicMock, patch -from variant_annotation.lib.translation.types import TranslationError, TranslationResult, WtCodonMode +from variant_annotation.lib.translation.types import ( + ProjectionPair, + TranslationError, + TranslationResult, + WtCodonMode, +) from mavedb.lib.types.workflow import JobExecutionOutcome from mavedb.models.allele import Allele @@ -60,8 +65,17 @@ def fake_construct(results_by_hgvs: dict, errors_by_hgvs: dict | None = None): The job correlates each TranslationResult/TranslationError back to its originating mapping record via object identity on ``result.input``, so the fake must echo the - exact VariantInput instances it was handed. Candidates are keyed by the input's + exact VariantInput instances it was handed. Results are keyed by the input's assay-level HGVS string. + + Each result spec is one of: + + * a list of ``(hgvs_c, hgvs_g)`` pairs — one ProjectionPair each. + ``hgvs_g`` may be ``None`` for a projection-failed, coding-only projection pair. + * a ``(pairs, hgvs_p)`` tuple pairing that list with a shared protein apex. + + Each pair becomes one ProjectionPair, mirroring the position-aligned + coding↔genomic rows the real reverse-translate CLI emits (with ``--one-row-per-input``). """ errors_by_hgvs = errors_by_hgvs or {} @@ -71,14 +85,13 @@ def _construct(inputs, *, transcripts, coordinates, config): if inp.hgvs in errors_by_hgvs: errors.append(TranslationError(input=inp, error=errors_by_hgvs[inp.hgvs])) elif inp.hgvs in results_by_hgvs: - entry = results_by_hgvs[inp.hgvs] - c_candidates, g_candidates = entry[0], entry[1] - hgvs_p = entry[2] if len(entry) > 2 else None + spec = results_by_hgvs[inp.hgvs] + # A bare list is the candidate pairs; a tuple pairs them with a protein apex. + pairs, hgvs_p = (spec[0], spec[1]) if isinstance(spec, tuple) else (spec, None) results.append( TranslationResult( input=inp, - hgvs_c_candidates=list(c_candidates), - hgvs_g_candidates=list(g_candidates), + projection_pairs=[ProjectionPair(hgvs_c=hgvs_c, hgvs_g=hgvs_g) for hgvs_c, hgvs_g in pairs], hgvs_p=hgvs_p, ) ) @@ -163,6 +176,34 @@ def _non_authoritative_links(session): return session.query(MappingRecordAllele).filter(MappingRecordAllele.is_authoritative.is_(False)).all() +def _live_links(session, record_id): + """Every live (valid_to IS NULL) link for a mapping record, authoritative or derived.""" + return ( + session.query(MappingRecordAllele) + .filter( + MappingRecordAllele.mapping_record_id == record_id, + MappingRecordAllele.valid_to.is_(None), + ) + .all() + ) + + +def _authoritative_link(session, record_id): + return ( + session.query(MappingRecordAllele) + .filter( + MappingRecordAllele.mapping_record_id == record_id, + MappingRecordAllele.is_authoritative.is_(True), + MappingRecordAllele.valid_to.is_(None), + ) + .one() + ) + + +def _record_for(session, variant_id): + return session.query(MappingRecord).filter(MappingRecord.variant_id == variant_id).one() + + @pytest.mark.unit class TestBuildTranslationConfig: """Unit tests for _build_translation_config — the optional translation_config job param.""" @@ -241,7 +282,8 @@ async def test_creates_genomic_and_coding_candidate_alleles( sample_score_set, ): """A mapped variant expands into one non-authoritative allele per equivalence-class - candidate, each linked to the originating mapping record.""" + candidate, each linked to the originating mapping record. The two members of a single + projection pair (coding + genomic projection) share one projection_group.""" variant = Variant( score_set_id=sample_score_set.id, urn="variant:1", @@ -256,7 +298,7 @@ async def test_creates_genomic_and_coding_candidate_alleles( assay_hgvs = "NM_000000.1:c.1A>G" g_candidate = "NC_000001.11:g.1000A>G" c_candidate = "NM_000001.1:c.5A>G" - construct = fake_construct({assay_hgvs: ([c_candidate], [g_candidate])}) + construct = fake_construct({assay_hgvs: [(c_candidate, g_candidate)]}) translate = fake_translate({g_candidate: "ga4gh:VA.genomic", c_candidate: "ga4gh:VA.coding"}) with ( @@ -286,12 +328,20 @@ async def test_creates_genomic_and_coding_candidate_alleles( assert coding_allele.transcript == "NM_000001.1" # The candidate alleles are linked to the same mapping record as the authoritative allele. - mapping_record = session.query(MappingRecord).filter(MappingRecord.variant_id == variant.id).one() + mapping_record = _record_for(session, variant.id) assert {link.mapping_record_id for link in non_auth_links} == {mapping_record.id} + # Both members of the single projection pair carry the same (first, index 0) group id — the + # coding↔genomic pairing survives the flatten that Phase B exists to stop. + assert {link.projection_group for link in non_auth_links} == {0} + events = _cross_level_events(session, sample_score_set.id) assert len(events) == 1 assert events[0].disposition == "present" + # The event metadata carries the paired projection pair shape, not the removed flat lists. + assert events[0].event_metadata["candidates"] == [ + {"hgvs_c": c_candidate, "hgvs_g": g_candidate, "variant_type": None} + ] async def test_transcript_is_queryable_via_derived_expression( self, @@ -319,7 +369,7 @@ async def test_transcript_is_queryable_via_derived_expression( assay_hgvs = "NM_000000.1:c.1A>G" g_candidate = "NC_000001.11:g.1000A>G" c_candidate = "NM_000001.1:c.5A>G" - construct = fake_construct({assay_hgvs: ([c_candidate], [g_candidate])}) + construct = fake_construct({assay_hgvs: [(c_candidate, g_candidate)]}) translate = fake_translate({g_candidate: "ga4gh:VA.genomic", c_candidate: "ga4gh:VA.coding"}) with ( @@ -346,7 +396,8 @@ async def test_protein_consequence_is_persisted_as_a_protein_allele( ): """The deterministic protein consequence (``result.hgvs_p``) is emitted as the protein-level member of the equivalence set -- a non-authoritative ``level=protein`` - allele linked to the record, not dropped.""" + allele linked to the record, not dropped. As the apex shared across all projection pairs it + belongs to no projection pair, so its link's projection_group is NULL.""" variant = Variant( score_set_id=sample_score_set.id, urn="variant:1", @@ -364,7 +415,7 @@ async def test_protein_consequence_is_persisted_as_a_protein_allele( # c_to_p emits a predicted consequence in parens; the job strips them before use. p_consequence = "NP_000001.1:p.(Met1Val)" p_stripped = "NP_000001.1:p.Met1Val" - construct = fake_construct({assay_hgvs: ([c_candidate], [g_candidate], p_consequence)}) + construct = fake_construct({assay_hgvs: ([(c_candidate, g_candidate)], p_consequence)}) translate = fake_translate( { g_candidate: "ga4gh:VA.genomic", @@ -382,8 +433,14 @@ async def test_protein_consequence_is_persisted_as_a_protein_allele( protein_allele = session.query(Allele).filter(Allele.vrs_digest == "ga4gh:VA.protein").one() assert protein_allele.level == AnnotationLayer.protein.value assert protein_allele.hgvs_p == p_stripped # stored without the prediction parens - # Linked to the record as a non-authoritative member of the equivalence set. - assert protein_allele.id in {link.allele_id for link in _non_authoritative_links(session)} + + # Linked to the record as a non-authoritative member of the equivalence set, but grouped + # with no pair (the apex is shared across every projection pair). + protein_link = next(link for link in _non_authoritative_links(session) if link.allele_id == protein_allele.id) + assert protein_link.projection_group is None + # The coding/genomic pair is still grouped together (group 0). + pair_links = [link for link in _non_authoritative_links(session) if link.allele_id != protein_allele.id] + assert {link.projection_group for link in pair_links} == {0} async def test_cis_phased_multivariant_candidate_is_persisted_as_a_block( self, @@ -395,8 +452,9 @@ async def test_cis_phased_multivariant_candidate_is_persisted_as_a_block( sample_independent_reverse_translation_run, sample_score_set, ): - """A bracketed genomic candidate (non-adjacent codon components) is translated into a - CisPhasedBlock and persisted like any other candidate allele, keyed by its CPB digest.""" + """A bracketed genomic projection (non-adjacent codon components) is translated into a + CisPhasedBlock and persisted like any other candidate allele, keyed by its CPB digest, and + grouped with its coding partner in the projection pair.""" variant = Variant( score_set_id=sample_score_set.id, urn="variant:1", @@ -409,10 +467,11 @@ async def test_cis_phased_multivariant_candidate_is_persisted_as_a_block( await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) assay_hgvs = "NM_000000.1:c.1A>G" + c_candidate = "NM_000001.1:c.5A>G" block_candidate = "NC_000001.11:g.[1000A>G;1002T>C]" - construct = fake_construct({assay_hgvs: ([], [block_candidate])}) + construct = fake_construct({assay_hgvs: [(c_candidate, block_candidate)]}) translate = fake_translate( - {block_candidate: "ga4gh:CPB.block"}, + {c_candidate: "ga4gh:VA.coding", block_candidate: "ga4gh:CPB.block"}, type_by_hgvs={block_candidate: "CisPhasedBlock"}, ) @@ -423,7 +482,7 @@ async def test_cis_phased_multivariant_candidate_is_persisted_as_a_block( result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) assert result.status == JobStatus.SUCCEEDED - assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 1} + assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 2} block_allele = session.query(Allele).filter(Allele.vrs_digest == "ga4gh:CPB.block").one() assert block_allele.level == AnnotationLayer.genomic.value @@ -432,7 +491,8 @@ async def test_cis_phased_multivariant_candidate_is_persisted_as_a_block( assert block_allele.transcript == "NC_000001.11" assert block_allele.post_mapped == {"type": "CisPhasedBlock", "id": "ga4gh:CPB.block"} - assert len(_non_authoritative_links(session)) == 1 + # The block (genomic projection) and its coding partner are one projection pair -> one group. + assert {link.projection_group for link in _non_authoritative_links(session)} == {0} async def test_duplicate_candidate_digests_are_deduped_per_record( self, @@ -444,8 +504,9 @@ async def test_duplicate_candidate_digests_are_deduped_per_record( sample_independent_reverse_translation_run, sample_score_set, ): - """When a coding and genomic candidate resolve to the same VRS digest, only one - allele/link is written for that mapping record (per the library's dedup contract).""" + """When the coding and genomic members of a projection pair resolve to the same VRS digest, + only one allele/link is written for that mapping record (per the library's dedup contract). + The surviving link still carries the pair's group.""" variant = Variant( score_set_id=sample_score_set.id, urn="variant:1", @@ -460,8 +521,8 @@ async def test_duplicate_candidate_digests_are_deduped_per_record( assay_hgvs = "NM_000000.1:c.1A>G" g_candidate = "NC_000001.11:g.1000A>G" c_candidate = "NM_000001.1:c.5A>G" - construct = fake_construct({assay_hgvs: ([c_candidate], [g_candidate])}) - # Both candidates collapse to the same VRS digest. + construct = fake_construct({assay_hgvs: [(c_candidate, g_candidate)]}) + # Both members of the projection pair collapse to the same VRS digest. translate = fake_translate({g_candidate: "ga4gh:VA.shared", c_candidate: "ga4gh:VA.shared"}) with ( @@ -472,10 +533,12 @@ async def test_duplicate_candidate_digests_are_deduped_per_record( assert result.status == JobStatus.SUCCEEDED assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 1} - assert len(_non_authoritative_links(session)) == 1 + non_auth_links = _non_authoritative_links(session) + assert len(non_auth_links) == 1 + assert non_auth_links[0].projection_group == 0 assert session.query(Allele).filter(Allele.vrs_digest == "ga4gh:VA.shared").count() == 1 - async def test_candidate_equal_to_authoritative_allele_is_not_relinked( + async def test_authoritative_allele_is_folded_into_its_projection_group( self, session, with_independent_processing_runs, @@ -485,10 +548,11 @@ async def test_candidate_equal_to_authoritative_allele_is_not_relinked( sample_independent_reverse_translation_run, sample_score_set, ): - """A derived candidate whose digest equals the record's authoritative (assay-measured) - allele is not linked again — the record already links that allele authoritatively, and a - derived duplicate would surface the measured variant twice. No new link and no new allele; - an empty derived set is still a success with nothing to add.""" + """Nucleotide assay: the measured allele appears among the RT candidates (the library makes + no exclusions) and already has a live authoritative link (group NULL). Rather than insert a + forbidden derived duplicate, the job folds the pair's group onto the existing + authoritative link. That group then resolves to {canonical coding, canonical genomic}: the + authoritative allele plus its sibling projection derived in the same projection pair.""" variant = Variant( score_set_id=sample_score_set.id, urn="variant:1", @@ -500,16 +564,16 @@ async def test_candidate_equal_to_authoritative_allele_is_not_relinked( session.commit() await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) - # Exactly one allele exists after mapping: the authoritative post-mapped allele. + # Exactly one allele exists after mapping: the authoritative (measured) allele. assert session.query(Allele).count() == 1 - authoritative_allele = session.query(Allele).one() - assert authoritative_allele.vrs_digest == TEST_GA4GH_IDENTIFIER + assert session.query(Allele).one().vrs_digest == TEST_GA4GH_IDENTIFIER assay_hgvs = "NM_000000.1:c.1A>G" c_candidate = "NM_000001.1:c.5A>G" - construct = fake_construct({assay_hgvs: ([c_candidate], [])}) - # The candidate resolves to the same digest as the authoritative allele. - translate = fake_translate({c_candidate: TEST_GA4GH_IDENTIFIER}) + g_sibling = "NC_000001.11:g.1000A>G" + construct = fake_construct({assay_hgvs: [(c_candidate, g_sibling)]}) + # The coding member IS the measured allele (folds in); the genomic member is its projection. + translate = fake_translate({c_candidate: TEST_GA4GH_IDENTIFIER, g_sibling: "ga4gh:VA.genomic"}) with ( patch(f"{RT_MODULE}.construct_equivalent_variants", construct), @@ -518,14 +582,198 @@ async def test_candidate_equal_to_authoritative_allele_is_not_relinked( result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) assert result.status == JobStatus.SUCCEEDED - assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 0} + # Only the genomic sibling is a newly created derived allele; the coding member folds into + # the existing authoritative link rather than creating a second link/allele. + assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 1} - # The authoritative allele was reused, not duplicated. - assert session.query(Allele).count() == 1 - # No derived link was added: the candidate equals the record's authoritative allele. - assert _non_authoritative_links(session) == [] + record = _record_for(session, variant.id) + auth_link = _authoritative_link(session, record.id) + derived_links = _non_authoritative_links(session) + assert len(derived_links) == 1 + + # The authoritative link was folded into the pair's group, and the sibling derived + # link shares it -- the pairing is resolvable from the authoritative allele's group. + assert auth_link.projection_group == 0 + assert derived_links[0].projection_group == 0 + + # group 0 resolves to {canonical c (authoritative), canonical g (projection)}: two live + # links, one authoritative and one derived. + group_links = [link for link in _live_links(session, record.id) if link.projection_group == 0] + assert len(group_links) == 2 + assert {link.is_authoritative for link in group_links} == {True, False} + assert auth_link.allele.vrs_digest == TEST_GA4GH_IDENTIFIER + assert derived_links[0].allele.vrs_digest == "ga4gh:VA.genomic" + + async def test_authoritative_fold_in_when_measured_at_genomic_level( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """The fold-in is symmetric in the measured level: when the genomic member of a projection pair + equals the record's authoritative allele, its group folds onto the authoritative link and + the coding sibling becomes the derived member of the same group.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NC_000001.11:g.1000A>G", + data={}, + ) + session.add(variant) + session.commit() + # Genomic assay layer + cdna identity mapping so RT resolves a coding transcript. + await _map_variants( + session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set, with_layers={"g", "c"} + ) + assert session.query(Allele).one().vrs_digest == TEST_GA4GH_IDENTIFIER + + assay_hgvs = "NC_000001.11:g.1000A>G" + c_sibling = "NM_000001.1:c.5A>G" + g_candidate = "NC_000001.11:g.1000A>G" + construct = fake_construct({assay_hgvs: [(c_sibling, g_candidate)]}) + # The genomic member IS the measured allele (folds in); the coding member is its projection. + translate = fake_translate({c_sibling: "ga4gh:VA.coding", g_candidate: TEST_GA4GH_IDENTIFIER}) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 1} + + record = _record_for(session, variant.id) + auth_link = _authoritative_link(session, record.id) + derived_links = _non_authoritative_links(session) + assert auth_link.projection_group == 0 + assert [link.projection_group for link in derived_links] == [0] + assert auth_link.allele.vrs_digest == TEST_GA4GH_IDENTIFIER + assert derived_links[0].allele.vrs_digest == "ga4gh:VA.coding" + + async def test_projection_failed_candidate_is_a_one_member_group( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """A projection pair whose genomic projection failed (hgvs_g is None) yields a well-formed + one-member (coding-only) group -- no crash, no desync, and the group holds just the coding + link. This is the case the old two-list shape silently dropped.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + assay_hgvs = "NM_000000.1:c.1A>G" + c_paired = "NM_000001.1:c.5A>G" + g_paired = "NC_000001.11:g.1000A>G" + c_orphan = "NM_000001.1:c.9A>T" # projection failed: no genomic partner + construct = fake_construct({assay_hgvs: [(c_paired, g_paired), (c_orphan, None)]}) + translate = fake_translate( + { + c_paired: "ga4gh:VA.coding0", + g_paired: "ga4gh:VA.genomic0", + c_orphan: "ga4gh:VA.coding1", + } + ) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 3} - async def test_independent_rerun_retires_prior_links_and_keeps_current_stable( + non_auth_links = _non_authoritative_links(session) + groups: dict[int, set[str]] = {} + for link in non_auth_links: + groups.setdefault(link.projection_group, set()).add(link.allele.vrs_digest) + + # Group 0 is the full projection pair; group 1 is the projection-failed coding-only member. + assert groups == { + 0: {"ga4gh:VA.coding0", "ga4gh:VA.genomic0"}, + 1: {"ga4gh:VA.coding1"}, + } + + async def test_protein_assay_leaves_authoritative_group_null_over_a_fan_out( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """Protein assay: the authoritative allele is the protein itself, which appears in no + coding/genomic projection pair, so nothing folds in and its link's group stays NULL. The + ambiguous fan-out becomes N distinct groups, each a coding/genomic pair.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + # Protein-only mapping: the authoritative allele is the protein allele. + await _map_variants( + session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set, with_layers={"p"} + ) + assert session.query(Allele).one().vrs_digest == TEST_GA4GH_IDENTIFIER + + assay_hgvs = "NP_000000.1:p.Met1Val" + # A degenerate protein consequence -> three coding/genomic projection pairs (the fan-out). + pairs = [ + ("NM_000001.1:c.3A>G", "NC_000001.11:g.100A>G"), + ("NM_000001.1:c.3A>C", "NC_000001.11:g.100A>C"), + ("NM_000001.1:c.3A>T", "NC_000001.11:g.100A>T"), + ] + # Protein assay: no protein apex is re-emitted (result.hgvs_p is None) -- the protein + # allele is already authoritative. + construct = fake_construct({assay_hgvs: pairs}) + translate = fake_translate({hgvs: f"ga4gh:VA.{hgvs}" for pair in pairs for hgvs in pair}) + + with ( + patch(f"{RT_MODULE}._coding_transcripts_for_proteins", lambda accessions: {"NP_000000.1": "NM_000001.1"}), + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 6} + + record = _record_for(session, variant.id) + # The protein authoritative allele is not part of any pair: its group stays NULL. + auth_link = _authoritative_link(session, record.id) + assert auth_link.projection_group is None + assert auth_link.allele.vrs_digest == TEST_GA4GH_IDENTIFIER + + # Three distinct groups, each a coding/genomic pair (two members). + groups: dict[int, int] = {} + for link in _non_authoritative_links(session): + groups[link.projection_group] = groups.get(link.projection_group, 0) + 1 + assert groups == {0: 2, 1: 2, 2: 2} + + async def test_independent_rerun_retires_prior_links_and_regroups( self, session, with_independent_processing_runs, @@ -536,9 +784,11 @@ async def test_independent_rerun_retires_prior_links_and_keeps_current_stable( sample_score_set, ): """An independent second run is idempotent: it retires the prior derived links (closing - valid_to) and writes fresh live ones, so the set of *current* non-authoritative links is - stable while the superseded links are retained as history — the partial unique index on - live links is never violated and no alleles are duplicated.""" + valid_to) and writes fresh live ones (re-grouped from scratch), so the set of *current* + non-authoritative links is stable while the superseded links are retained as history -- + the partial unique index on live links is never violated and no alleles are duplicated. + projection_group is regenerated per run; it is a within-record grouping key, not a stable + identity across re-maps.""" variant = Variant( score_set_id=sample_score_set.id, urn="variant:1", @@ -553,10 +803,10 @@ async def test_independent_rerun_retires_prior_links_and_keeps_current_stable( assay_hgvs = "NM_000000.1:c.1A>G" g_candidate = "NC_000001.11:g.1000A>G" c_candidate = "NM_000001.1:c.5A>G" - construct = fake_construct({assay_hgvs: ([c_candidate], [g_candidate])}) + construct = fake_construct({assay_hgvs: [(c_candidate, g_candidate)]}) translate = fake_translate({g_candidate: "ga4gh:VA.genomic", c_candidate: "ga4gh:VA.coding"}) - # First run: two live derived links. + # First run: two live derived links, both in group 0. with ( patch(f"{RT_MODULE}.construct_equivalent_variants", construct), patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), @@ -567,6 +817,7 @@ async def test_independent_rerun_retires_prior_links_and_keeps_current_stable( first_links = _non_authoritative_links(session) assert len(first_links) == 2 assert all(link.valid_to is None for link in first_links) + assert {link.projection_group for link in first_links} == {0} # Second, independent run with identical inputs. rerun = JobRun( @@ -592,13 +843,14 @@ async def test_independent_rerun_retires_prior_links_and_keeps_current_stable( live = [link for link in links if link.valid_to is None] retired = [link for link in links if link.valid_to is not None] - # The live set is stable (still exactly the two candidates) and the prior run's links are - # retained as closed history rather than deleted. + # The live set is stable (still exactly the two candidates, re-grouped into group 0) and the + # prior run's links are retained as closed history rather than deleted. assert len(live) == 2 assert len(retired) == 2 + assert {link.projection_group for link in live} == {0} # Gap-free handoff: the prior links closed at exactly the instant the new links opened, under - # one supersession timestamp — so a point-in-time query never lands in a hole between runs, + # one supersession timestamp -- so a point-in-time query never lands in a hole between runs, # even though the translation loop ran (and could have committed) between retire and insert. assert {link.valid_to for link in retired} == {link.valid_from for link in live} assert len({link.valid_from for link in live}) == 1 @@ -683,12 +935,12 @@ async def test_partial_success_succeeds_with_mixed_annotations( session.commit() await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) - g_candidate = "NC_000001.11:g.1000A>G" + c_candidate = "NM_000001.1:c.5A>G" construct = fake_construct( - {"NM_000000.1:c.1A>G": ([], [g_candidate])}, + {"NM_000000.1:c.1A>G": [(c_candidate, None)]}, errors_by_hgvs={"NM_000000.1:c.2G>T": "no consequence"}, ) - translate = fake_translate({g_candidate: "ga4gh:VA.genomic"}) + translate = fake_translate({c_candidate: "ga4gh:VA.coding"}) with ( patch(f"{RT_MODULE}.construct_equivalent_variants", construct), @@ -713,8 +965,8 @@ async def test_partial_candidate_translation_failure_keeps_success_with_metadata sample_independent_reverse_translation_run, sample_score_set, ): - """One candidate translating and another failing VRS translation leaves the variant a - SUCCESS (one allele created) while retaining the dropped candidate in metadata.""" + """One member of a projection pair translating and the other failing VRS translation leaves the + variant a SUCCESS (one allele created) while retaining the dropped candidate in metadata.""" variant = Variant( score_set_id=sample_score_set.id, urn="variant:1", @@ -727,12 +979,12 @@ async def test_partial_candidate_translation_failure_keeps_success_with_metadata await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) assay_hgvs = "NM_000000.1:c.1A>G" - good_candidate = "NC_000001.11:g.1000A>G" - bad_candidate = "NC_000001.11:g.999A>T" - construct = fake_construct({assay_hgvs: ([], [good_candidate, bad_candidate])}) + good_coding = "NM_000001.1:c.5A>G" + bad_genomic = "NC_000001.11:g.999A>T" + construct = fake_construct({assay_hgvs: [(good_coding, bad_genomic)]}) translate = fake_translate( - {good_candidate: "ga4gh:VA.genomic"}, - errors_by_hgvs={bad_candidate: "untranslatable form"}, + {good_coding: "ga4gh:VA.coding"}, + errors_by_hgvs={bad_genomic: "untranslatable form"}, ) with ( @@ -748,7 +1000,7 @@ async def test_partial_candidate_translation_failure_keeps_success_with_metadata events = _cross_level_events(session, sample_score_set.id, disposition="present") assert len(events) == 1 failed_candidates = events[0].event_metadata["failed_candidates"] - assert failed_candidates == [{"hgvs": bad_candidate, "level": "genomic", "error": "untranslatable form"}] + assert failed_candidates == [{"hgvs": bad_genomic, "level": "genomic", "error": "untranslatable form"}] async def test_all_candidates_failing_translation_marks_variant_failed( self, @@ -774,8 +1026,8 @@ async def test_all_candidates_failing_translation_marks_variant_failed( await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) assay_hgvs = "NM_000000.1:c.1A>G" - bad_candidate = "NC_000001.11:g.999A>T" - construct = fake_construct({assay_hgvs: ([], [bad_candidate])}) + bad_candidate = "NM_000001.1:c.9A>T" # a coding-only projection pair whose sole member fails + construct = fake_construct({assay_hgvs: [(bad_candidate, None)]}) translate = fake_translate({}, errors_by_hgvs={bad_candidate: "untranslatable form"}) with ( @@ -794,7 +1046,7 @@ async def test_all_candidates_failing_translation_marks_variant_failed( metadata = failed_events[0].event_metadata assert metadata["hgvs_input"] == assay_hgvs assert metadata["failed_candidates"] == [ - {"hgvs": bad_candidate, "level": "genomic", "error": "untranslatable form"} + {"hgvs": bad_candidate, "level": "cdna", "error": "untranslatable form"} ] async def test_unresolved_transcript_is_tallied_as_skip_but_recorded_as_failed( @@ -876,15 +1128,21 @@ async def test_genomic_accession_coding_target_is_reverse_translated( ) assay_hgvs = "NC_000001.11:g.1000A>G" - g_candidate = "NC_000001.11:g.1000A>G" + c_candidate = "NM_000001.1:c.5A>G" + g_candidate = "NC_000001.11:g.2000A>G" with ( - patch(f"{RT_MODULE}.construct_equivalent_variants", fake_construct({assay_hgvs: ([], [g_candidate])})), - patch(f"{RT_MODULE}.translate_hgvs_to_variation", fake_translate({g_candidate: "ga4gh:VA.genomic"})), + patch( + f"{RT_MODULE}.construct_equivalent_variants", fake_construct({assay_hgvs: [(c_candidate, g_candidate)]}) + ), + patch( + f"{RT_MODULE}.translate_hgvs_to_variation", + fake_translate({c_candidate: "ga4gh:VA.coding", g_candidate: "ga4gh:VA.genomic"}), + ), ): result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) assert result.status == JobStatus.SUCCEEDED - assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 1} + assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 2} events = _cross_level_events(session, sample_score_set.id) assert events and all(e.disposition == "present" for e in events) @@ -944,8 +1202,9 @@ async def test_uses_latest_cdna_row_within_the_run( session.commit() assay_hgvs = "NC_000001.11:g.1000A>G" - g_candidate = "NC_000001.11:g.1000A>G" - delegate = fake_construct({assay_hgvs: ([], [g_candidate])}) + c_candidate = "NM_000001.1:c.5A>G" + g_candidate = "NC_000001.11:g.2000A>G" + delegate = fake_construct({assay_hgvs: [(c_candidate, g_candidate)]}) captured: dict = {} def capturing_construct(inputs, *, transcripts, coordinates, config): @@ -1116,9 +1375,10 @@ async def test_translation_config_param_overrides_job_defaults( await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) assay_hgvs = "NM_000000.1:c.1A>G" + c_candidate = "NM_000001.1:c.5A>G" g_candidate = "NC_000001.11:g.1000A>G" - delegate = fake_construct({assay_hgvs: ([], [g_candidate])}) - translate = fake_translate({g_candidate: "ga4gh:VA.genomic"}) + delegate = fake_construct({assay_hgvs: [(c_candidate, g_candidate)]}) + translate = fake_translate({c_candidate: "ga4gh:VA.coding", g_candidate: "ga4gh:VA.genomic"}) captured: dict = {} @@ -1168,9 +1428,10 @@ async def test_translation_config_defaults_when_param_absent( await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) assay_hgvs = "NM_000000.1:c.1A>G" + c_candidate = "NM_000001.1:c.5A>G" g_candidate = "NC_000001.11:g.1000A>G" - delegate = fake_construct({assay_hgvs: ([], [g_candidate])}) - translate = fake_translate({g_candidate: "ga4gh:VA.genomic"}) + delegate = fake_construct({assay_hgvs: [(c_candidate, g_candidate)]}) + translate = fake_translate({c_candidate: "ga4gh:VA.coding", g_candidate: "ga4gh:VA.genomic"}) captured: dict = {} @@ -1215,13 +1476,13 @@ async def test_protein_level_transcript_resolved_via_uta( ) assay_hgvs = "NP_000000.1:p.Met1Val" - g_candidate = "NC_000001.11:g.1000A>G" - translate = fake_translate({g_candidate: "ga4gh:VA.genomic"}) + c_candidate = "NM_000111.1:c.3A>G" + translate = fake_translate({c_candidate: "ga4gh:VA.coding"}) # Capture the transcript hint each VariantInput was built with, then delegate to the # normal stub for the translation result. captured_hints: dict[str, str | None] = {} - delegate = fake_construct({assay_hgvs: ([], [g_candidate])}) + delegate = fake_construct({assay_hgvs: [(c_candidate, None)]}) def capturing_construct(inputs, *, transcripts, coordinates, config): captured_hints.update({inp.hgvs: inp.transcript for inp in inputs}) From feb9c55eec41dba080514f7ff4810396a206bb9c Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Thu, 9 Jul 2026 10:31:13 -0700 Subject: [PATCH 072/252] feat(allele-measurements): add include_nucleotide_siblings flag and remove legacy lookup endpoint - Add `include_nucleotide_siblings` parameter to `get_allele_measurements` and expose it as a query param on `GET /clingen-alleles/{id}/measurements`; for a CA query, widens the equivalence class through the queried change's protein consequence to surface sibling nt variants encoding the same amino-acid change (relationship=nucleotide_encoding) - Fix relationship labeling to key off measured-allele level rather than entry level, correcting the sibling-nt branch so sibling nt records get nucleotide_encoding instead of protein_consequence - Remove `POST /variants/clingen-allele-id-lookups` and its associated view models (ClingenAlleleIdVariantLookupsRequest, ClingenAlleleVariants, ClingenAlleleIdVariantLookupResponse, VariantEffectMeasurementWithShortScoreSet) --- src/mavedb/lib/allele_measurements.py | 47 ++- src/mavedb/routers/clingen_alleles.py | 12 + src/mavedb/routers/variants.py | 399 +------------------------- src/mavedb/view_models/variant.py | 39 +-- tests/lib/test_allele_measurements.py | 33 +++ tests/routers/test_clingen_allele.py | 36 +++ 6 files changed, 123 insertions(+), 443 deletions(-) diff --git a/src/mavedb/lib/allele_measurements.py b/src/mavedb/lib/allele_measurements.py index eabb414f5..a3048cb83 100644 --- a/src/mavedb/lib/allele_measurements.py +++ b/src/mavedb/lib/allele_measurements.py @@ -8,8 +8,9 @@ consequence (**related**); a ``PA`` anchors on the protein allele, gathering the protein measurements of that change (**direct**) *and* every nt measurement that encodes it (**related**). The asymmetry falls out of *which* allele anchors the co-membership — a sibling nt change that also encodes the same protein -is not pulled onto a CA page, because its record links the protein but not the anchor nt allele. See -design §7.5. +is not pulled onto a CA page, because its record links the protein but not the anchor nt allele. +``include_nucleotide_siblings`` opts out of that asymmetry for discovery surfaces — see +:func:`get_allele_measurements`. Distinct from :func:`lib.variant_detail.get_variant_detail`, which is the *record-scoped*, all-levels detail of one *selected* measurement. This list is the *cross-record* union — who else measured @@ -53,9 +54,13 @@ class MeasurementRelationship(str, Enum): to the query. """ - direct = "direct" # the measurement was assayed *at* this allele - protein_consequence = "protein_consequence" # CA query, protein measurement of consequence(N) - nucleotide_encoding = "nucleotide_encoding" # PA query, nt measurement encoding P + # the measurement was assayed *at* this allele + direct = "direct" + # CA query, protein measurement of consequence(N) + protein_consequence = "protein_consequence" + # A nt measurement encoding the queried change's protein consequence: a PA query's encodings of P, or — + # only under ``include_nucleotide_siblings`` — a CA query's sibling nt changes that share consequence(N). + nucleotide_encoding = "nucleotide_encoding" @dataclass(frozen=True) @@ -148,10 +153,16 @@ def get_allele_measurements( *, user_data: Optional[UserData], include_superseded: bool = False, + include_nucleotide_siblings: bool = False, as_of: Optional[datetime] = None, ) -> list[AlleleMeasurement]: """List the measurements in ``clingen_allele_id``'s cross-layer equivalence class. Returns ``[]`` when the id resolves to no allele or no live record. + + ``include_nucleotide_siblings`` (a ``CA`` entry only; a no-op for ``PA``) widens the class through the + queried change's protein consequence to also surface the *sibling* nt changes — other nucleotide + variants that encode the same amino-acid change and were themselves assayed at the nucleotide level + (``relationship=nucleotide_encoding``). """ anchor = db.execute(select(Allele.id, Allele.level).where(Allele.clingen_allele_id == clingen_allele_id)).all() if not anchor: @@ -160,6 +171,23 @@ def get_allele_measurements( anchor_ids = [row.id for row in anchor] entry_is_protein = any(row.level == AnnotationLayer.protein.value for row in anchor) + # Sibling nucleotide changes (search discovery, CA only): fold the queried change's protein consequence + # — the protein alleles co-membered with the anchor nt alleles — into the anchor, so the single record + # union below also reaches every record encoding that consequence. a PA query already anchors on the protein, so + # this is exactly what makes a CA+siblings query behave like one. (This query would be redundant for a PA entry). + if include_nucleotide_siblings and not entry_is_protein: + anchor_link = aliased(MappingRecordAllele) + anchor_ids += db.scalars( + select(MappingRecordAllele.allele_id) + .join(anchor_link, anchor_link.mapping_record_id == MappingRecordAllele.mapping_record_id) + .join(Allele, Allele.id == MappingRecordAllele.allele_id) + .where(anchor_link.allele_id.in_(anchor_ids)) + .where(anchor_link.live_at(as_of)) + .where(MappingRecordAllele.live_at(as_of)) + .where(Allele.level == AnnotationLayer.protein.value) + .distinct() + ).all() + record_ids = db.scalars( select(MappingRecordAllele.mapping_record_id) .where(MappingRecordAllele.allele_id.in_(anchor_ids)) @@ -207,12 +235,15 @@ def get_allele_measurements( if not is_current and not include_superseded: continue + # Label by the measured level, not the entry level: direct when the measured allele *is* the query, + # else protein→protein_consequence / nucleotide→nucleotide_encoding. Without siblings the sibling-nt + # branch is unreachable (a record links the anchor only via itself or its protein consequence). if measured_allele.clingen_allele_id == clingen_allele_id: relationship = MeasurementRelationship.direct - elif entry_is_protein: - relationship = MeasurementRelationship.nucleotide_encoding - else: + elif measured_allele.level == AnnotationLayer.protein.value: relationship = MeasurementRelationship.protein_consequence + else: + relationship = MeasurementRelationship.nucleotide_encoding assay_level = record.assay_level measurement = AlleleMeasurement( diff --git a/src/mavedb/routers/clingen_alleles.py b/src/mavedb/routers/clingen_alleles.py index b3b626c02..64c6c1544 100644 --- a/src/mavedb/routers/clingen_alleles.py +++ b/src/mavedb/routers/clingen_alleles.py @@ -50,6 +50,16 @@ def get_clingen_allele_measurements( "measurements are a deliberate power-user / citation path, never surfaced by discovery." ), ), + include_nucleotide_siblings: bool = Query( + default=False, + description=( + "For a nucleotide (CA) query only: also return sibling nucleotide changes — other DNA " + "variants encoding the same protein consequence that were themselves assayed at the " + "nucleotide level (relationship 'nucleotide_encoding'). Default false: the variant page " + "anchors strictly on the queried allele. Discovery surfaces (search) set it to surface all " + "evidence bearing on the consequence. No-op for a protein (PA) query." + ), + ), as_of: Optional[datetime] = Query( default=None, description=( @@ -72,6 +82,7 @@ def get_clingen_allele_measurements( "requested_resource": clingen_allele_id, "as_of": as_of, "include_superseded": include_superseded, + "include_nucleotide_siblings": include_nucleotide_siblings, } ) response.headers["X-As-Of"] = as_of.isoformat() if as_of is not None else "current" @@ -81,5 +92,6 @@ def get_clingen_allele_measurements( clingen_allele_id, user_data=user_data, include_superseded=include_superseded, + include_nucleotide_siblings=include_nucleotide_siblings, as_of=as_of, ) diff --git a/src/mavedb/routers/variants.py b/src/mavedb/routers/variants.py index 66f302df6..d7cfdfc15 100644 --- a/src/mavedb/routers/variants.py +++ b/src/mavedb/routers/variants.py @@ -1,6 +1,4 @@ -import itertools import logging -import re from datetime import datetime from typing import Optional @@ -8,8 +6,7 @@ from fastapi.exceptions import HTTPException from sqlalchemy import select from sqlalchemy.exc import MultipleResultsFound -from sqlalchemy.orm import Session, joinedload -from sqlalchemy.sql import or_ +from sqlalchemy.orm import Session from mavedb import deps from mavedb.lib.authentication import get_current_user @@ -18,20 +15,12 @@ from mavedb.lib.permissions import Action, assert_permission, has_permission from mavedb.lib.types.authentication import UserData from mavedb.lib.variant_detail import get_variant_detail -from mavedb.models.mapped_variant import MappedVariant -from mavedb.models.score_set import ScoreSet from mavedb.models.variant import Variant -from mavedb.models.variant_translation import VariantTranslation from mavedb.routers.shared import ( ACCESS_CONTROL_ERROR_RESPONSES, - BASE_400_RESPONSE, PUBLIC_ERROR_RESPONSES, ROUTER_BASE_PREFIX, ) -from mavedb.view_models.variant import ( - ClingenAlleleIdVariantLookupResponse, - ClingenAlleleIdVariantLookupsRequest, -) from mavedb.view_models.variant_detail import VariantDetail TAG_NAME = "Variants" @@ -51,392 +40,6 @@ } -@router.post( - "/variants/clingen-allele-id-lookups", - status_code=200, - response_model=list[ClingenAlleleIdVariantLookupResponse], - responses={ - **BASE_400_RESPONSE, - **ACCESS_CONTROL_ERROR_RESPONSES, - }, - summary="Lookup variants by ClinGen Allele IDs", -) -def lookup_variants( - *, - request: ClingenAlleleIdVariantLookupsRequest, - db: Session = Depends(deps.get_db), - user_data: UserData = Depends(get_current_user), -): - """ - Lookup variants by ClinGen Allele IDs. - """ - save_to_logging_context({"requested_resource": "clingen-allele-id-lookups"}) - save_to_logging_context({"clingen_allele_ids_to_lookup": request.clingen_allele_ids}) - logger.debug(msg="Looking up variants by Clingen Allele IDs", extra=logging_context()) - - # sort multi-variant components lexicographically, as they are in the database - request.clingen_allele_ids = [",".join(sorted(allele_id.split(","))) for allele_id in request.clingen_allele_ids] - exact_match_variants = db.execute( - select(Variant, MappedVariant.clingen_allele_id) - .join(MappedVariant) - .options(joinedload(Variant.score_set).joinedload(ScoreSet.experiment)) - .where(MappedVariant.clingen_allele_id.in_(request.clingen_allele_ids)) - .where(MappedVariant.current == True) # noqa: E712 - ).all() - - save_to_logging_context({"num_variants_matching_clingen_allele_ids": len(exact_match_variants)}) - logger.debug(msg="Found variants with exactly matching ClinGen Allele IDs", extra=logging_context()) - - num_variants_matching_clingen_allele_ids_and_permitted = 0 - - variants_by_allele_id: dict[str, dict] = { - allele_id: { - "clingen_allele_id": allele_id, - "exact_match": {"clingen_allele_id": allele_id, "variant_effect_measurements": []}, - "equivalent_nt": [], - "equivalent_aa": [], - } - for allele_id in request.clingen_allele_ids - } - for variant, allele_id in exact_match_variants: - if has_permission(user_data, variant.score_set, Action.READ).permitted: - num_variants_matching_clingen_allele_ids_and_permitted += 1 - variants_by_allele_id[allele_id]["exact_match"]["variant_effect_measurements"].append(variant) - - save_to_logging_context( - {"clingen_allele_ids_with_permitted_variants": num_variants_matching_clingen_allele_ids_and_permitted} - ) - - for allele_id in request.clingen_allele_ids: - if not variants_by_allele_id[allele_id]["exact_match"]["variant_effect_measurements"]: - variants_by_allele_id[allele_id]["exact_match"] = None - - for allele_id in request.clingen_allele_ids: - # validate and determine whether multi-variant - # NOTE: assuming we never have more than 2 components in a multi-variant - single_clingen_allele_id_re = r"^[CP]A\d+$" - multi_clingen_allele_id_re = r"^[CP]A\d+,[CP]A\d+$" - single_or_multi_clingen_allele_id_re = r"^[CP]A\d+(,[CP]A\d+)?$" - - if re.fullmatch(single_or_multi_clingen_allele_id_re, allele_id) is None: - raise HTTPException( - status_code=400, - detail=f"Invalid Clingen Allele ID '{allele_id}'", - ) - - if re.fullmatch(single_clingen_allele_id_re, allele_id): - if allele_id.startswith("PA"): - subquery = ( - db.execute( - select(VariantTranslation.nt_clingen_id).where(VariantTranslation.aa_clingen_id == allele_id) - ) - .scalars() - .all() - ) - related_clingen_ids = ( - db.execute( - select(VariantTranslation).where( - or_( - VariantTranslation.aa_clingen_id == allele_id, - VariantTranslation.nt_clingen_id.in_(subquery), - ) - ) - ) - .scalars() - .all() - ) - # exclude requested clingen allele id from "related_clingen_ids" to avoid duplicates - related_aa_clingen_ids = [ - var_translation.aa_clingen_id - for var_translation in related_clingen_ids - if var_translation.aa_clingen_id != allele_id - ] - related_aa_variants = db.execute( - select(Variant, MappedVariant.clingen_allele_id) - .join(MappedVariant) - .options(joinedload(Variant.score_set).joinedload(ScoreSet.experiment)) - .where(MappedVariant.clingen_allele_id.in_(related_aa_clingen_ids)) - .where(MappedVariant.current == True) # noqa: E712 - ).all() - related_nt_clingen_ids = [var_translation.nt_clingen_id for var_translation in related_clingen_ids] - related_nt_variants = db.execute( - select(Variant, MappedVariant.clingen_allele_id) - .join(MappedVariant) - .options(joinedload(Variant.score_set).joinedload(ScoreSet.experiment)) - .where(MappedVariant.clingen_allele_id.in_(related_nt_clingen_ids)) - .where(MappedVariant.current == True) # noqa: E712 - ).all() - elif allele_id.startswith("CA"): - subquery = ( - db.execute( - select(VariantTranslation.aa_clingen_id).where(VariantTranslation.nt_clingen_id == allele_id) - ) - .scalars() - .all() - ) - related_clingen_ids = ( - db.execute( - select(VariantTranslation).where( - or_( - VariantTranslation.nt_clingen_id == allele_id, - VariantTranslation.aa_clingen_id.in_(subquery), - ) - ) - ) - .scalars() - .all() - ) - related_aa_clingen_ids = [var_translation.aa_clingen_id for var_translation in related_clingen_ids] - related_aa_variants = db.execute( - select(Variant, MappedVariant.clingen_allele_id) - .join(MappedVariant) - .options(joinedload(Variant.score_set).joinedload(ScoreSet.experiment)) - .where(MappedVariant.clingen_allele_id.in_(related_aa_clingen_ids)) - .where(MappedVariant.current == True) # noqa: E712 - ).all() - # exclude requested clingen allele id from "related_clingen_ids" to avoid duplicates - related_nt_clingen_ids = [ - var_translation.nt_clingen_id - for var_translation in related_clingen_ids - if var_translation.nt_clingen_id != allele_id - ] - related_nt_variants = db.execute( - select(Variant, MappedVariant.clingen_allele_id) - .join(MappedVariant) - .options(joinedload(Variant.score_set).joinedload(ScoreSet.experiment)) - .where(MappedVariant.clingen_allele_id.in_(related_nt_clingen_ids)) - .where(MappedVariant.current == True) # noqa: E712 - ).all() - - num_variants_matching_clingen_allele_ids_and_permitted = 0 - equivalent_aa_variants = {} - for variant, related_allele_id in related_aa_variants: - if has_permission(user_data, variant.score_set, Action.READ).permitted: - if related_allele_id not in equivalent_aa_variants: - equivalent_aa_variants[related_allele_id] = { - "clingen_allele_id": related_allele_id, - "variant_effect_measurements": [], - } - equivalent_aa_variants[related_allele_id]["variant_effect_measurements"].append(variant) - - variants_by_allele_id[allele_id]["equivalent_aa"] = [ - equivalent_aa_variants[related_allele_id] for related_allele_id in equivalent_aa_variants - ] - - equivalent_nt_variants = {} - for variant, related_allele_id in related_nt_variants: - if has_permission(user_data, variant.score_set, Action.READ).permitted: - if related_allele_id not in equivalent_nt_variants: - equivalent_nt_variants[related_allele_id] = { - "clingen_allele_id": related_allele_id, - "variant_effect_measurements": [], - } - equivalent_nt_variants[related_allele_id]["variant_effect_measurements"].append(variant) - - variants_by_allele_id[allele_id]["equivalent_nt"] = [ - equivalent_nt_variants[related_allele_id] for related_allele_id in equivalent_nt_variants - ] - - elif re.fullmatch(multi_clingen_allele_id_re, allele_id): - # validate each component allele id - already done via re - # allow more than 2 components? - # full matches are determined the same exact way as single variant (done above already) - # equivalent aa/nt: if all components of variant are represented by equivalent or exact match, but not all exact matches because those are already represented above - # so basically go through variant translations table and create every possible combo of components except for full exact match, - # then search the db for those combos - - allele_id_components = allele_id.split(",") - if all(component.startswith("PA") for component in allele_id_components): - related_clingen_id_components = [] # list of lists, one list for each component - for component in allele_id_components: - subquery = ( - db.execute( - select(VariantTranslation.nt_clingen_id).where( - VariantTranslation.aa_clingen_id == component - ) - ) - .scalars() - .all() - ) - related_clingen_id_components.append( - db.execute( - select(VariantTranslation).where( - or_( - VariantTranslation.aa_clingen_id == component, - VariantTranslation.nt_clingen_id.in_(subquery), - ) - ) - ) - .scalars() - .all() - ) - # create every possible combination of those two, including the original components, except for the version with both original components - # assuming that aa variants should always be paired with aa variants, and nt variants with nt variants, - # create lists of just the aa variants, and lists of just the nt variants. - # assuming only 2 components for any multivariant in db. - aa_clingen_id_first_allele = [ - translation.aa_clingen_id for translation in related_clingen_id_components[0] - ] - # original component should also be included, even if it is not in the variant translations table - aa_clingen_id_first_allele.append(allele_id_components[0]) - aa_clingen_id_second_allele = [ - translation.aa_clingen_id for translation in related_clingen_id_components[1] - ] - # original component should also be included, even if it is not in the variant translations table - aa_clingen_id_first_allele.append(allele_id_components[1]) - related_aa_clingen_id_combinations = list( - itertools.product(aa_clingen_id_first_allele, aa_clingen_id_second_allele) - ) - # turn each inner list into a single string and sort each pair lexicographically, since this is how they are stored in the db - joined_related_aa_clingen_id_combinations = [ - ",".join(sorted(list(combination))) # type: ignore - for combination in related_aa_clingen_id_combinations - ] - related_nt_clingen_id_first_allele = [ - translation.nt_clingen_id for translation in related_clingen_id_components[0] - ] - related_nt_clingen_id_second_allele = [ - translation.nt_clingen_id for translation in related_clingen_id_components[1] - ] - related_nt_clingen_id_combinations = list( - itertools.product(related_nt_clingen_id_first_allele, related_nt_clingen_id_second_allele) - ) - joined_related_nt_clingen_id_combinations = [ - ",".join(sorted(list(combination))) # type: ignore - for combination in related_nt_clingen_id_combinations - ] - - # query the db for variants with these combinations - related_aa_variants = db.execute( - select(Variant, MappedVariant.clingen_allele_id) - .join(MappedVariant) - .options(joinedload(Variant.score_set).joinedload(ScoreSet.experiment)) - .where(MappedVariant.clingen_allele_id.in_(joined_related_aa_clingen_id_combinations)) - .where(MappedVariant.clingen_allele_id != allele_id) - .where(MappedVariant.current == True) # noqa: E712 - ).all() - related_nt_variants = db.execute( - select(Variant, MappedVariant.clingen_allele_id) - .join(MappedVariant) - .options(joinedload(Variant.score_set).joinedload(ScoreSet.experiment)) - .where(MappedVariant.clingen_allele_id.in_(joined_related_nt_clingen_id_combinations)) - .where(MappedVariant.current == True) # noqa: E712 - ).all() - - elif all(component.startswith("CA") for component in allele_id_components): - related_clingen_id_components = [] # list of lists, one list for each component - for component in allele_id_components: - subquery = ( - db.execute( - select(VariantTranslation.aa_clingen_id).where( - VariantTranslation.nt_clingen_id == allele_id - ) - ) - .scalars() - .all() - ) - related_clingen_id_components.append( - db.execute( - select(VariantTranslation).where( - or_( - VariantTranslation.nt_clingen_id == allele_id, - VariantTranslation.aa_clingen_id.in_(subquery), - ) - ) - ) - .scalars() - .all() - ) - # create every possible combination of those two, including the original components, except for the version with both original components - # assuming that aa variants should always be paired with aa variants, and nt variants with nt variants, - # create lists of just the aa variants, and lists of just the nt variants. - # assuming only 2 components for any multivariant in db. - aa_clingen_id_first_allele = [ - translation.aa_clingen_id for translation in related_clingen_id_components[0] - ] - aa_clingen_id_second_allele = [ - translation.aa_clingen_id for translation in related_clingen_id_components[1] - ] - related_aa_clingen_id_combinations = list( - itertools.product(aa_clingen_id_first_allele, aa_clingen_id_second_allele) - ) - # turn each inner list into a single string and sort each pair lexicographically, since this is how they are stored in the db - joined_related_aa_clingen_id_combinations = [ - ",".join(sorted(list(combination))) # type: ignore - for combination in related_aa_clingen_id_combinations - ] - - related_nt_clingen_id_first_allele = [ - translation.nt_clingen_id for translation in related_clingen_id_components[0] - ] - # original component should also be included, even if it is not in the variant translations table - related_nt_clingen_id_first_allele.append(allele_id_components[0]) - related_nt_clingen_id_second_allele = [ - translation.nt_clingen_id for translation in related_clingen_id_components[1] - ] - # original component should also be included, even if it is not in the variant translations table - related_nt_clingen_id_first_allele.append(allele_id_components[1]) - related_nt_clingen_id_combinations = list( - itertools.product(related_nt_clingen_id_first_allele, related_nt_clingen_id_second_allele) - ) - joined_related_nt_clingen_id_combinations = [ - ",".join(sorted(list(combination))) # type: ignore - for combination in related_nt_clingen_id_combinations - ] - # query the db for variants with these combinations - related_aa_variants = db.execute( - select(Variant, MappedVariant.clingen_allele_id) - .join(MappedVariant) - .options(joinedload(Variant.score_set).joinedload(ScoreSet.experiment)) - .where(MappedVariant.clingen_allele_id.in_(joined_related_aa_clingen_id_combinations)) - .where(MappedVariant.current == True) # noqa: E712 - ).all() - related_nt_variants = db.execute( - select(Variant, MappedVariant.clingen_allele_id) - .join(MappedVariant) - .options(joinedload(Variant.score_set).joinedload(ScoreSet.experiment)) - .where(MappedVariant.clingen_allele_id.in_(joined_related_nt_clingen_id_combinations)) - .where(MappedVariant.clingen_allele_id != allele_id) - .where(MappedVariant.current == True) # noqa: E712 - ).all() - - else: - raise HTTPException( - status_code=400, - detail=f"Invalid Clingen Allele ID '{allele_id}': all components must be either PA or CA", - ) - - equivalent_aa_variants = {} - for variant, related_allele_id in related_aa_variants: - if has_permission(user_data, variant.score_set, Action.READ).permitted: - if related_allele_id not in equivalent_aa_variants: - equivalent_aa_variants[related_allele_id] = { - "clingen_allele_id": related_allele_id, - "variant_effect_measurements": [], - } - equivalent_aa_variants[related_allele_id]["variant_effect_measurements"].append(variant) - - variants_by_allele_id[allele_id]["equivalent_aa"] = [ - equivalent_aa_variants[related_allele_id] for related_allele_id in equivalent_aa_variants - ] - - equivalent_nt_variants = {} - for variant, related_allele_id in related_nt_variants: - if has_permission(user_data, variant.score_set, Action.READ).permitted: - if related_allele_id not in equivalent_nt_variants: - equivalent_nt_variants[related_allele_id] = { - "clingen_allele_id": related_allele_id, - "variant_effect_measurements": [], - } - equivalent_nt_variants[related_allele_id]["variant_effect_measurements"].append(variant) - - variants_by_allele_id[allele_id]["equivalent_nt"] = [ - equivalent_nt_variants[related_allele_id] for related_allele_id in equivalent_nt_variants - ] - - return [variants_by_allele_id[allele_id] for allele_id in request.clingen_allele_ids] - - @router.get( "/variants/{urn}", status_code=200, diff --git a/src/mavedb/view_models/variant.py b/src/mavedb/view_models/variant.py index ad02023df..c32abcab6 100644 --- a/src/mavedb/view_models/variant.py +++ b/src/mavedb/view_models/variant.py @@ -1,15 +1,12 @@ from datetime import date -from typing import TYPE_CHECKING, Any, Optional +from typing import Any, Optional from pydantic import model_validator from mavedb.lib.validation.exceptions import ValidationError from mavedb.view_models import record_type_validator, set_record_type from mavedb.view_models.base.base import BaseModel -from mavedb.view_models.mapped_variant import MappedVariant, SavedMappedVariant - -if TYPE_CHECKING: - from mavedb.view_models.score_set import ShortScoreSet +from mavedb.view_models.mapped_variant import SavedMappedVariant class VariantEffectMeasurementBase(BaseModel): @@ -74,35 +71,3 @@ class VariantEffectMeasurement(SavedVariantEffectMeasurement): """Variant effect measurement view model returned to most clients""" pass - - -class VariantEffectMeasurementWithShortScoreSet(SavedVariantEffectMeasurement): - """Variant effect measurement view model with mapped variants and a limited set of score set details""" - - score_set: "ShortScoreSet" - mapped_variants: list[MappedVariant] - - -class ClingenAlleleIdVariantLookupsRequest(BaseModel): - """A request to search for variants matching a list of ClinGen allele IDs""" - - clingen_allele_ids: list[str] - - -class ClingenAlleleVariants(BaseModel): - """The assayed variants sharing one ClinGen allele id, with their variant effect measurements. - - An allele-side grouping (keyed by ClinGen allele id), it is the mapped-allele view of "which - measurements share this molecular identity," not the assayed variant itself.""" - - clingen_allele_id: str - variant_effect_measurements: list[VariantEffectMeasurementWithShortScoreSet] - - -class ClingenAlleleIdVariantLookupResponse(BaseModel): - """Response model for a variant lookup by ClinGen allele ID""" - - clingen_allele_id: str - exact_match: Optional[ClingenAlleleVariants] = None - equivalent_nt: list[ClingenAlleleVariants] = [] - equivalent_aa: list[ClingenAlleleVariants] = [] diff --git a/tests/lib/test_allele_measurements.py b/tests/lib/test_allele_measurements.py index 6c49f6713..e10cf95bd 100644 --- a/tests/lib/test_allele_measurements.py +++ b/tests/lib/test_allele_measurements.py @@ -223,6 +223,39 @@ def test_sibling_nt_not_pulled_onto_ca_page(session, setup_lib_db_with_score_set assert all(m.relationship == "nucleotide_encoding" for m in pa_page) +@pytest.mark.integration +def test_sibling_nt_pulled_onto_ca_page_with_flag(session, setup_lib_db_with_score_set): + """``include_nucleotide_siblings`` widens a CA page through the protein consequence: the sibling nt + change encoding the same protein is pulled in as a ``nucleotide_encoding`` while the direct measurement + is unchanged. A no-op for a PA query, which already returns every encoding.""" + score_set = setup_lib_db_with_score_set + nt1 = _allele(session, "nt-1", level="cdna", clingen_allele_id="CA111") + nt2 = _allele(session, "nt-2", level="cdna", clingen_allele_id="CA222") + prot = _allele(session, "prot-P", level="protein", clingen_allele_id="PA9") + + a = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}) + ra = _record(session, a, assay_level="cdna") + _link(session, ra, nt1, is_authoritative=True) + _link(session, ra, prot) + + c = _variant(session, score_set, 2, data={"score_data": {"score": 1.0}}) + rc = _record(session, c, assay_level="cdna") + _link(session, rc, nt2, is_authoritative=True) + _link(session, rc, prot) + + result = get_allele_measurements(session, "CA111", user_data=_user_data(session), include_nucleotide_siblings=True) + by_urn = {m.variant_urn: m for m in result} + assert set(by_urn) == {a.urn, c.urn} + assert by_urn[a.urn].relationship == "direct" + assert by_urn[c.urn].relationship == "nucleotide_encoding" + # Display order: the directly-measured change precedes its sibling nucleotide encoding. + assert [m.variant_urn for m in result] == [a.urn, c.urn] + + # No-op for a protein anchor — it already returns every nt encoding. + pa = get_allele_measurements(session, "PA9", user_data=_user_data(session), include_nucleotide_siblings=True) + assert {m.variant_urn for m in pa} == {a.urn, c.urn} + + @pytest.mark.integration def test_private_score_set_measurement_excluded(session, setup_lib_db_with_score_set): """Score-set READ gates inclusion: a measurement in a score set the caller cannot read never leaks, diff --git a/tests/routers/test_clingen_allele.py b/tests/routers/test_clingen_allele.py index 7d75739db..e0236a134 100644 --- a/tests/routers/test_clingen_allele.py +++ b/tests/routers/test_clingen_allele.py @@ -104,6 +104,42 @@ def test_superseded_measurement_is_opt_in(client, session, data_provider, data_f assert body[0]["supersededByScoreSet"] == newer["urn"] +def test_nucleotide_siblings_are_opt_in(client, session, data_provider, data_files, setup_router_db): + """The sibling nt bucket is a discovery opt-in: a different DNA variant encoding the same protein + consequence is absent by default and pulled in as a ``nucleotide_encoding`` under + ``include_nucleotide_siblings``.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + prot = Allele(vrs_digest="prot-digest", level="protein", clingen_allele_id="PA9") + session.add(prot) + session.commit() + + # Two coding measurements encoding the same protein consequence (PA9), each carrying its own CA. + for suffix, caid, digest in ((1, "CA111", "cdna-1"), (2, "CA222", "cdna-2")): + variant = session.scalar(select(VariantDbModel).where(VariantDbModel.urn == f"{score_set['urn']}#{suffix}")) + record = MappingRecord(variant_id=variant.id, assay_level="cdna", mapping_api_version="test.0.0") + session.add(record) + session.commit() + nt = Allele(vrs_digest=digest, level="cdna", clingen_allele_id=caid) + session.add(nt) + session.commit() + session.add(MappingRecordAllele(mapping_record_id=record.id, allele_id=nt.id, is_authoritative=True)) + session.add(MappingRecordAllele(mapping_record_id=record.id, allele_id=prot.id)) + session.commit() + + default = client.get("/api/v1/clingen-alleles/CA111/measurements") + assert {m["variantUrn"] for m in default.json()} == {f"{score_set['urn']}#1"} + + widened = client.get("/api/v1/clingen-alleles/CA111/measurements", params={"include_nucleotide_siblings": True}) + assert widened.status_code == 200 + by_urn = {m["variantUrn"]: m for m in widened.json()} + assert set(by_urn) == {f"{score_set['urn']}#1", f"{score_set['urn']}#2"} + assert by_urn[f"{score_set['urn']}#1"]["relationship"] == "direct" + assert by_urn[f"{score_set['urn']}#2"]["relationship"] == "nucleotide_encoding" + + def test_anonymous_cannot_see_private_measurement( client, session, data_provider, data_files, setup_router_db, anonymous_app_overrides ): From 0a365d05f6f11714d8f9e4b8df8d2f19b9fbbbd2 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Fri, 10 Jul 2026 13:45:11 -0700 Subject: [PATCH 073/252] refactor(annotations): rename AnnotationLayer to SequenceLevel and type assay_level MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rename the AnnotationLayer enum to SequenceLevel to reflect that the same closed set (protein/cdna/genomic) serves several duties across the schema — the assayed level, the alignment level, and an allele's level — rather than reading as dcd-mapping's QC vocabulary alone. The name also harmonizes with the *_level columns and fields it types. Carry the enum through to consumers so assay_level is documented as a closed set instead of a free string: - type assay_level as Optional[SequenceLevel] on the lean-variant, variant-detail, and allele-measurement responses, so OpenAPI emits the enum members - type the matching lib transit dataclasses the same way, coercing the stored column string to the enum at each construction boundary - rename the mapping worker's local to sequence_level where it holds the enum (the dcd wire-code loop var keeps its name) The stored values and wire format are unchanged (SequenceLevel is a str enum), so no data migration is required and existing clients are unaffected. --- .../migrate_target_gene_mapping_qc.py | 40 +++++++++---------- src/mavedb/__init__.py | 2 +- src/mavedb/lib/allele_measurements.py | 14 +++---- src/mavedb/lib/cat_vrs.py | 10 ++--- src/mavedb/lib/score_set_variants.py | 36 ++++++++++------- src/mavedb/lib/variant_detail.py | 14 +++---- src/mavedb/models/enums/annotation_layer.py | 31 -------------- src/mavedb/models/enums/sequence_level.py | 34 ++++++++++++++++ src/mavedb/models/mapped_variant.py | 4 +- src/mavedb/models/mapping_record.py | 24 +++++++++-- src/mavedb/models/target_gene_mapping.py | 4 +- src/mavedb/view_models/allele_measurement.py | 3 +- src/mavedb/view_models/lean_variant.py | 5 ++- src/mavedb/view_models/mapped_variant.py | 4 +- src/mavedb/view_models/target_gene_mapping.py | 4 +- src/mavedb/view_models/variant_detail.py | 3 +- .../worker/jobs/variant_processing/mapping.py | 20 +++++----- .../variant_processing/reverse_translation.py | 12 +++--- tests/lib/test_score_set_variants.py | 2 +- .../jobs/variant_processing/test_mapping.py | 10 ++--- .../test_reverse_translation.py | 16 ++++---- 21 files changed, 160 insertions(+), 132 deletions(-) delete mode 100644 src/mavedb/models/enums/annotation_layer.py create mode 100644 src/mavedb/models/enums/sequence_level.py diff --git a/alembic/manual_migrations/migrate_target_gene_mapping_qc.py b/alembic/manual_migrations/migrate_target_gene_mapping_qc.py index 4e793ffe5..41eec4665 100644 --- a/alembic/manual_migrations/migrate_target_gene_mapping_qc.py +++ b/alembic/manual_migrations/migrate_target_gene_mapping_qc.py @@ -76,7 +76,7 @@ from mavedb.models import * # noqa: F401,F403 pylint: disable=wildcard-import from mavedb.db.session import SessionLocal from mavedb.lib.variants import HGVS_G_REGEX, HGVS_P_REGEX, get_hgvs_from_post_mapped -from mavedb.models.enums.annotation_layer import AnnotationLayer +from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.models.mapped_variant import MappedVariant from mavedb.models.score_set import ScoreSet from mavedb.models.target_gene import TargetGene @@ -102,18 +102,18 @@ _NUCLEOTIDE_ALPHABET = frozenset("ACGTUNRYSWKMBDHV-") -def _layer_from_hgvs(hgvs: str) -> Optional[AnnotationLayer]: - """Classify an HGVS string into an :class:`AnnotationLayer`.""" +def _layer_from_hgvs(hgvs: str) -> Optional[SequenceLevel]: + """Classify an HGVS string into an :class:`SequenceLevel`.""" if HGVS_G_REGEX.search(hgvs): - return AnnotationLayer.genomic + return SequenceLevel.genomic if HGVS_P_REGEX.search(hgvs): - return AnnotationLayer.protein + return SequenceLevel.protein if _HGVS_C_REGEX.search(hgvs): - return AnnotationLayer.cdna + return SequenceLevel.cdna return None -def _layer_from_vrs_sequence(post_mapped: dict) -> Optional[AnnotationLayer]: +def _layer_from_vrs_sequence(post_mapped: dict) -> Optional[SequenceLevel]: """ Infer layer from VRS sequence alphabet (protein vs nucleotide). Returns protein if any allele/member sequence contains a non-nucleotide letter. @@ -150,15 +150,15 @@ def extract_sequences(obj): for seq in extract_sequences(post_mapped): found_seq = True if any(letter.upper() not in _NUCLEOTIDE_ALPHABET for letter in seq if letter.isalpha()): - return AnnotationLayer.protein + return SequenceLevel.protein if found_seq: - return AnnotationLayer.genomic + return SequenceLevel.genomic return None -def _alignment_level_for(target: TargetGene) -> Optional[AnnotationLayer]: +def _alignment_level_for(target: TargetGene) -> Optional[SequenceLevel]: """Recover the alignment layer for legacy mapped variants of ``target``. Tries ``post_mapped_metadata`` first, then ``pre_mapped_metadata`` (same @@ -177,16 +177,16 @@ def _alignment_level_for(target: TargetGene) -> Optional[AnnotationLayer]: return None if has_genomic: - return AnnotationLayer.genomic + return SequenceLevel.genomic if has_protein: - return AnnotationLayer.protein + return SequenceLevel.protein if has_cdna: - return AnnotationLayer.cdna + return SequenceLevel.cdna return None -def _layer_from_sequence_type(target: TargetGene) -> Optional[AnnotationLayer]: +def _layer_from_sequence_type(target: TargetGene) -> Optional[SequenceLevel]: """Infer alignment layer from ``target_sequence.sequence_type`` and ``category``. Last-resort fallback for targets where all variant-level mappings failed @@ -201,16 +201,16 @@ def _layer_from_sequence_type(target: TargetGene) -> Optional[AnnotationLayer]: return None seq_type = target.target_sequence.sequence_type if seq_type == "protein": - return AnnotationLayer.protein + return SequenceLevel.protein if seq_type == "dna" and target.category is not None and target.category.value == "protein_coding": - return AnnotationLayer.genomic + return SequenceLevel.genomic return None def _populate_layer_by_target_id( db: Session, targets_by_score_set_id: dict[int, list[TargetGene]], - layer_by_target_id: dict[int, Optional[AnnotationLayer]], + layer_by_target_id: dict[int, Optional[SequenceLevel]], layer_via_hgvs_target_ids: set[int], layer_via_sequence_type_target_ids: set[int], ) -> None: @@ -344,7 +344,7 @@ def do_migration(db: Session) -> None: # Cache (target_gene_id, alignment_level, tool_version) -> persisted TargetGeneMapping # so we create one row per distinct combination and reuse it across every # mapped variant that maps into it. - cache: dict[tuple[int, AnnotationLayer, str], TargetGeneMapping] = {} + cache: dict[tuple[int, SequenceLevel, str], TargetGeneMapping] = {} total = db.scalar(sa.select(sa.func.count(MappedVariant.id))) or 0 print(f" {total} mapped variants to consider.") @@ -393,7 +393,7 @@ def do_migration(db: Session) -> None: # only need to determine it once per target -- and any single attributable mapped # variant for that target lets us attribute every sibling. The via_* sets track which # fallback was used so the diagnostic report shows attribution breadth. - layer_by_target_id: dict[int, Optional[AnnotationLayer]] = {} + layer_by_target_id: dict[int, Optional[SequenceLevel]] = {} layer_via_hgvs_target_ids: set[int] = set() layer_via_sequence_type_target_ids: set[int] = set() _populate_layer_by_target_id( @@ -420,7 +420,7 @@ def do_migration(db: Session) -> None: # Group ids to update by (tgm_id, level) so we can issue one bulk UPDATE # per group instead of a per-row ORM flush. - updates_by_group: dict[tuple[int, AnnotationLayer], list[int]] = defaultdict(list) + updates_by_group: dict[tuple[int, SequenceLevel], list[int]] = defaultdict(list) for row in chunk: last_id = row.id diff --git a/src/mavedb/__init__.py b/src/mavedb/__init__.py index 4669471c8..0b82b3395 100644 --- a/src/mavedb/__init__.py +++ b/src/mavedb/__init__.py @@ -6,7 +6,7 @@ logger = module_logging.getLogger(__name__) __project__ = "mavedb-api" -__version__ = "2026.2.6" +__version__ = "2026.2.6-dev" logger.info(f"MaveDB {__version__}") diff --git a/src/mavedb/lib/allele_measurements.py b/src/mavedb/lib/allele_measurements.py index a3048cb83..eaa988baf 100644 --- a/src/mavedb/lib/allele_measurements.py +++ b/src/mavedb/lib/allele_measurements.py @@ -37,7 +37,7 @@ from mavedb.lib.types.authentication import UserData from mavedb.lib.variants import variant_score from mavedb.models.allele import Allele -from mavedb.models.enums.annotation_layer import AnnotationLayer +from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.models.mapping_record import MappingRecord from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.score_calibration import ScoreCalibration @@ -76,7 +76,7 @@ class AlleleMeasurement: variant_urn: str score: Optional[float] - assay_level: Optional[str] + assay_level: Optional[SequenceLevel] relationship: str assay_level_hgvs: Optional[str] submitted_hgvs: Optional[str] @@ -169,7 +169,7 @@ def get_allele_measurements( return [] anchor_ids = [row.id for row in anchor] - entry_is_protein = any(row.level == AnnotationLayer.protein.value for row in anchor) + entry_is_protein = any(row.level == SequenceLevel.protein.value for row in anchor) # Sibling nucleotide changes (search discovery, CA only): fold the queried change's protein consequence # — the protein alleles co-membered with the anchor nt alleles — into the anchor, so the single record @@ -184,7 +184,7 @@ def get_allele_measurements( .where(anchor_link.allele_id.in_(anchor_ids)) .where(anchor_link.live_at(as_of)) .where(MappingRecordAllele.live_at(as_of)) - .where(Allele.level == AnnotationLayer.protein.value) + .where(Allele.level == SequenceLevel.protein.value) .distinct() ).all() @@ -240,19 +240,19 @@ def get_allele_measurements( # branch is unreachable (a record links the anchor only via itself or its protein consequence). if measured_allele.clingen_allele_id == clingen_allele_id: relationship = MeasurementRelationship.direct - elif measured_allele.level == AnnotationLayer.protein.value: + elif measured_allele.level == SequenceLevel.protein.value: relationship = MeasurementRelationship.protein_consequence else: relationship = MeasurementRelationship.nucleotide_encoding - assay_level = record.assay_level + assay_level = SequenceLevel(record.assay_level) if record.assay_level else None measurement = AlleleMeasurement( variant_urn=variant.urn or "", score=variant_score(variant), assay_level=assay_level, relationship=relationship, assay_level_hgvs=record.hgvs_assay_level, - submitted_hgvs=variant.hgvs_pro if assay_level == AnnotationLayer.protein.value else variant.hgvs_nt, + submitted_hgvs=variant.hgvs_pro if assay_level == SequenceLevel.protein.value else variant.hgvs_nt, score_set_urn=score_set.urn or "", score_set_title=score_set.title or "", primary_classification=_preferred_classification(db, variant, user_data=user_data), diff --git a/src/mavedb/lib/cat_vrs.py b/src/mavedb/lib/cat_vrs.py index 498c13542..10f1920ec 100644 --- a/src/mavedb/lib/cat_vrs.py +++ b/src/mavedb/lib/cat_vrs.py @@ -31,7 +31,7 @@ from mavedb.lib.annotation.util import vrs_object_from_mapped_variant from mavedb.lib.logging.context import logging_context from mavedb.models.allele import Allele -from mavedb.models.enums.annotation_layer import AnnotationLayer +from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.models.mapping_record_allele import MappingRecordAllele logger = logging.getLogger(__name__) @@ -60,7 +60,7 @@ class CatVrsMode(str, Enum): REVERSE_TRANSLATION = "reverse_translation" # Mode 2 — protein measured; score is implied. -_NUCLEOTIDE_LEVELS = {AnnotationLayer.genomic.value, AnnotationLayer.cdna.value} +_NUCLEOTIDE_LEVELS = {SequenceLevel.genomic.value, SequenceLevel.cdna.value} @dataclass @@ -80,11 +80,11 @@ def _relation_for(defining_level: Optional[str], member_level: Optional[str]) -> the (defining, member) level pair, never on a chain between members. """ # Mode 2 — protein measured. Every nt member encodes it; the protein member is the defining. - if defining_level == AnnotationLayer.protein.value: + if defining_level == SequenceLevel.protein.value: return CatVrsRelation.ENCODES if member_level in _NUCLEOTIDE_LEVELS else None # Mode 1 — nt measured (genomic or cdna). - if member_level == AnnotationLayer.protein.value: + if member_level == SequenceLevel.protein.value: return CatVrsRelation.TRANSLATION_OF if member_level in _NUCLEOTIDE_LEVELS: return CatVrsRelation.COORDINATE_REPRESENTATION_OF @@ -141,7 +141,7 @@ def build_categorical_variant(links: list[MappingRecordAllele], *, name: str) -> return None defining_level = defining_allele.level - mode = CatVrsMode.REVERSE_TRANSLATION if defining_level == AnnotationLayer.protein.value else CatVrsMode.PROJECTION + mode = CatVrsMode.REVERSE_TRANSLATION if defining_level == SequenceLevel.protein.value else CatVrsMode.PROJECTION members: list[VrsAllele | CisPhasedBlock | iriReference] = [defining_vrs] member_relations: dict[str, CatVrsRelation] = {} diff --git a/src/mavedb/lib/score_set_variants.py b/src/mavedb/lib/score_set_variants.py index b2a2e060a..f0c75390f 100644 --- a/src/mavedb/lib/score_set_variants.py +++ b/src/mavedb/lib/score_set_variants.py @@ -37,7 +37,7 @@ from mavedb.lib.hgvs import parse_simple_substitution from mavedb.lib.variants import score_from_variant_data from mavedb.models.allele import Allele -from mavedb.models.enums.annotation_layer import AnnotationLayer +from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.models.mapping_record import MappingRecord from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.score_set import ScoreSet @@ -84,7 +84,7 @@ class LeanVariantRecord: """One pre-chewed per-variant record: the selection key (``variant_urn``), the baseline ``score``, a representative (lossy) VEP ``consequence``, the bridge identifiers into the annotation dimensions (``clingen_allele_id``, ``assay_level_digest``), the submitted HGVS at each level, and the mapped - (reference-frame) representation as an ``assay_level`` pointer (an ``AnnotationLayer`` value naming + (reference-frame) representation as an ``assay_level`` pointer (an ``SequenceLevel`` value naming the measured/canonical slot) plus a ``mapped`` :class:`MappedTriple`. ``mapped[assay_level]`` is the measured representation; ``mapped.cdna`` is the level-invariant search key. Any field is ``None`` when its source is absent (unmapped variant → null ``assay_level`` + empty triple).""" @@ -97,10 +97,16 @@ class LeanVariantRecord: hgvs_nt: Optional[HgvsField] hgvs_pro: Optional[HgvsField] hgvs_splice: Optional[HgvsField] - assay_level: Optional[str] + assay_level: Optional[SequenceLevel] mapped: MappedTriple +def _level(value: Optional[str]) -> Optional[SequenceLevel]: + """Coerce a stored assay-level string (the ``mapping_records.assay_level`` column) into the closed + :class:`SequenceLevel` set. ``None`` for an absent value.""" + return SequenceLevel(value) if value else None + + def _hgvs_field_for_str(hgvs: Optional[str]) -> Optional[HgvsField]: """Wrap an HGVS string as an :class:`HgvsField`, attaching a parsed block when it is a placeable simple substitution. ``None`` for an absent string.""" @@ -114,7 +120,7 @@ def _hgvs_field_for_str(hgvs: Optional[str]) -> Optional[HgvsField]: def _mapped_triple( *, - assay_level: Optional[str], + assay_level: Optional[SequenceLevel], assay_level_hgvs: Optional[str], sibling_level: Optional[str], sibling_hgvs: Optional[str], @@ -133,19 +139,19 @@ def _mapped_triple( """ slots: dict[str, Optional[HgvsField]] = {} # Protein assay: the measured slot *is* protein. No canonical c/g — do not fabricate one. - if assay_level == AnnotationLayer.protein.value: - slots[AnnotationLayer.protein.value] = _hgvs_field_for_str(assay_level_hgvs) - elif assay_level in (AnnotationLayer.cdna.value, AnnotationLayer.genomic.value): - slots[assay_level] = _hgvs_field_for_str(assay_level_hgvs) + if assay_level == SequenceLevel.protein: + slots[SequenceLevel.protein.value] = _hgvs_field_for_str(assay_level_hgvs) + elif assay_level in (SequenceLevel.cdna, SequenceLevel.genomic): + slots[assay_level.value] = _hgvs_field_for_str(assay_level_hgvs) # The other nucleotide level, from the projection_group sibling (null where unpopulated). if sibling_level is not None: slots[sibling_level] = _hgvs_field_for_str(sibling_hgvs) - slots[AnnotationLayer.protein.value] = _hgvs_field_for_str(protein_hgvs) + slots[SequenceLevel.protein.value] = _hgvs_field_for_str(protein_hgvs) return MappedTriple( - genomic=slots.get(AnnotationLayer.genomic.value), - cdna=slots.get(AnnotationLayer.cdna.value), - protein=slots.get(AnnotationLayer.protein.value), + genomic=slots.get(SequenceLevel.genomic.value), + cdna=slots.get(SequenceLevel.cdna.value), + protein=slots.get(SequenceLevel.protein.value), ) @@ -258,7 +264,7 @@ def get_lean_score_set_variants( # covers record-liveness — no prot_record.live_at needed; the MR join is just the scoping bridge. .where(prot_link.live_at(as_of)) # Only the protein-level allele. - .where(prot_allele.level == AnnotationLayer.protein.value) + .where(prot_allele.level == SequenceLevel.protein.value) # DISTINCT ON requires ORDER BY to lead with its column; allele.id then picks which row survives # per record — inert today (only one protein allele exists). .order_by(prot_link.mapping_record_id, prot_allele.id) @@ -279,9 +285,9 @@ def get_lean_score_set_variants( hgvs_nt=_hgvs_field_for_str(row.hgvs_nt), hgvs_pro=_hgvs_field_for_str(row.hgvs_pro), hgvs_splice=_hgvs_field_for_str(row.hgvs_splice), - assay_level=row.assay_level, + assay_level=_level(row.assay_level), mapped=_mapped_triple( - assay_level=row.assay_level, + assay_level=_level(row.assay_level), assay_level_hgvs=row.hgvs_assay_level, sibling_level=row.sibling_level, sibling_hgvs=row.sibling_hgvs, diff --git a/src/mavedb/lib/variant_detail.py b/src/mavedb/lib/variant_detail.py index e833545a8..1f3cde103 100644 --- a/src/mavedb/lib/variant_detail.py +++ b/src/mavedb/lib/variant_detail.py @@ -28,7 +28,7 @@ from mavedb.lib.alleles import get_live_record_allele_links from mavedb.lib.cat_vrs import categorical_variant_for_variant from mavedb.lib.score_calibrations import calibration_preference_key -from mavedb.models.enums.annotation_layer import AnnotationLayer +from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.models.mapping_record import MappingRecord from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification @@ -114,7 +114,7 @@ class VariantDetail: classifications: list[VariantClassificationRecord] # Flat, UI-ergonomic assay-level fields. - assay_level: Optional[str] + assay_level: Optional[SequenceLevel] target_hgvs: Optional[str] # submitted, target/assay coordinates reference_hgvs: Optional[str] # mapped, reference coordinates, assay level assay_level_digest: Optional[str] @@ -170,7 +170,7 @@ def _classifications_for_variant( ] -def _derivation_for(*, is_authoritative: bool, assay_level: Optional[str]) -> AlleleDerivation: +def _derivation_for(*, is_authoritative: bool, assay_level: Optional[SequenceLevel]) -> AlleleDerivation: """The provenance of a linked allele's representation, from ``is_authoritative`` + the assay level. The measured allele is ``authoritative``. Every *other* allele's confidence is set by the *only* @@ -188,16 +188,16 @@ def _derivation_for(*, is_authoritative: bool, assay_level: Optional[str]) -> Al """ if is_authoritative: return AlleleDerivation.AUTHORITATIVE - if assay_level == AnnotationLayer.protein.value: + if assay_level == SequenceLevel.protein.value: return AlleleDerivation.CANDIDATE return AlleleDerivation.PROJECTION -def _submitted_assay_level_hgvs(variant: Variant, assay_level: Optional[str]) -> Optional[str]: +def _submitted_assay_level_hgvs(variant: Variant, assay_level: Optional[SequenceLevel]) -> Optional[str]: """The depositor-submitted HGVS in the variant's assay frame: protein for a protein assay, otherwise the nucleotide expression (genomic or coding share ``hgvs_nt`` and ``hgvs_splice`` is never an index column).""" - if assay_level == AnnotationLayer.protein.value: + if assay_level == SequenceLevel.protein.value: return variant.hgvs_pro return variant.hgvs_nt @@ -226,7 +226,7 @@ def get_variant_detail( record = db.scalar( select(MappingRecord).where(MappingRecord.variant_id == variant.id).where(MappingRecord.live_at(as_of)) ) - assay_level = record.assay_level if record is not None else None + assay_level = SequenceLevel(record.assay_level) if record is not None else None reference_hgvs = record.hgvs_assay_level if record is not None else None # The live allele links: the authoritative allele gives the assay-level digest + ClinGen id; all diff --git a/src/mavedb/models/enums/annotation_layer.py b/src/mavedb/models/enums/annotation_layer.py deleted file mode 100644 index 2115f8e99..000000000 --- a/src/mavedb/models/enums/annotation_layer.py +++ /dev/null @@ -1,31 +0,0 @@ -from enum import Enum - - -class AnnotationLayer(str, Enum): - """Annotation layer for a variant mapping result. - - Mirrors the ``AnnotationLayer`` enum produced by the dcd-mapping QC API. - Values use full names so they round-trip readably through the database - column; the dcd-mapping payload uses short single-character codes - (``p`` / ``c`` / ``g``) which the worker translates via :func:`from_wire`. - """ - - protein = "protein" - cdna = "cdna" - genomic = "genomic" - - @classmethod - def from_wire(cls, code: str) -> "AnnotationLayer": - """Translate a dcd-mapping single-character code into an ``AnnotationLayer``.""" - try: - return _WIRE_TO_LAYER[code] - except KeyError as exc: - raise ValueError(f"Unknown annotation_level wire code: {code!r}") from exc - - -# Module-level so it doesn't get mistaken for an enum member. -_WIRE_TO_LAYER: dict[str, AnnotationLayer] = { - "p": AnnotationLayer.protein, - "c": AnnotationLayer.cdna, - "g": AnnotationLayer.genomic, -} diff --git a/src/mavedb/models/enums/sequence_level.py b/src/mavedb/models/enums/sequence_level.py new file mode 100644 index 000000000..016c36f96 --- /dev/null +++ b/src/mavedb/models/enums/sequence_level.py @@ -0,0 +1,34 @@ +from enum import Enum + + +class SequenceLevel(str, Enum): + """The molecular sequence level of a variant representation: genomic DNA, coding DNA, or protein. + + A single, duty-neutral closed set reused across several columns that each carry a different + semantic meaning: the level a variant was *assayed* at (``assay_level``), the level dcd-mapping + *aligned* it at (``alignment_level``), and the level of a stored allele (``level``). + + Values use full names so they round-trip readably through the database column; the dcd-mapping + payload uses short single-character codes (``p`` / ``c`` / ``g``) which the worker translates via + :func:`from_wire`. + """ + + protein = "protein" + cdna = "cdna" + genomic = "genomic" + + @classmethod + def from_wire(cls, code: str) -> "SequenceLevel": + """Translate a dcd-mapping single-character code into a ``SequenceLevel``.""" + try: + return _WIRE_TO_LEVEL[code] + except KeyError as exc: + raise ValueError(f"Unknown sequence level wire code: {code!r}") from exc + + +# Module-level so it doesn't get mistaken for an enum member. +_WIRE_TO_LEVEL: dict[str, SequenceLevel] = { + "p": SequenceLevel.protein, + "c": SequenceLevel.cdna, + "g": SequenceLevel.genomic, +} diff --git a/src/mavedb/models/mapped_variant.py b/src/mavedb/models/mapped_variant.py index 40325cf17..10914f778 100644 --- a/src/mavedb/models/mapped_variant.py +++ b/src/mavedb/models/mapped_variant.py @@ -7,7 +7,7 @@ from mavedb.db.base import Base from mavedb.models.clinical_control_mapped_variant import mapped_variants_clinical_controls_association_table -from mavedb.models.enums.annotation_layer import AnnotationLayer +from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.models.gnomad_variant_mapped_variant import gnomad_variants_mapped_variants_association_table if TYPE_CHECKING: @@ -44,7 +44,7 @@ class MappedVariant(Base): # Per-mapping QC annotations from dcd-mapping ScoreAnnotation. alignment_level = Column( - Enum(AnnotationLayer, create_constraint=True, length=16, native_enum=False, validate_strings=True), + Enum(SequenceLevel, create_constraint=True, length=16, native_enum=False, validate_strings=True), nullable=True, ) at_mismatched_locus = Column(Boolean, nullable=True) diff --git a/src/mavedb/models/mapping_record.py b/src/mavedb/models/mapping_record.py index 132a69b8b..dda9ef16e 100644 --- a/src/mavedb/models/mapping_record.py +++ b/src/mavedb/models/mapping_record.py @@ -9,7 +9,7 @@ from mavedb.db.base import Base from mavedb.db.mixins import ValidTime from mavedb.lib.hgvs import extract_accession -from mavedb.models.enums.annotation_layer import AnnotationLayer +from mavedb.models.enums.sequence_level import SequenceLevel if TYPE_CHECKING: from .mapping_record_allele import MappingRecordAllele @@ -30,8 +30,24 @@ class MappingRecord(ValidTime, Base): vrs_digest = Column(String, nullable=True) pre_mapped: Optional[Any] = Column(JSONB(none_as_null=True), nullable=True) - # Level at which the variant was *assayed* — distinct from alignment_level (QC). - assay_level = Column(String(length=16), nullable=False) + # Level at which the variant was *assayed* — distinct from alignment_level (QC). Same closed + # SequenceLevel set as its sibling below. The prod CHECK (ck_mapping_records_assay_level_valid) + # predates this column being typed as an Enum; native_enum=False keeps it a VARCHAR(16), so + # declaring the Enum here needs no migration — it self-documents the model and gives the + # metadata-built (test) schema the same guard. + assay_level: Mapped[SequenceLevel] = Column( + # Distinct `name` so the generated CHECK doesn't collide with alignment_level's in this same + # table (both default to "sequencelevel", which Postgres rejects as a duplicate per table). + Enum( + SequenceLevel, + name="assay_level", + create_constraint=True, + length=16, + native_enum=False, + validate_strings=True, + ), + nullable=False, + ) hgvs_assay_level = Column(String, nullable=True) @hybrid_property @@ -60,7 +76,7 @@ def _transcript_expression(cls): # Per-mapping QC fields from dcd-mapping. alignment_level = Column( - Enum(AnnotationLayer, create_constraint=True, length=16, native_enum=False, validate_strings=True), + Enum(SequenceLevel, create_constraint=True, length=16, native_enum=False, validate_strings=True), nullable=True, ) at_mismatched_locus = Column(Boolean, nullable=True) diff --git a/src/mavedb/models/target_gene_mapping.py b/src/mavedb/models/target_gene_mapping.py index 52761e34b..5ead4a887 100644 --- a/src/mavedb/models/target_gene_mapping.py +++ b/src/mavedb/models/target_gene_mapping.py @@ -19,7 +19,7 @@ from sqlalchemy.orm import Mapped, relationship from mavedb.db.base import Base -from mavedb.models.enums.annotation_layer import AnnotationLayer +from mavedb.models.enums.sequence_level import SequenceLevel if TYPE_CHECKING: from mavedb.models.mapped_variant import MappedVariant @@ -36,7 +36,7 @@ class TargetGeneMapping(Base): target_gene: Mapped["TargetGene"] = relationship("TargetGene", back_populates="target_gene_mappings") alignment_level = Column( - Enum(AnnotationLayer, create_constraint=True, length=16, native_enum=False, validate_strings=True), + Enum(SequenceLevel, create_constraint=True, length=16, native_enum=False, validate_strings=True), nullable=False, ) preferred = Column(Boolean, nullable=False, default=False, server_default="false") diff --git a/src/mavedb/view_models/allele_measurement.py b/src/mavedb/view_models/allele_measurement.py index d61197524..8ac03275f 100644 --- a/src/mavedb/view_models/allele_measurement.py +++ b/src/mavedb/view_models/allele_measurement.py @@ -8,6 +8,7 @@ from typing import Optional from mavedb.lib.allele_measurements import MeasurementRelationship +from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.view_models.base.base import BaseModel from mavedb.view_models.score_calibration import SavedFunctionalClassification @@ -27,7 +28,7 @@ class AlleleMeasurement(BaseModel): variant_urn: str score: Optional[float] = None - assay_level: Optional[str] = None + assay_level: Optional[SequenceLevel] = None relationship: MeasurementRelationship assay_level_hgvs: Optional[str] = None submitted_hgvs: Optional[str] = None diff --git a/src/mavedb/view_models/lean_variant.py b/src/mavedb/view_models/lean_variant.py index 31f163524..474484e40 100644 --- a/src/mavedb/view_models/lean_variant.py +++ b/src/mavedb/view_models/lean_variant.py @@ -10,6 +10,7 @@ from typing import Optional +from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.view_models.base.base import BaseModel @@ -52,7 +53,7 @@ class LeanVariant(BaseModel): ``variantUrn`` is the universal selection key; ``assayLevelDigest`` bridges into the digest-keyed annotation dimensions. The submitted HGVS (``hgvsNt``/``hgvsPro``/``hgvsSplice``, target frame) carry the depositor's frame for the heatmap's raw↔mapped toggle. The mapped (reference) frame is the - ``mapped`` :class:`MappedTriple` plus the ``assayLevel`` pointer (an ``AnnotationLayer`` value) naming + ``mapped`` :class:`MappedTriple` plus the ``assayLevel`` pointer (an ``SequenceLevel`` value) naming the measured/canonical slot: ``mapped[assayLevel]`` is the measured representation and ``mapped.cdna`` the level-invariant search key. Fields are omitted when null. """ @@ -65,7 +66,7 @@ class LeanVariant(BaseModel): hgvs_nt: Optional[HgvsField] = None hgvs_pro: Optional[HgvsField] = None hgvs_splice: Optional[HgvsField] = None - assay_level: Optional[str] = None + assay_level: Optional[SequenceLevel] = None mapped: MappedTriple = MappedTriple() class Config: diff --git a/src/mavedb/view_models/mapped_variant.py b/src/mavedb/view_models/mapped_variant.py index 2d4856c88..218f0ba8d 100644 --- a/src/mavedb/view_models/mapped_variant.py +++ b/src/mavedb/view_models/mapped_variant.py @@ -7,7 +7,7 @@ from pydantic import model_validator from mavedb.lib.validation.exceptions import ValidationError -from mavedb.models.enums.annotation_layer import AnnotationLayer +from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.view_models import record_type_validator, set_record_type from mavedb.view_models.base.base import BaseModel from mavedb.view_models.target_gene_mapping import SavedTargetGeneMapping, TargetGeneMapping @@ -28,7 +28,7 @@ class MappedVariantBase(BaseModel): current: bool # Per-mapping QC annotations/warnings. - alignment_level: Optional[AnnotationLayer] = None + alignment_level: Optional[SequenceLevel] = None at_mismatched_locus: Optional[bool] = None near_gap: Optional[bool] = None diff --git a/src/mavedb/view_models/target_gene_mapping.py b/src/mavedb/view_models/target_gene_mapping.py index 13e64d1a7..535508a42 100644 --- a/src/mavedb/view_models/target_gene_mapping.py +++ b/src/mavedb/view_models/target_gene_mapping.py @@ -7,13 +7,13 @@ from datetime import date from typing import Any, Optional -from mavedb.models.enums.annotation_layer import AnnotationLayer +from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.view_models import record_type_validator, set_record_type from mavedb.view_models.base.base import BaseModel class TargetGeneMappingBase(BaseModel): - alignment_level: AnnotationLayer + alignment_level: SequenceLevel preferred: bool = False reference_assembly: Optional[str] = None diff --git a/src/mavedb/view_models/variant_detail.py b/src/mavedb/view_models/variant_detail.py index 1ae03def8..b14d2fa7a 100644 --- a/src/mavedb/view_models/variant_detail.py +++ b/src/mavedb/view_models/variant_detail.py @@ -8,6 +8,7 @@ from typing import Any, Optional +from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.view_models.allele_annotation import AlleleAnnotations from mavedb.view_models.base.base import BaseModel from mavedb.view_models.score_calibration import SavedFunctionalClassification @@ -76,7 +77,7 @@ class VariantDetail(BaseModel): counts: Optional[dict[str, Any]] = None classifications: list[VariantClassification] = [] - assay_level: Optional[str] = None + assay_level: Optional[SequenceLevel] = None target_hgvs: Optional[str] = None reference_hgvs: Optional[str] = None assay_level_digest: Optional[str] = None diff --git a/src/mavedb/worker/jobs/variant_processing/mapping.py b/src/mavedb/worker/jobs/variant_processing/mapping.py index b2f588b72..14d62fcdf 100644 --- a/src/mavedb/worker/jobs/variant_processing/mapping.py +++ b/src/mavedb/worker/jobs/variant_processing/mapping.py @@ -29,7 +29,7 @@ from mavedb.lib.variant_translations import get_or_create_allele from mavedb.lib.variants import get_hgvs_from_post_mapped from mavedb.models.allele import Allele as AlleleDbModel -from mavedb.models.enums.annotation_layer import AnnotationLayer +from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.models.enums.annotation_type import AnnotationType from mavedb.models.enums.disposition import Disposition from mavedb.models.enums.job_pipeline import FailureCategory @@ -184,7 +184,7 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan for annotation_layer in reference_metadata[target_gene_identifier]["layers"]: # ``annotation_layer`` arrives as a dcd-mapping wire code (``p``/``c``/``g``); # we persist metadata under the corresponding full-name enum value. - layer_name = AnnotationLayer.from_wire(annotation_layer).value + layer_name = SequenceLevel.from_wire(annotation_layer).value layer_premapped = reference_metadata[target_gene_identifier]["layers"][annotation_layer].get( "computed_reference_sequence" ) @@ -225,7 +225,7 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan target_gene_mapping = TargetGeneMapping( target_gene=target_gene, - alignment_level=AnnotationLayer.from_wire(level_value), + alignment_level=SequenceLevel.from_wire(level_value), preferred=bool(tm.get("preferred", False)), reference_assembly=tm.get("reference_assembly"), reference_accession=tm.get("reference_accession"), @@ -314,7 +314,7 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan pre_mapped_allele: dict = mapped_score.get("pre_mapped") or {} post_mapped_allele: dict = mapped_score.get("post_mapped") or {} - annotation_layer = AnnotationLayer.from_wire(score_alignment_level) + sequence_level = SequenceLevel.from_wire(score_alignment_level) assay_level_hgvs = get_hgvs_from_post_mapped(post_mapped_allele, combine_cis=True) # dcd-mapping guarantees every mapped score is attributable to a TargetGeneMapping @@ -331,13 +331,13 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan variant_id=variant.id, vrs_digest=pre_mapped_allele.get("id"), pre_mapped=pre_mapped_allele or None, - assay_level=annotation_layer, + assay_level=sequence_level, hgvs_assay_level=assay_level_hgvs, mapped_date=mapping_results["mapped_date"], vrs_version=mapped_score.get("vrs_version", None), mapping_api_version=tool_version, target_gene_mapping_id=target_gene_mapping_row.id, - alignment_level=annotation_layer, + alignment_level=sequence_level, at_mismatched_locus=mapped_score.get("at_mismatched_locus"), near_gap=mapped_score.get("near_gap"), ) @@ -380,10 +380,10 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan if post_mapped_allele: allele_draft = AlleleDbModel( vrs_digest=post_mapped_allele["id"], - level=annotation_layer, - hgvs_g=assay_level_hgvs if annotation_layer == AnnotationLayer.genomic else None, - hgvs_c=assay_level_hgvs if annotation_layer == AnnotationLayer.cdna else None, - hgvs_p=assay_level_hgvs if annotation_layer == AnnotationLayer.protein else None, + level=sequence_level, + hgvs_g=assay_level_hgvs if sequence_level == SequenceLevel.genomic else None, + hgvs_c=assay_level_hgvs if sequence_level == SequenceLevel.cdna else None, + hgvs_p=assay_level_hgvs if sequence_level == SequenceLevel.protein else None, post_mapped=post_mapped_allele, ) authoritative_allele = get_or_create_allele(job_manager.db, allele_draft) diff --git a/src/mavedb/worker/jobs/variant_processing/reverse_translation.py b/src/mavedb/worker/jobs/variant_processing/reverse_translation.py index 8d861270a..a6cdfc257 100644 --- a/src/mavedb/worker/jobs/variant_processing/reverse_translation.py +++ b/src/mavedb/worker/jobs/variant_processing/reverse_translation.py @@ -37,7 +37,7 @@ from mavedb.lib.variant_translations import get_or_create_allele from mavedb.lib.vrs_utils import translate_hgvs_to_variation from mavedb.models.allele import Allele as AlleleDbModel -from mavedb.models.enums.annotation_layer import AnnotationLayer +from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.models.enums.annotation_type import AnnotationType from mavedb.models.enums.disposition import Disposition from mavedb.models.enums.event_reason import EventReason @@ -221,7 +221,7 @@ async def reverse_translate_variants_for_score_set( ) .join(TargetGene, TargetGene.id == TargetGeneMapping.target_gene_id) .where(TargetGene.score_set_id == score_set_id) - .where(TargetGeneMapping.alignment_level == AnnotationLayer.cdna) + .where(TargetGeneMapping.alignment_level == SequenceLevel.cdna) .where(TargetGeneMapping.reference_accession.isnot(None)) .order_by(TargetGeneMapping.id) ) @@ -398,18 +398,18 @@ async def reverse_translate_variants_for_score_set( # failed, yielding a well-formed one-member (coding only) group rather than a desync. The # group id is assigned once per pair here, so the two members are guaranteed to carry the # same id even though they translate and link independently below. - members: list[tuple[str, AnnotationLayer, str, int | None]] = [] + members: list[tuple[str, SequenceLevel, str, int | None]] = [] for group_id, pair in enumerate(result.projection_pairs): - members.append((pair.hgvs_c, AnnotationLayer.cdna, "hgvs_c", group_id)) + members.append((pair.hgvs_c, SequenceLevel.cdna, "hgvs_c", group_id)) if pair.hgvs_g is not None: - members.append((pair.hgvs_g, AnnotationLayer.genomic, "hgvs_g", group_id)) + members.append((pair.hgvs_g, SequenceLevel.genomic, "hgvs_g", group_id)) # The protein consequence is the apex of the equivalence set — shared across every # pair, a member of none — so it carries no group (None). Prediction parens # (p.(Ala222Val)) are stripped before translation and storage. None for protein-assay # inputs, where the protein is already the authoritative allele. if result.hgvs_p: - members.append((strip_protein_prediction_parens(result.hgvs_p), AnnotationLayer.protein, "hgvs_p", None)) + members.append((strip_protein_prediction_parens(result.hgvs_p), SequenceLevel.protein, "hgvs_p", None)) for hgvs, level, hgvs_field, projection_group in members: # A candidate may not be translatable (intronic projection, malformed expression). diff --git a/tests/lib/test_score_set_variants.py b/tests/lib/test_score_set_variants.py index 4c0cbb5b0..6ac84bb12 100644 --- a/tests/lib/test_score_set_variants.py +++ b/tests/lib/test_score_set_variants.py @@ -275,7 +275,7 @@ def test_ordered_by_variant_number(session, setup_lib_db_with_score_set): @pytest.mark.integration -def test_as_of_reconstructs_the_historical_annotation_layer(session, setup_lib_db_with_score_set): +def test_as_of_reconstructs_the_historical_assay_level(session, setup_lib_db_with_score_set): """A re-map retires the old record and inserts a new one; as_of reconstructs the mapped layer live at a past instant, while the immutable submitted HGVS and score are unaffected.""" score_set = setup_lib_db_with_score_set diff --git a/tests/worker/jobs/variant_processing/test_mapping.py b/tests/worker/jobs/variant_processing/test_mapping.py index 96321d816..d4134ab05 100644 --- a/tests/worker/jobs/variant_processing/test_mapping.py +++ b/tests/worker/jobs/variant_processing/test_mapping.py @@ -13,7 +13,7 @@ from mavedb.lib.mapping import EXCLUDED_PREMAPPED_ANNOTATION_KEYS from mavedb.lib.types.workflow import JobExecutionOutcome from mavedb.models.allele import Allele -from mavedb.models.enums.annotation_layer import AnnotationLayer +from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.models.enums.job_pipeline import JobStatus, PipelineStatus from mavedb.models.enums.mapping_state import MappingState from mavedb.models.mapping_record import MappingRecord @@ -514,7 +514,7 @@ async def dummy_mapping_job(): assert result.status == JobStatus.SUCCEEDED cdna_tgms = ( - session.query(TargetGeneMapping).filter(TargetGeneMapping.alignment_level == AnnotationLayer.cdna).all() + session.query(TargetGeneMapping).filter(TargetGeneMapping.alignment_level == SequenceLevel.cdna).all() ) assert len(cdna_tgms) == len(sample_score_set.target_genes) tgm = cdna_tgms[0] @@ -894,7 +894,7 @@ async def dummy_mapping_job(): session.commit() mapping_record = MappingRecord( variant_id=variant.id, - assay_level=AnnotationLayer.genomic, + assay_level=SequenceLevel.genomic, mapped_date=date(2023, 1, 1), mapping_api_version="v1.0.0", ) @@ -902,7 +902,7 @@ async def dummy_mapping_job(): session.commit() # Link the prior record to an authoritative allele. Re-mapping must retire this link too, # not just the record — a live link dangling off a retired record breaks the temporal model. - prior_allele = Allele(vrs_digest="ga4gh:VA.prior", level=AnnotationLayer.genomic.value) + prior_allele = Allele(vrs_digest="ga4gh:VA.prior", level=SequenceLevel.genomic.value) session.add(prior_allele) session.commit() prior_link = MappingRecordAllele( @@ -1421,7 +1421,7 @@ async def test_map_variants_for_score_set_updates_current_mapped_variants( for variant in variants: mapping_record = MappingRecord( variant_id=variant.id, - assay_level=AnnotationLayer.genomic, + assay_level=SequenceLevel.genomic, mapped_date=date(2023, 1, 1), mapping_api_version="v1.0.0", ) diff --git a/tests/worker/jobs/variant_processing/test_reverse_translation.py b/tests/worker/jobs/variant_processing/test_reverse_translation.py index 7bc33cf99..030d17020 100644 --- a/tests/worker/jobs/variant_processing/test_reverse_translation.py +++ b/tests/worker/jobs/variant_processing/test_reverse_translation.py @@ -18,7 +18,7 @@ from mavedb.lib.types.workflow import JobExecutionOutcome from mavedb.models.allele import Allele -from mavedb.models.enums.annotation_layer import AnnotationLayer +from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.models.enums.job_pipeline import JobStatus from mavedb.models.enums.target_category import TargetCategory from mavedb.models.job_run import JobRun @@ -315,14 +315,14 @@ async def test_creates_genomic_and_coding_candidate_alleles( assert len(non_auth_links) == 2 genomic_allele = session.query(Allele).filter(Allele.vrs_digest == "ga4gh:VA.genomic").one() - assert genomic_allele.level == AnnotationLayer.genomic.value + assert genomic_allele.level == SequenceLevel.genomic.value assert genomic_allele.hgvs_g == g_candidate assert genomic_allele.hgvs_c is None assert genomic_allele.transcript == "NC_000001.11" assert genomic_allele.post_mapped == {"type": "Allele", "id": "ga4gh:VA.genomic"} coding_allele = session.query(Allele).filter(Allele.vrs_digest == "ga4gh:VA.coding").one() - assert coding_allele.level == AnnotationLayer.cdna.value + assert coding_allele.level == SequenceLevel.cdna.value assert coding_allele.hgvs_c == c_candidate assert coding_allele.hgvs_g is None assert coding_allele.transcript == "NM_000001.1" @@ -431,7 +431,7 @@ async def test_protein_consequence_is_persisted_as_a_protein_allele( await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) protein_allele = session.query(Allele).filter(Allele.vrs_digest == "ga4gh:VA.protein").one() - assert protein_allele.level == AnnotationLayer.protein.value + assert protein_allele.level == SequenceLevel.protein.value assert protein_allele.hgvs_p == p_stripped # stored without the prediction parens # Linked to the record as a non-authoritative member of the equivalence set, but grouped @@ -485,7 +485,7 @@ async def test_cis_phased_multivariant_candidate_is_persisted_as_a_block( assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 2} block_allele = session.query(Allele).filter(Allele.vrs_digest == "ga4gh:CPB.block").one() - assert block_allele.level == AnnotationLayer.genomic.value + assert block_allele.level == SequenceLevel.genomic.value # The full bracketed expression is stored verbatim; its accession anchors the row. assert block_allele.hgvs_g == block_candidate assert block_allele.transcript == "NC_000001.11" @@ -1152,7 +1152,7 @@ def _run_mapped_date(self, session, target_gene_id): session.query(TargetGeneMapping) .filter( TargetGeneMapping.target_gene_id == target_gene_id, - TargetGeneMapping.alignment_level == AnnotationLayer.genomic, + TargetGeneMapping.alignment_level == SequenceLevel.genomic, ) .first() .mapped_date @@ -1192,7 +1192,7 @@ async def test_uses_latest_cdna_row_within_the_run( session.add( TargetGeneMapping( target_gene_id=target_gene.id, - alignment_level=AnnotationLayer.cdna, + alignment_level=SequenceLevel.cdna, reference_accession="NM_111111.1", preferred=False, tool_version="pytest.0.0", @@ -1253,7 +1253,7 @@ async def test_ignores_stale_cdna_row_from_a_different_run( session.add( TargetGeneMapping( target_gene_id=target_gene.id, - alignment_level=AnnotationLayer.cdna, + alignment_level=SequenceLevel.cdna, reference_accession="NM_888888.1", preferred=False, tool_version="pytest.0.0", From ccf5b69234997045a19a60af6f7306c1866b2787 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Fri, 10 Jul 2026 14:07:48 -0700 Subject: [PATCH 074/252] fix(docs): remove design references from endpoint documentation --- src/mavedb/lib/allele_annotations.py | 4 ++-- src/mavedb/lib/variant_detail.py | 2 +- src/mavedb/view_models/allele_measurement.py | 2 +- src/mavedb/view_models/variant_detail.py | 6 +++--- tests/routers/test_clingen_allele.py | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/mavedb/lib/allele_annotations.py b/src/mavedb/lib/allele_annotations.py index 1b6cf112b..4e161dbf5 100644 --- a/src/mavedb/lib/allele_annotations.py +++ b/src/mavedb/lib/allele_annotations.py @@ -2,8 +2,8 @@ Gathers a set of alleles' external annotations — VEP consequence, gnomAD frequency, ClinVar assertions — keyed by ``vrs_digest``. The digest is the join key the serving envelopes use to ride -annotations *alongside* the spec-pure Cat-VRS members (design §8): Cat-VRS itself carries no -annotation slot, so annotations travel beside it in a flat digest-keyed map. +annotations *alongside* the spec-pure Cat-VRS members: Cat-VRS itself carries no annotation slot, so +annotations travel beside it in a flat digest-keyed map. All three sources are ``ValidTime`` and ``allele_id``-keyed with no reverse collection on ``Allele`` (they are navigated set-wise from the link tables). ``as_of`` reconstructs them at a past diff --git a/src/mavedb/lib/variant_detail.py b/src/mavedb/lib/variant_detail.py index 1f3cde103..987e0e3ea 100644 --- a/src/mavedb/lib/variant_detail.py +++ b/src/mavedb/lib/variant_detail.py @@ -7,7 +7,7 @@ digest-keyed external-annotation map (:mod:`lib.allele_annotations`). Also carries the per-calibration functional ``classifications`` the variant falls into, and its version standing (``is_current`` / ``superseded_by_score_set``) so a superseded variant self-describes rather -than reading as current (design §9.2). +than reading as current. Temporal scope of ``as_of``: only the **molecular** layer is versioned — Cat-VRS membership and the VEP/gnomAD/ClinVar annotations reconstruct at the past instant. Scores are immutable (``Variant`` is diff --git a/src/mavedb/view_models/allele_measurement.py b/src/mavedb/view_models/allele_measurement.py index 8ac03275f..592285d12 100644 --- a/src/mavedb/view_models/allele_measurement.py +++ b/src/mavedb/view_models/allele_measurement.py @@ -1,5 +1,5 @@ """Response view model for the variant page's measurements list -(``GET /clingen-alleles/{caid}/measurements``, design §7.5). +(``GET /clingen-alleles/{caid}/measurements``). The pydantic serialization boundary over the ``lib.allele_measurements`` transit dataclass (``from_attributes`` coerces it directly); aliases camelize per the shared base config. diff --git a/src/mavedb/view_models/variant_detail.py b/src/mavedb/view_models/variant_detail.py index b14d2fa7a..190f41d7a 100644 --- a/src/mavedb/view_models/variant_detail.py +++ b/src/mavedb/view_models/variant_detail.py @@ -1,4 +1,4 @@ -"""Response view models for the assayed variant-detail view (``GET /variants/{urn}``, design §7.1). +"""Response view models for the assayed variant-detail view (``GET /variants/{urn}``). The pydantic serialization boundary over the ``lib.variant_detail`` transit dataclasses (``from_attributes`` coerces them directly); aliases camelize per the shared base config. Kept in its @@ -34,7 +34,7 @@ class AlleleIdentity(BaseModel): """The MaveDB molecular-identity facts for one of the variant's linked alleles. An entry of the ``alleles`` sidecar, keyed by VRS digest. ``level`` + ``hgvs`` (the reference-frame - HGVS) are what the UI labels the per-level annotation panel by — never the digest (design §7.5). + HGVS) are what the UI labels the per-level annotation panel by — never the digest. Three independent axes: - ``relation`` (Cat-VRS, structural): member→defining relation; ``null`` when it *is* the measured @@ -58,7 +58,7 @@ class Config: class VariantDetail(BaseModel): - """The assayed variant-detail envelope (``GET /variants/{urn}``, design §7.1). + """The assayed variant-detail envelope (``GET /variants/{urn}``). Two tiers: flat, UI-ergonomic assay fields (the ``targetHgvs``/``referenceHgvs`` coordinate pair is a client-side toggle, no refetch) plus the spec-pure GA4GH ``molecularRepresentation`` diff --git a/tests/routers/test_clingen_allele.py b/tests/routers/test_clingen_allele.py index e0236a134..97d4fed43 100644 --- a/tests/routers/test_clingen_allele.py +++ b/tests/routers/test_clingen_allele.py @@ -1,6 +1,6 @@ # ruff: noqa: E402 """Router tests for the ClinGen-allele measurements endpoint -(``GET /clingen-alleles/{caid}/measurements``, design §7.5). +(``GET /clingen-alleles/{caid}/measurements``). Exercise the HTTP surface: the equivalence-class list serializes and validates, the ``as_of`` content-time is echoed in ``X-As-Of``, an unknown id is an empty list (not a 404), superseded From 19072d607b161545690cb960c7fa2662e3cd4352 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Sun, 12 Jul 2026 13:43:33 -0700 Subject: [PATCH 075/252] feat(allele-measurements): rank current measurements before superseded Add an is_current dimension as the top precedence in _ordering_key so current measurements sort ahead of superseded ones, above the existing direct-vs-related, evidence-strength, published-date, and urn keys. - Extend _ordering_key with the current/superseded ordinal - Add pure unit tests pinning the full precedence chain and the null-published-date fallback - Add router tests covering direct-before-related, current-before- superseded, newest-published-first, and the urn tiebreak in the serialized response body --- src/mavedb/lib/allele_measurements.py | 10 ++- tests/lib/test_allele_measurements.py | 71 ++++++++++++++- tests/routers/test_clingen_allele.py | 119 +++++++++++++++++++++++++- 3 files changed, 191 insertions(+), 9 deletions(-) diff --git a/src/mavedb/lib/allele_measurements.py b/src/mavedb/lib/allele_measurements.py index eaa988baf..9419ea244 100644 --- a/src/mavedb/lib/allele_measurements.py +++ b/src/mavedb/lib/allele_measurements.py @@ -131,13 +131,15 @@ def preference(pair: tuple[ScoreCalibrationFunctionalClassification, ScoreCalibr def _ordering_key(measurement: AlleleMeasurement, published_date: Optional[date]) -> tuple: """Sort order: - 1. Direct measurements, then related (protein_consequence / nucleotide_encoding) - 2. Within each, the strongest evidence first (pathogenic wins ties) - 3. Within each, the newest-published first - 4. Within each, the URN for a stable tiebreak + 1. Current measurements first, then superseded (the latter only if ``include_superseded``) + 2. Direct measurements first, then related (protein_consequence / nucleotide_encoding) + 3. Within each, the strongest evidence first (pathogenic wins ties) + 4. Within each, the newest-published first + 5. Within each, the URN for a stable tiebreak """ magnitude, direction = classification_evidence_strength(measurement.primary_classification) return ( + 0 if measurement.is_current else 1, 0 if measurement.relationship == MeasurementRelationship.direct else 1, 0 if measurement.primary_classification is not None else 1, -magnitude, diff --git a/tests/lib/test_allele_measurements.py b/tests/lib/test_allele_measurements.py index e10cf95bd..d64b33e6a 100644 --- a/tests/lib/test_allele_measurements.py +++ b/tests/lib/test_allele_measurements.py @@ -15,7 +15,14 @@ pytest.importorskip("psycopg2") -from mavedb.lib.allele_measurements import get_allele_measurements +from datetime import date + +from mavedb.lib.allele_measurements import ( + AlleleMeasurement, + MeasurementRelationship, + _ordering_key, + get_allele_measurements, +) from mavedb.lib.types.authentication import UserData from mavedb.models.allele import Allele from mavedb.models.mapping_record import MappingRecord @@ -402,3 +409,65 @@ def test_ordering_direct_first_then_strongest_pathogenic(session, setup_lib_db_w @pytest.mark.integration def test_unknown_clingen_id_returns_empty(session, setup_lib_db_with_score_set): assert get_allele_measurements(session, "CA000", user_data=_user_data(session)) == [] + + +def _oddspaths_classification(ratio): + """A transient functional classification carrying only an oddspaths ratio — enough for + ``classification_evidence_strength`` (which reads ``acmg_classification`` [None here] then the ratio) + to rank it, with no DB round-trip.""" + return ScoreCalibrationFunctionalClassification(oddspaths_ratio=ratio) + + +def _measurement(urn, *, relationship=MeasurementRelationship.direct, classification=None, is_current=True): + """A transit ``AlleleMeasurement`` with just the fields ``_ordering_key`` reads; the rest are filler.""" + return AlleleMeasurement( + variant_urn=urn, + score=None, + assay_level=None, + relationship=relationship, + assay_level_hgvs=None, + submitted_hgvs=None, + score_set_urn="", + score_set_title="", + primary_classification=classification, + is_current=is_current, + superseded_by_score_set=None, + ) + + +def test_ordering_key_ranks_every_dimension(): + """Pin the full precedence chain of ``_ordering_key`` as a pure unit (no DB, no calibration fixtures): + current → direct → classified → strongest magnitude → pathogenic direction → newest published → urn. + Each adjacent pair below differs by exactly the next-lower key, so sorting a shuffled copy must + reproduce this exact order.""" + strong_path = _oddspaths_classification(100.0) # magnitude |ln 100|, pathogenic + strong_benign = _oddspaths_classification(0.01) # equal magnitude, benign + weak_path = _oddspaths_classification(2.0) # smaller magnitude, pathogenic + + # (measurement, published_date), already in the expected ranked order. + ranked = [ + (_measurement("u#1", classification=strong_path), date(2024, 1, 1)), # strongest evidence, pathogenic + (_measurement("u#2", classification=strong_benign), date(2024, 1, 1)), # equal magnitude, benign loses + (_measurement("u#3", classification=weak_path), date(2024, 1, 1)), # weaker magnitude + (_measurement("u#4a", classification=None), date(2024, 6, 1)), # unclassified; newer published leads its tie + (_measurement("u#4b", classification=None), date(2024, 1, 1)), # unclassified, older published + (_measurement("u#4c", classification=None), date(2024, 1, 1)), # unclassified, same date → urn tiebreak + (_measurement("u#5", relationship=MeasurementRelationship.protein_consequence), date(2024, 6, 1)), # related + (_measurement("u#6", is_current=False), date(2024, 6, 1)), # superseded sorts last + ] + + shuffled = [ranked[i] for i in (5, 0, 7, 2, 4, 1, 6, 3)] + ordered = sorted(shuffled, key=lambda pair: _ordering_key(pair[0], pair[1])) + + assert [m.variant_urn for m, _ in ordered] == [m.variant_urn for m, _ in ranked] + + +def test_ordering_key_missing_published_date_sorts_after_dated(): + """A null published date is treated as oldest (``0`` ordinal), so a dated measurement leads an + otherwise-identical undated one.""" + dated = (_measurement("u#dated"), date(2024, 1, 1)) + undated = (_measurement("u#undated"), None) + + ordered = sorted([undated, dated], key=lambda pair: _ordering_key(pair[0], pair[1])) + + assert [m.variant_urn for m, _ in ordered] == ["u#dated", "u#undated"] diff --git a/tests/routers/test_clingen_allele.py b/tests/routers/test_clingen_allele.py index 97d4fed43..8c01ec8ef 100644 --- a/tests/routers/test_clingen_allele.py +++ b/tests/routers/test_clingen_allele.py @@ -4,9 +4,12 @@ Exercise the HTTP surface: the equivalence-class list serializes and validates, the ``as_of`` content-time is echoed in ``X-As-Of``, an unknown id is an empty list (not a 404), superseded -measurements are opt-in, and a private score set's measurement never leaks. +measurements are opt-in, a private score set's measurement never leaks, and the response body's +order reflects the ranked sort (current-then-superseded, direct-then-related, newest-published, urn). """ +from datetime import date + import pytest arq = pytest.importorskip("arq") @@ -27,9 +30,10 @@ from tests.helpers.util.user import change_ownership -def _seed_cdna_measurement(session, variant_urn, *, clingen_allele_id="CA123"): +def _seed_cdna_measurement(session, variant_urn, *, clingen_allele_id="CA123", vrs_digest="cdna-digest"): """Give a variant a live coding-measured mapping record whose authoritative allele carries the - ClinGen id — the minimal shape for a direct measurement on that CA's page.""" + ClinGen id — the minimal shape for a direct measurement on that CA's page. ``vrs_digest`` is + content-unique (DB constraint), so pass a distinct one when seeding more than one per test.""" variant = session.scalar(select(VariantDbModel).where(VariantDbModel.urn == variant_urn)) record = MappingRecord( variant_id=variant.id, @@ -40,7 +44,7 @@ def _seed_cdna_measurement(session, variant_urn, *, clingen_allele_id="CA123"): session.add(record) session.commit() - measured = Allele(vrs_digest="cdna-digest", level="cdna", clingen_allele_id=clingen_allele_id) + measured = Allele(vrs_digest=vrs_digest, level="cdna", clingen_allele_id=clingen_allele_id) session.add(measured) session.commit() session.add(MappingRecordAllele(mapping_record_id=record.id, allele_id=measured.id, is_authoritative=True)) @@ -140,6 +144,113 @@ def test_nucleotide_siblings_are_opt_in(client, session, data_provider, data_fil assert by_urn[f"{score_set['urn']}#2"]["relationship"] == "nucleotide_encoding" +def test_direct_measurements_sort_before_related(client, session, data_provider, data_files, setup_router_db): + """The list is ordered, and direct measurements precede related ones in the body: a CA+siblings query + returns the directly-assayed change first, then the sibling nucleotide encoding.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + prot = Allele(vrs_digest="prot-digest", level="protein", clingen_allele_id="PA9") + session.add(prot) + session.commit() + + for suffix, caid, digest in ((1, "CA111", "cdna-1"), (2, "CA222", "cdna-2")): + variant = session.scalar(select(VariantDbModel).where(VariantDbModel.urn == f"{score_set['urn']}#{suffix}")) + record = MappingRecord(variant_id=variant.id, assay_level="cdna", mapping_api_version="test.0.0") + session.add(record) + session.commit() + nt = Allele(vrs_digest=digest, level="cdna", clingen_allele_id=caid) + session.add(nt) + session.commit() + session.add(MappingRecordAllele(mapping_record_id=record.id, allele_id=nt.id, is_authoritative=True)) + session.add(MappingRecordAllele(mapping_record_id=record.id, allele_id=prot.id)) + session.commit() + + response = client.get("/api/v1/clingen-alleles/CA111/measurements", params={"include_nucleotide_siblings": True}) + + assert response.status_code == 200 + body = response.json() + assert [(m["variantUrn"], m["relationship"]) for m in body] == [ + (f"{score_set['urn']}#1", "direct"), + (f"{score_set['urn']}#2", "nucleotide_encoding"), + ] + + +def test_current_measurements_sort_before_superseded(client, session, data_provider, data_files, setup_router_db): + """With superseded measurements opted in, current ones sort ahead of superseded ones in the body.""" + experiment = create_experiment(client) + older = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + newer = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + older_db = session.scalar(select(ScoreSetDbModel).where(ScoreSetDbModel.urn == older["urn"])) + newer_db = session.scalar(select(ScoreSetDbModel).where(ScoreSetDbModel.urn == newer["urn"])) + newer_db.superseded_score_set_id = older_db.id + session.commit() + _seed_cdna_measurement(session, f"{older['urn']}#1", vrs_digest="cdna-older") + _seed_cdna_measurement(session, f"{newer['urn']}#1", vrs_digest="cdna-newer") + + response = client.get("/api/v1/clingen-alleles/CA123/measurements", params={"include_superseded": True}) + + assert response.status_code == 200 + body = response.json() + assert [(m["variantUrn"], m["isCurrent"]) for m in body] == [ + (f"{newer['urn']}#1", True), + (f"{older['urn']}#1", False), + ] + + +def test_measurements_sort_newest_published_first(client, session, data_provider, data_files, setup_router_db): + """Between two otherwise-equal current, unclassified measurements the newest-published sorts first — + published date outranks the urn tiebreak (here the newer set has the later urn, yet still leads).""" + experiment = create_experiment(client) + first = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + second = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + first_db = session.scalar(select(ScoreSetDbModel).where(ScoreSetDbModel.urn == first["urn"])) + second_db = session.scalar(select(ScoreSetDbModel).where(ScoreSetDbModel.urn == second["urn"])) + first_db.published_date = date(2024, 1, 1) + second_db.published_date = date(2024, 6, 1) # later-published, and later urn + session.commit() + _seed_cdna_measurement(session, f"{first['urn']}#1", vrs_digest="cdna-first") + _seed_cdna_measurement(session, f"{second['urn']}#1", vrs_digest="cdna-second") + + response = client.get("/api/v1/clingen-alleles/CA123/measurements") + + assert response.status_code == 200 + assert [m["variantUrn"] for m in response.json()] == [f"{second['urn']}#1", f"{first['urn']}#1"] + + +def test_measurements_tiebreak_on_urn(client, session, data_provider, data_files, setup_router_db): + """With every other sort key equal (both current, direct, unclassified, same published date) the urn is + the final, stable tiebreak — ascending.""" + experiment = create_experiment(client) + first = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + second = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + for urn in (first["urn"], second["urn"]): + score_set = session.scalar(select(ScoreSetDbModel).where(ScoreSetDbModel.urn == urn)) + score_set.published_date = date(2024, 1, 1) + session.commit() + _seed_cdna_measurement(session, f"{first['urn']}#1", vrs_digest="cdna-first") + _seed_cdna_measurement(session, f"{second['urn']}#1", vrs_digest="cdna-second") + + response = client.get("/api/v1/clingen-alleles/CA123/measurements") + + assert response.status_code == 200 + returned = [m["variantUrn"] for m in response.json()] + assert returned == sorted([f"{first['urn']}#1", f"{second['urn']}#1"]) + + def test_anonymous_cannot_see_private_measurement( client, session, data_provider, data_files, setup_router_db, anonymous_app_overrides ): From 0ff6cd0ca1c728b88af71ac797c8f21ff2f5454a Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Sun, 12 Jul 2026 21:27:36 -0700 Subject: [PATCH 076/252] feat(clinical-controls): tag control links with annotating allele digest Reshape the score-set clinical-controls response around the allele-link substrate and off the legacy mapped-variant model. - Introduce ControlVariantLink in the clinical-controls lib, carrying the VRS digest of the allele each ClinVar control annotates so clients can apply D78 precedence against the variant's authoritative assay-level digest - Extract the response leaf model out of mapped_variant.py into clinical_control.py as ClinvarVariantLink, dropping its dead variant-URN coercion validator and unused from_attributes - Rename the container view models to ClinicalControlWithClinvarLinks (and Saved variant) and the field mapped_variants -> clinvar_links - Update the router builder and clinical-control tests, including coverage for two controls reaching one variant via distinct alleles --- src/mavedb/lib/clinical_controls.py | 45 +++++++++++---- src/mavedb/routers/score_sets.py | 12 ++-- src/mavedb/view_models/clinical_control.py | 17 ++++-- src/mavedb/view_models/mapped_variant.py | 16 ------ tests/helpers/constants.py | 8 +-- tests/routers/test_score_set.py | 65 +++++++++++++++++++--- 6 files changed, 115 insertions(+), 48 deletions(-) diff --git a/src/mavedb/lib/clinical_controls.py b/src/mavedb/lib/clinical_controls.py index 5aa10b1ce..b852ad39e 100644 --- a/src/mavedb/lib/clinical_controls.py +++ b/src/mavedb/lib/clinical_controls.py @@ -9,6 +9,7 @@ place. """ +from dataclasses import dataclass from datetime import datetime from typing import Any, Optional, TypeVar @@ -25,6 +26,21 @@ SelectT = TypeVar("SelectT", bound=Select[Any]) +@dataclass(frozen=True) +class ControlVariantLink: + """One (score-set variant, annotated allele) pair that a clinical control reaches. + + ``allele_digest`` is the VRS digest of the allele the control actually annotates — the entity + ClinVar made its call on. A client applies **D78 precedence** by comparing it to the variant's + authoritative ``assay_level_digest``: a match is a call *on the assayed level* (which wins), a + mismatch is a call on a *projection sibling* (the fallback that only fires when the assayed level + is unannotated). Without the digest the two are indistinguishable and precedence is unimplementable. + """ + + variant_urn: str + allele_digest: str + + def _scope_to_live_score_set_controls(stmt: SelectT, score_set_id: int, as_of: Optional[datetime]) -> SelectT: """Join ``stmt`` (already selecting from :class:`ClinvarControl`) down the allele-link chain to the score set's variants and constrain every ``ValidTime`` hop to ``as_of``. ``distinct`` because the @@ -50,12 +66,15 @@ def get_clinical_controls_with_variant_urns( as_of: Optional[datetime] = None, db_name: Optional[str] = None, db_version: Optional[str] = None, -) -> list[tuple[ClinvarControl, list[str]]]: - """The live ClinVar controls for a score set, each paired with the URNs of the score-set variants - that link to it. Optionally narrowed to one control DB (``db_name``) and/or release (``db_version``). - Controls come back in first-seen order; each control's variant URNs preserve query order.""" +) -> list[tuple[ClinvarControl, list[ControlVariantLink]]]: + """The live ClinVar controls for a score set, each paired with the score-set variants that link to + it — every link carrying the digest of the allele the control annotates (see + :class:`ControlVariantLink`). Optionally narrowed to one control DB (``db_name``) and/or release + (``db_version``). Controls come back in first-seen order; each control's links preserve query order.""" stmt = _scope_to_live_score_set_controls( - select(ClinvarControl, Variant.urn.label("variant_urn")), score_set_id, as_of + select(ClinvarControl, Variant.urn.label("variant_urn"), Allele.vrs_digest.label("allele_digest")), + score_set_id, + as_of, ) if db_name is not None: stmt = stmt.where(ClinvarControl.db_name == db_name) @@ -63,18 +82,20 @@ def get_clinical_controls_with_variant_urns( stmt = stmt.where(ClinvarControl.db_version == db_version) controls: dict[int, ClinvarControl] = {} - urns_by_control: dict[int, list[str]] = {} - for ctrl, variant_urn in db.execute(stmt).tuples(): - # A persisted variant reached through the join always carries a URN (unique, set at creation); - # narrow the nullable column so the URN list matches the view model's required `str`. + links_by_control: dict[int, list[ControlVariantLink]] = {} + + # A persisted variant reached through the join always carries a URN (unique, set at creation); + # narrow the nullable column so the URN matches the view model's required `str`. + for ctrl, variant_urn, allele_digest in db.execute(stmt).tuples(): if variant_urn is None: continue + if ctrl.id not in controls: controls[ctrl.id] = ctrl - urns_by_control[ctrl.id] = [] - urns_by_control[ctrl.id].append(variant_urn) + links_by_control[ctrl.id] = [] + links_by_control[ctrl.id].append(ControlVariantLink(variant_urn=variant_urn, allele_digest=allele_digest)) - return [(ctrl, urns_by_control[ctrl.id]) for ctrl in controls.values()] + return [(ctrl, links_by_control[ctrl.id]) for ctrl in controls.values()] def get_clinical_control_options( diff --git a/src/mavedb/routers/score_sets.py b/src/mavedb/routers/score_sets.py index 413d64a07..7a68e8550 100644 --- a/src/mavedb/routers/score_sets.py +++ b/src/mavedb/routers/score_sets.py @@ -2440,7 +2440,7 @@ async def publish_score_set( @router.get( "/score-sets/{urn}/clinical-controls", status_code=200, - response_model=list[clinical_control.ClinicalControlWithMappedVariants], + response_model=list[clinical_control.ClinicalControlWithClinvarLinks], response_model_exclude_none=True, responses={**ACCESS_CONTROL_ERROR_RESPONSES}, summary="Get clinical controls for a score set", @@ -2461,7 +2461,7 @@ async def get_clinical_controls_for_score_set( user_data: UserData = Depends(get_current_user), db: Optional[str] = None, version: Optional[str] = None, -) -> list[clinical_control.ClinicalControlWithMappedVariants]: +) -> list[clinical_control.ClinicalControlWithClinvarLinks]: """ Fetch relevant clinical controls for a given score set. """ @@ -2503,7 +2503,7 @@ async def get_clinical_controls_for_score_set( save_to_logging_context({"resource_count": len(controls)}) return [ - clinical_control.ClinicalControlWithMappedVariants.model_validate( + clinical_control.ClinicalControlWithClinvarLinks.model_validate( { "id": ctrl.id, "db_identifier": ctrl.db_identifier, @@ -2514,10 +2514,12 @@ async def get_clinical_controls_for_score_set( "db_name": ctrl.db_name, "modification_date": ctrl.modification_date, "creation_date": ctrl.creation_date, - "mapped_variants": [{"variant_urn": v_urn} for v_urn in variant_urns], + "clinvar_links": [ + {"variant_urn": link.variant_urn, "allele_digest": link.allele_digest} for link in links + ], } ) - for ctrl, variant_urns in controls + for ctrl, links in controls ] diff --git a/src/mavedb/view_models/clinical_control.py b/src/mavedb/view_models/clinical_control.py index 85cd78348..7e6a17d06 100644 --- a/src/mavedb/view_models/clinical_control.py +++ b/src/mavedb/view_models/clinical_control.py @@ -8,7 +8,7 @@ from mavedb.view_models.base.base import BaseModel if TYPE_CHECKING: - from mavedb.view_models.mapped_variant import MappedVariantCreate, MappedVariantForClinicalControl + from mavedb.view_models.mapped_variant import MappedVariantCreate class ClinicalControlBase(BaseModel): @@ -41,8 +41,17 @@ class Config: from_attributes = True -class SavedClinicalControlWithMappedVariants(SavedClinicalControl): - mapped_variants: Sequence["MappedVariantForClinicalControl"] +class ClinvarVariantLink(BaseModel): + """One score-set variant a ClinVar control reaches, tagged with the annotated allele's digest.""" + + variant_urn: str + # VRS digest of the allele this control annotates. Many controls + # may share the same variant_urn, but each will have a different allele_digest. + allele_digest: Optional[str] = None + + +class SavedClinicalControlWithClinvarLinks(SavedClinicalControl): + clinvar_links: Sequence[ClinvarVariantLink] # Properties to return to non-admin clients @@ -50,7 +59,7 @@ class ClinicalControl(SavedClinicalControl): pass -class ClinicalControlWithMappedVariants(SavedClinicalControlWithMappedVariants): +class ClinicalControlWithClinvarLinks(SavedClinicalControlWithClinvarLinks): pass diff --git a/src/mavedb/view_models/mapped_variant.py b/src/mavedb/view_models/mapped_variant.py index 218f0ba8d..7c03fd103 100644 --- a/src/mavedb/view_models/mapped_variant.py +++ b/src/mavedb/view_models/mapped_variant.py @@ -111,22 +111,6 @@ class MappedVariantWithControls(SavedMappedVariantWithControls): gnomad_variants: Sequence["GnomADVariant"] -class MappedVariantForClinicalControl(BaseModel): - variant_urn: str - - class Config: - from_attributes = True - - @model_validator(mode="before") - def generate_score_set_urn_list(cls, data: Any): - if not hasattr(data, "variant_urn") and hasattr(data, "variant"): - try: - data.__setattr__("variant_urn", None if not data.variant else data.variant.urn) - except AttributeError as exc: - raise ValidationError(f"Unable to create {cls.__name__} without attribute: {exc}.") # type: ignore - return data - - # ruff: noqa: E402 from mavedb.view_models.clinical_control import ClinicalControl, ClinicalControlBase, SavedClinicalControl from mavedb.view_models.gnomad_variant import GnomADVariant, GnomADVariantBase, SavedGnomADVariant diff --git a/tests/helpers/constants.py b/tests/helpers/constants.py index bf78d38fe..8766c2f53 100644 --- a/tests/helpers/constants.py +++ b/tests/helpers/constants.py @@ -1781,14 +1781,14 @@ TEST_SAVED_CLINVAR_CONTROL = { - "recordType": "ClinicalControlWithMappedVariants", + "recordType": "ClinicalControlWithClinvarLinks", "dbIdentifier": "183058", "geneSymbol": "PTEN", "clinicalSignificance": "Likely benign", "clinicalReviewStatus": "criteria provided, multiple submitters, no conflicts", "dbName": "ClinVar", "dbVersion": "11_2024", - "mappedVariants": [], + "clinvarLinks": [], } @@ -1803,14 +1803,14 @@ TEST_SAVED_GENERIC_CLINICAL_CONTROL = { - "recordType": "ClinicalControlWithMappedVariants", + "recordType": "ClinicalControlWithClinvarLinks", "dbIdentifier": "ABC123", "geneSymbol": "BRCA1", "clinicalSignificance": "benign", "clinicalReviewStatus": "lots of convincing evidence", "dbName": "GenDB", "dbVersion": "2024", - "mappedVariants": [], + "clinvarLinks": [], } diff --git a/tests/routers/test_score_set.py b/tests/routers/test_score_set.py index 632c0fa15..942f032c7 100644 --- a/tests/routers/test_score_set.py +++ b/tests/routers/test_score_set.py @@ -2892,9 +2892,7 @@ def test_search_score_sets_not_affected_by_experiment_metadata( assert response.json()["numScoreSets"] == num_score_sets -def test_cannot_create_multiple_superseding_versions( - session, data_provider, client, setup_router_db, data_files -): +def test_cannot_create_multiple_superseding_versions(session, data_provider, client, setup_router_db, data_files): """Attempting to create multiple superseding versions should fail.""" experiment = create_experiment(client, {"title": "Original Experiment"}) score_set = create_seq_score_set(client, experiment["urn"], update={"title": "Original Score Set"}) @@ -2920,7 +2918,9 @@ def test_cannot_create_multiple_superseding_versions( response = client.post("/api/v1/score-sets/", json=score_set_post_payload) assert response.status_code == 409 - assert (f"This score set has been superseded by score set: {first_superseding['urn']}.") in response.json()["detail"] + assert (f"This score set has been superseded by score set: {first_superseding['urn']}.") in response.json()[ + "detail" + ] def test_search_score_sets_not_affected_by_an_unpublishing_superseding_versions( @@ -3867,15 +3867,66 @@ def test_can_fetch_current_clinical_controls_for_score_set(client, setup_router_ response_data = response.json() assert len(response_data) == 2 for control in response_data: - mapped_variants = control.pop("mappedVariants") - assert len(mapped_variants) == 1 + clinvar_links = control.pop("clinvarLinks") + assert len(clinvar_links) == 1 + # Each link carries the digest of the allele the control annotates — the D78 precedence signal. + assert clinvar_links[0]["alleleDigest"] in ("clinical-control-allele-0", "clinical-control-allele-1") assert all( control[k] in (TEST_SAVED_CLINVAR_CONTROL[k], TEST_SAVED_GENERIC_CLINICAL_CONTROL[k]) for k in TEST_SAVED_CLINVAR_CONTROL.keys() - if k != "mappedVariants" + if k != "clinvarLinks" ) +def test_clinical_controls_tag_each_link_with_the_annotating_allele_digest( + client, setup_router_db, session, data_provider, data_files +): + """A protein-change variant whose DNA siblings carry conflicting ClinVar calls surfaces *both* + controls against the *same* variant URN — distinguishable only by the digest of the allele each + one annotates. That digest is D78's precedence signal: the control on the variant's authoritative + (assayed-level) allele is the direct call; the control on a projection sibling is the fallback. + Without it the two collapse to one variant and precedence is unimplementable.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + variant_urn = session.scalars( + select(VariantDbModel.urn) + .join(ScoreSetDbModel) + .where(ScoreSetDbModel.urn == score_set["urn"]) + .order_by(VariantDbModel.id) + ).first() + + # One variant, two alleles in its equivalence class: the assayed (authoritative) allele carries the + # ClinVar control (id 1); a projection sibling carries the generic control (id 2). + seed_mapping_record( + session, + variant_urn, + assay_level="protein", + alleles=[ + AlleleSpec(digest="assayed-level-allele", level="protein", is_authoritative=True, clinvar_control_ids=[1]), + AlleleSpec( + digest="projection-sibling-allele", level="cdna", is_authoritative=False, clinvar_control_ids=[2] + ), + ], + ) + + response = client.get(f"/api/v1/score-sets/{score_set['urn']}/clinical-controls") + assert response.status_code == 200 + + controls = {c["dbIdentifier"]: c for c in response.json()} + assert len(controls) == 2 + + assayed = controls[TEST_SAVED_CLINVAR_CONTROL["dbIdentifier"]] # control 1, on the authoritative allele + sibling = controls[TEST_SAVED_GENERIC_CLINICAL_CONTROL["dbIdentifier"]] # control 2, on the sibling + + # Both controls point at the same variant, so only the annotating allele's digest tells them apart. + assert [link["variantUrn"] for link in assayed["clinvarLinks"]] == [variant_urn] + assert [link["variantUrn"] for link in sibling["clinvarLinks"]] == [variant_urn] + assert assayed["clinvarLinks"][0]["alleleDigest"] == "assayed-level-allele" + assert sibling["clinvarLinks"][0]["alleleDigest"] == "projection-sibling-allele" + + @pytest.mark.parametrize("clinical_control", [TEST_SAVED_CLINVAR_CONTROL, TEST_SAVED_GENERIC_CLINICAL_CONTROL]) @pytest.mark.parametrize( "parameters", [[("db", "dbName")], [("version", "dbVersion")], [("db", "dbName"), ("version", "dbVersion")]] From bb369d9e30c1f6122472e341a57a94c17179a63b Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Mon, 13 Jul 2026 13:33:51 -0700 Subject: [PATCH 077/252] refactor(allele-measurements): expose preferred classification, exclude RUO Rename the measurement's inline classification field from ``primary_classification``/``primaryClassification`` to ``preferred_classification``/``preferredClassification`` to reflect that the surfaced value is the UI's default from the primary-first preference cascade, not necessarily the promoted primary calibration. - Exclude research-use-only calibrations outright from the preferred classification on this clinical surface, rather than merely deprioritizing them as the shared cascade does - Add a test covering RUO exclusion and fix a self-contradictory cascade fixture that marked a calibration primary while asserting an investigator-provided one should win --- src/mavedb/lib/allele_measurements.py | 28 ++++++++++------- src/mavedb/view_models/allele_measurement.py | 7 +++-- tests/lib/test_allele_measurements.py | 33 +++++++++++++++----- 3 files changed, 45 insertions(+), 23 deletions(-) diff --git a/src/mavedb/lib/allele_measurements.py b/src/mavedb/lib/allele_measurements.py index 9419ea244..6ca2a6d23 100644 --- a/src/mavedb/lib/allele_measurements.py +++ b/src/mavedb/lib/allele_measurements.py @@ -70,8 +70,8 @@ class AlleleMeasurement: ``assay_level`` is the level at which *this* measurement was assayed (protein/cdna/genomic) — the clinically load-bearing fact, always shown. ``relationship`` says how it relates to the queried - ClinGen id. ``primary_classification`` is the primary readable functional classification (``None`` - when the score set has none or the caller cannot read the calibration). + ClinGen id. ``preferred_classification`` is the readable functional classification the UI defaults to + (primary-first cascade, RUO excluded; ``None`` when none applies or the calibration is unreadable). """ variant_urn: str @@ -82,7 +82,7 @@ class AlleleMeasurement: submitted_hgvs: Optional[str] score_set_urn: str score_set_title: str - primary_classification: Optional[ScoreCalibrationFunctionalClassification] + preferred_classification: Optional[ScoreCalibrationFunctionalClassification] is_current: bool superseded_by_score_set: Optional[str] @@ -94,12 +94,15 @@ def _preferred_classification( A variant falls into one classification per calibration (non-overlapping ranges); we surface the one the UI would default to, mirroring its calibration preference cascade: ``primary`` first, then - ``investigator_provided``, then non-``research_use_only``. Within a tier we take the strongest - evidence (the VA-Spec posture — the strongest calibration is the evidence calibration), then ``id`` - only for determinism. The UI's "any with ranges / any calibration" tail steps collapse here: this - query only surfaces calibrations that already classify the variant. Only calibrations the caller can - read (calibration READ) are considered, so a private calibration is skipped rather than blanking a - readable call. Kept in sync with the UI's `activeCalibrationOptions` default selection. + ``investigator_provided``. Within a tier we take the strongest evidence (the VA-Spec posture — the + strongest calibration is the evidence calibration), then ``id`` only for determinism. + + This is a clinical surface, so research-use-only calibrations are excluded outright (not merely deprioritized + as in the shared cascade) — a RUO call is never the classification shown here. + + Only calibrations the caller can read (calibration READ) are considered, so a private calibration is + skipped rather than blanking a readable call. Kept in sync with the UI's `activeCalibrationOptions` + default selection. """ candidates = [ (classification, calibration) @@ -111,6 +114,7 @@ def _preferred_classification( ) .join(ScoreCalibration, ScoreCalibration.id == ScoreCalibrationFunctionalClassification.calibration_id) .where(classification_variants.c.variant_id == variant.id) + .where(ScoreCalibration.research_use_only.is_(False)) ).all() if has_permission(user_data, calibration, Action.READ).permitted ] @@ -137,11 +141,11 @@ def _ordering_key(measurement: AlleleMeasurement, published_date: Optional[date] 4. Within each, the newest-published first 5. Within each, the URN for a stable tiebreak """ - magnitude, direction = classification_evidence_strength(measurement.primary_classification) + magnitude, direction = classification_evidence_strength(measurement.preferred_classification) return ( 0 if measurement.is_current else 1, 0 if measurement.relationship == MeasurementRelationship.direct else 1, - 0 if measurement.primary_classification is not None else 1, + 0 if measurement.preferred_classification is not None else 1, -magnitude, direction, -(published_date.toordinal()) if published_date is not None else 0, @@ -257,7 +261,7 @@ def get_allele_measurements( submitted_hgvs=variant.hgvs_pro if assay_level == SequenceLevel.protein.value else variant.hgvs_nt, score_set_urn=score_set.urn or "", score_set_title=score_set.title or "", - primary_classification=_preferred_classification(db, variant, user_data=user_data), + preferred_classification=_preferred_classification(db, variant, user_data=user_data), is_current=is_current, superseded_by_score_set=superseding.urn if superseding is not None else None, ) diff --git a/src/mavedb/view_models/allele_measurement.py b/src/mavedb/view_models/allele_measurement.py index 592285d12..0baebaddd 100644 --- a/src/mavedb/view_models/allele_measurement.py +++ b/src/mavedb/view_models/allele_measurement.py @@ -20,8 +20,9 @@ class AlleleMeasurement(BaseModel): ``genomic``) — always shown, since the measured level is the clinically load-bearing fact. ``relationship`` says how the measurement relates to the queried ClinGen id: ``direct`` (assayed at this allele), ``protein_consequence`` (a protein measurement of a nt query's consequence), or - ``nucleotide_encoding`` (a nt measurement encoding a protein query). ``primaryClassification`` is the - primary readable functional classification, omitted when absent or gated. ``isCurrent`` / + ``nucleotide_encoding`` (a nt measurement encoding a protein query). ``preferredClassification`` is the + readable functional classification the UI defaults to (primary-first cascade, RUO excluded), omitted + when absent or gated. ``isCurrent`` / ``supersededByScoreSet`` let a superseded measurement (surfaced only under ``include_superseded``) self-describe; ``supersededByScoreSet`` is the superseding *score set*'s URN. """ @@ -34,7 +35,7 @@ class AlleleMeasurement(BaseModel): submitted_hgvs: Optional[str] = None score_set_urn: str score_set_title: str - primary_classification: Optional[SavedFunctionalClassification] = None + preferred_classification: Optional[SavedFunctionalClassification] = None is_current: bool superseded_by_score_set: Optional[str] = None diff --git a/tests/lib/test_allele_measurements.py b/tests/lib/test_allele_measurements.py index d64b33e6a..a4621c33b 100644 --- a/tests/lib/test_allele_measurements.py +++ b/tests/lib/test_allele_measurements.py @@ -294,15 +294,15 @@ def test_calibration_gate_withholds_classification(session, setup_lib_db_with_sc anonymous = get_allele_measurements(session, "CA123", user_data=None) assert len(anonymous) == 1 - assert anonymous[0].primary_classification is None + assert anonymous[0].preferred_classification is None owner = get_allele_measurements(session, "CA123", user_data=_user_data(session)) - assert owner[0].primary_classification is not None - assert owner[0].primary_classification.functional_classification.value == "abnormal" + assert owner[0].preferred_classification is not None + assert owner[0].preferred_classification.functional_classification.value == "abnormal" @pytest.mark.integration -def test_primary_classification_cascade_beats_strength(session, setup_lib_db_with_score_set): +def test_preferred_classification_cascade_beats_strength(session, setup_lib_db_with_score_set): """With no primary calibration, the preference cascade dominates evidence strength: an investigator-provided calibration wins over a stronger non-RUO one and a stronger-still RUO one.""" score_set = setup_lib_db_with_score_set @@ -317,11 +317,11 @@ def test_primary_classification_cascade_beats_strength(session, setup_lib_db_wit result = get_allele_measurements(session, "CA123", user_data=_user_data(session)) # Investigator-provided (weakest) is chosen over the stronger non-RUO and RUO calibrations. - assert result[0].primary_classification.oddspaths_ratio == 2.0 + assert result[0].preferred_classification.oddspaths_ratio == 2.0 @pytest.mark.integration -def test_primary_classification_strongest_within_tier(session, setup_lib_db_with_score_set): +def test_preferred_classification_strongest_within_tier(session, setup_lib_db_with_score_set): """Within one cascade tier (here two plain non-RUO calibrations), the strongest evidence wins.""" score_set = setup_lib_db_with_score_set nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") @@ -333,7 +333,24 @@ def test_primary_classification_strongest_within_tier(session, setup_lib_db_with result = get_allele_measurements(session, "CA123", user_data=_user_data(session)) - assert result[0].primary_classification.oddspaths_ratio == 100.0 + assert result[0].preferred_classification.oddspaths_ratio == 100.0 + + +@pytest.mark.integration +def test_research_use_only_calibrations_are_excluded(session, setup_lib_db_with_score_set): + """A research-use-only calibration is excluded from the cascade (even if it is the strongest + evidence).""" + score_set = setup_lib_db_with_score_set + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + variant = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}) + _link(session, _record(session, variant, assay_level="cdna"), nt, is_authoritative=True) + + _calibration(session, score_set, variants=[variant], primary=False, oddspaths_ratio=100.0, research_use_only=True) + _calibration(session, score_set, variants=[variant], primary=False, oddspaths_ratio=1000.0, research_use_only=True) + + result = get_allele_measurements(session, "CA123", user_data=_user_data(session)) + + assert result[0].preferred_classification is None @pytest.mark.integration @@ -429,7 +446,7 @@ def _measurement(urn, *, relationship=MeasurementRelationship.direct, classifica submitted_hgvs=None, score_set_urn="", score_set_title="", - primary_classification=classification, + preferred_classification=classification, is_current=is_current, superseded_by_score_set=None, ) From c1fb812b0065513438fd3732cfd0bf430e6bd7d9 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 15 Jul 2026 09:42:47 -0700 Subject: [PATCH 078/252] feat(clingen): batch CAR submissions and add HTTP timeouts Reverse translation can produce very large allele counts, so the CAR submission now chunks HGVS into fixed-size batches instead of a single PUT. Each batch is dispatched and reconciled independently, so a transient failure or a broken one-result-per-input response fails only that batch's alleles rather than the whole run. - add DEFAULT_CAR_SUBMISSION_BATCH_SIZE (env-configurable) and loop the dispatch/reconcile logic per batch, scaling progress across chunks - add an explicit (connect, read) timeout to all ClinGen HTTP calls so a stalled server can no longer hang the worker indefinitely; widen the LDH authenticate catch to RequestException - cover batch splitting and per-batch failure isolation with tests --- src/mavedb/lib/clingen/constants.py | 9 + src/mavedb/lib/clingen/services.py | 12 +- .../worker/jobs/external_services/clingen.py | 278 ++++++++++-------- .../jobs/external_services/test_clingen.py | 165 +++++++++++ 4 files changed, 332 insertions(+), 132 deletions(-) diff --git a/src/mavedb/lib/clingen/constants.py b/src/mavedb/lib/clingen/constants.py index 2276ee187..60f3a26cb 100644 --- a/src/mavedb/lib/clingen/constants.py +++ b/src/mavedb/lib/clingen/constants.py @@ -14,8 +14,17 @@ LDH_ENTITY_ENDPOINT = "maveDb" # for some reason, not the same :/ DEFAULT_LDH_SUBMISSION_BATCH_SIZE = 100 +DEFAULT_CAR_SUBMISSION_BATCH_SIZE = int(os.getenv("CAR_SUBMISSION_BATCH_SIZE", "10000")) +"""Number of HGVS strings sent to the ClinGen Allele Registry per PUT. Reverse translation can +produce very large allele counts, so submissions are chunked rather than sent as a single payload.""" CLINGEN_CACHE_WARMING_CONCURRENCY = int(os.getenv("CLINGEN_CACHE_WARMING_CONCURRENCY", "5")) """Maximum number of concurrent requests to make to the ClinGen API when pre-warming the cache for mapped variants.""" + +CLINGEN_CONNECT_TIMEOUT = float(os.getenv("CLINGEN_CONNECT_TIMEOUT", "5")) +CLINGEN_READ_TIMEOUT = float(os.getenv("CLINGEN_READ_TIMEOUT", "300")) +CLINGEN_HTTP_TIMEOUT = (CLINGEN_CONNECT_TIMEOUT, CLINGEN_READ_TIMEOUT) +"""(connect, read) timeout for ClinGen HTTP calls. Fail fast on connect, with a generous read +budget because a large batch can take CAR a while to respond.""" LDH_SUBMISSION_ENDPOINT = f"https://genboree.org/mq/brdg/pulsar/{CLIN_GEN_TENANT}/ldh/submissions/{LDH_ENTITY_ENDPOINT}" LDH_ACCESS_ENDPOINT = os.getenv("LDH_ACCESS_ENDPOINT", "https://genboree.org/ldh") LDH_MAVE_ACCESS_ENDPOINT = f"{LDH_ACCESS_ENDPOINT}/{LDH_ENTITY_NAME}/id" diff --git a/src/mavedb/lib/clingen/services.py b/src/mavedb/lib/clingen/services.py index 085d9103f..c8e80259f 100644 --- a/src/mavedb/lib/clingen/services.py +++ b/src/mavedb/lib/clingen/services.py @@ -8,7 +8,7 @@ import requests from jose import jwt -from mavedb.lib.clingen.constants import GENBOREE_ACCOUNT_NAME, GENBOREE_ACCOUNT_PASSWORD +from mavedb.lib.clingen.constants import CLINGEN_HTTP_TIMEOUT, GENBOREE_ACCOUNT_NAME, GENBOREE_ACCOUNT_PASSWORD from mavedb.lib.logging.context import format_raised_exception_info_as_dict, logging_context, save_to_logging_context from mavedb.lib.types.clingen import ClinGenAllele, ClinGenSubmissionError, LdhSubmission from mavedb.lib.utils import batched @@ -80,6 +80,7 @@ def dispatch_submissions( response = requests.put( url=request_url, data="\n".join(content_submissions), + timeout=CLINGEN_HTTP_TIMEOUT, ) response.raise_for_status() @@ -165,10 +166,10 @@ def authenticate(self) -> str: auth_url = f"https://genboree.org/auth/usr/gb:{GENBOREE_ACCOUNT_NAME}/auth" auth_body = {"type": "plain", "val": GENBOREE_ACCOUNT_PASSWORD} - auth_response = requests.post(auth_url, json=auth_body) try: + auth_response = requests.post(auth_url, json=auth_body, timeout=CLINGEN_HTTP_TIMEOUT) auth_response.raise_for_status() - except requests.exceptions.HTTPError as exc: + except requests.exceptions.RequestException as exc: save_to_logging_context(format_raised_exception_info_as_dict(exc)) logger.error(msg="Failed to authenticate with Genboree services.", exc_info=exc, extra=logging_context()) raise exc @@ -221,16 +222,17 @@ def dispatch_submissions( logger.info(msg=f"Dispatching {len(submissions)} ldh submissions...", extra=logging_context()) for idx, content in enumerate(submissions): try: - logger.debug(msg=f"Dispatching submission {idx+1}.", extra=logging_context()) + logger.debug(msg=f"Dispatching submission {idx + 1}.", extra=logging_context()) response = requests.put( url=self.url, json=content, headers={"Authorization": f"Bearer {self.authenticate()}", "Content-Type": "application/json"}, + timeout=CLINGEN_HTTP_TIMEOUT, ) response.raise_for_status() submission_successes.append(response.json()) logger.info( - msg=f"Successfully dispatched ldh submission ({idx+1} / {len(submissions)}).", + msg=f"Successfully dispatched ldh submission ({idx + 1} / {len(submissions)}).", extra=logging_context(), ) diff --git a/src/mavedb/worker/jobs/external_services/clingen.py b/src/mavedb/worker/jobs/external_services/clingen.py index e5a438ee1..d563e658c 100644 --- a/src/mavedb/worker/jobs/external_services/clingen.py +++ b/src/mavedb/worker/jobs/external_services/clingen.py @@ -21,6 +21,7 @@ from mavedb.lib.clingen.constants import ( CAR_SUBMISSION_ENDPOINT, CLIN_GEN_SUBMISSION_ENABLED, + DEFAULT_CAR_SUBMISSION_BATCH_SIZE, DEFAULT_LDH_SUBMISSION_BATCH_SIZE, LDH_SUBMISSION_ENDPOINT, ) @@ -30,6 +31,7 @@ ClinGenLdhService, ) from mavedb.lib.types.workflow import JobExecutionOutcome +from mavedb.lib.utils import batched from mavedb.lib.variants import get_hgvs_from_post_mapped from mavedb.models.allele import Allele as AlleleModel from mavedb.models.enums.annotation_type import AnnotationType @@ -113,12 +115,14 @@ async def submit_score_set_mappings_to_car(ctx: dict, job_id: int, job_manager: correlation_id = job.job_params["correlation_id"] # type: ignore force_reregister = bool(job.job_params.get("force_reregister", False)) # type: ignore[union-attr] - job_manager.save_to_context({ - "application": "mavedb-worker", - "function": "submit_score_set_mappings_to_car", - "resource": score_set.urn, - "correlation_id": correlation_id, - }) + job_manager.save_to_context( + { + "application": "mavedb-worker", + "function": "submit_score_set_mappings_to_car", + "resource": score_set.urn, + "correlation_id": correlation_id, + } + ) job_manager.update_progress(0, 100, "Starting CAR mapped resource submission.") logger.info(msg="Started CAR mapped resource submission", extra=job_manager.logging_context()) @@ -276,9 +280,6 @@ def _outcome_data() -> dict[str, int]: job_manager.update_progress(15, 100, "Submitting alleles to CAR.") car_service = ClinGenAlleleRegistryService(url=CAR_SUBMISSION_ENDPOINT) - hgvs_list = list(hgvs_to_allele_ids.keys()) - registered_alleles = car_service.dispatch_submissions(hgvs_list) - job_manager.update_progress(60, 100, "Processing registered alleles from CAR.") # Bulk-load every allele that could be linked in one query, keyed by id, rather than # issuing a SELECT per CAID inside the response loop. @@ -287,133 +288,153 @@ def _outcome_data() -> dict[str, int]: for allele in job_manager.db.scalars(select(AlleleModel).where(AlleleModel.id.in_(submitted_allele_ids))).all() } - # CAR's contract is one result per submitted HGVS, in order — that is what makes the - # positional zip below valid. A different count (including the empty list dispatch returns - # on request failure) means alignment cannot be trusted for ANY position, so we register - # nothing and fail the whole batch rather than risk writing a CAID to the wrong allele. - aligned = len(registered_alleles) == len(hgvs_list) - if not aligned: - logger.error( - msg=( - f"CAR returned {len(registered_alleles)} results for {len(hgvs_list)} submitted HGVS; " - "positional alignment cannot be trusted. Failing the entire batch." - ), - extra=job_manager.logging_context(), - ) - - for hgvs_string, response in zip(hgvs_list, registered_alleles if aligned else []): - allele_ids_for_hgvs = hgvs_to_allele_ids[hgvs_string] + # Reverse translation can push allele counts high enough that a single PUT is untenable, so + # dispatch in fixed-size chunks. Each batch is reconciled independently: a transient failure + # (or a broken one-result-per-input contract) fails only that batch's alleles, not the run. + batches = list(batched(list(hgvs_to_allele_ids.keys()), DEFAULT_CAR_SUBMISSION_BATCH_SIZE)) + job_manager.save_to_context( + { + "car_submission_batch_size": DEFAULT_CAR_SUBMISSION_BATCH_SIZE, + "car_submission_batch_count": len(batches), + } + ) - if "errorType" in response: - logger.warning( - msg=f"CAR rejected HGVS '{hgvs_string}' ({response.get('errorType', 'unknown')}): {response.get('message', 'unknown')}", + for batch_idx, hgvs_list in enumerate(batches): + # Spread the 15→60 window across batches so progress advances as each chunk is dispatched. + job_manager.update_progress( + 15 + (45 * batch_idx) // len(batches), + 100, + f"Submitting alleles to CAR (batch {batch_idx + 1} / {len(batches)}).", + ) + registered_alleles = car_service.dispatch_submissions(hgvs_list) + + # CAR's contract is one result per submitted HGVS, in order — that is what makes the + # positional zip below valid. A different count (including the empty list dispatch returns + # on request failure) means alignment cannot be trusted for ANY position, so we register + # nothing and fail this batch rather than risk writing a CAID to the wrong allele. + aligned = len(registered_alleles) == len(hgvs_list) + if not aligned: + logger.error( + msg=( + f"CAR returned {len(registered_alleles)} results for {len(hgvs_list)} submitted HGVS; " + "positional alignment cannot be trusted. Failing this batch." + ), extra=job_manager.logging_context(), ) - for allele_id in allele_ids_for_hgvs: - failed_allele_ids.add(allele_id) - _annotate_caid( - annotation_manager, - allele_id, - Disposition.FAILED, - EventReason.SERVICE_REJECTED, - error_message="Failed to register allele with ClinGen Allele Registry.", - metadata={ - "submitted_hgvs": hgvs_string, - "car_error_type": response.get("errorType"), - "car_error_message": response.get("message"), - }, - ) - continue + for hgvs_string, response in zip(hgvs_list, registered_alleles if aligned else []): + allele_ids_for_hgvs = hgvs_to_allele_ids[hgvs_string] - # A response that is neither an error nor a registration (no "@id") is malformed. - caid_iri = response.get("@id") - if not caid_iri: - logger.error( - msg=f"CAR returned a response for HGVS '{hgvs_string}' with neither an error nor an allele identifier.", - extra=job_manager.logging_context(), - ) - for allele_id in allele_ids_for_hgvs: - failed_allele_ids.add(allele_id) - _annotate_caid( - annotation_manager, - allele_id, - Disposition.FAILED, - EventReason.MALFORMED_RESPONSE, - error_message="ClinGen Allele Registry returned a malformed response with no allele identifier.", - metadata={"submitted_hgvs": hgvs_string}, + if "errorType" in response: + logger.warning( + msg=f"CAR rejected HGVS '{hgvs_string}' ({response.get('errorType', 'unknown')}): {response.get('message', 'unknown')}", + extra=job_manager.logging_context(), ) + for allele_id in allele_ids_for_hgvs: + failed_allele_ids.add(allele_id) + _annotate_caid( + annotation_manager, + allele_id, + Disposition.FAILED, + EventReason.SERVICE_REJECTED, + error_message="Failed to register allele with ClinGen Allele Registry.", + metadata={ + "submitted_hgvs": hgvs_string, + "car_error_type": response.get("errorType"), + "car_error_message": response.get("message"), + }, + ) - continue - - caid = caid_iri.split("/")[-1] - for allele_id in allele_ids_for_hgvs: - entry = allele_data[allele_id] - prior_caid = entry.existing_caid + continue - # TODO#780 - CAID is immutable — a different value returned by CAR is a hard invariant - # violation. Do not overwrite; record a failure with full audit context. - if prior_caid and prior_caid != caid: + # A response that is neither an error nor a registration (no "@id") is malformed. + caid_iri = response.get("@id") + if not caid_iri: logger.error( - msg=( - f"CAR returned a different CAID for allele {allele_id}: " - f"stored={prior_caid!r}, returned={caid!r}. " - "Not overwriting. Investigate immediately." - ), + msg=f"CAR returned a response for HGVS '{hgvs_string}' with neither an error nor an allele identifier.", extra=job_manager.logging_context(), ) + for allele_id in allele_ids_for_hgvs: + failed_allele_ids.add(allele_id) + _annotate_caid( + annotation_manager, + allele_id, + Disposition.FAILED, + EventReason.MALFORMED_RESPONSE, + error_message="ClinGen Allele Registry returned a malformed response with no allele identifier.", + metadata={"submitted_hgvs": hgvs_string}, + ) - failed_allele_ids.add(allele_id) - _annotate_caid( - annotation_manager, - allele_id, - Disposition.FAILED, - EventReason.CAID_CONFLICT, - error_message="CAR returned a CAID that conflicts with the stored value.", - metadata={ - "clingen_allele_id": prior_caid, - "conflicting_caid": caid, - "submitted_hgvs": hgvs_string, - }, - ) - - # CAID is new or matches the stored value — link it to the allele and record success. - else: - linked_allele_ids.add(allele_id) - allele = alleles_by_id[allele_id] - allele.clingen_allele_id = caid + continue - reason = EventReason.RECONFIRMED if prior_caid else EventReason.CREATED - if prior_caid: - logger.info( - msg=f"Force re-registration confirmed same CAID {caid!r} for allele {allele_id}.", + caid = caid_iri.split("/")[-1] + for allele_id in allele_ids_for_hgvs: + entry = allele_data[allele_id] + prior_caid = entry.existing_caid + + # TODO#780 - CAID is immutable — a different value returned by CAR is a hard invariant + # violation. Do not overwrite; record a failure with full audit context. + if prior_caid and prior_caid != caid: + logger.error( + msg=( + f"CAR returned a different CAID for allele {allele_id}: " + f"stored={prior_caid!r}, returned={caid!r}. " + "Not overwriting. Investigate immediately." + ), extra=job_manager.logging_context(), ) + failed_allele_ids.add(allele_id) + _annotate_caid( + annotation_manager, + allele_id, + Disposition.FAILED, + EventReason.CAID_CONFLICT, + error_message="CAR returned a CAID that conflicts with the stored value.", + metadata={ + "clingen_allele_id": prior_caid, + "conflicting_caid": caid, + "submitted_hgvs": hgvs_string, + }, + ) + + # CAID is new or matches the stored value — link it to the allele and record success. + else: + linked_allele_ids.add(allele_id) + allele = alleles_by_id[allele_id] + allele.clingen_allele_id = caid + + reason = EventReason.RECONFIRMED if prior_caid else EventReason.CREATED + if prior_caid: + logger.info( + msg=f"Force re-registration confirmed same CAID {caid!r} for allele {allele_id}.", + extra=job_manager.logging_context(), + ) + + _annotate_caid( + annotation_manager, + allele_id, + Disposition.PRESENT, + reason, + metadata={"clingen_allele_id": caid}, + ) + + # Submitted HGVS with no trustworthy response: the truncated tail when the counts line up + # (network drop, service-side omission), or the entire batch when the response count + # violated CAR's one-result-per-input contract and we rejected it above. + unattributed_hgvs = hgvs_list if not aligned else hgvs_list[len(registered_alleles) :] + for hgvs_string in unattributed_hgvs: + for allele_id in hgvs_to_allele_ids[hgvs_string]: + failed_allele_ids.add(allele_id) _annotate_caid( annotation_manager, allele_id, - Disposition.PRESENT, - reason, - metadata={"clingen_allele_id": caid}, + Disposition.FAILED, + EventReason.API_ERROR, + error_message="Failed to register allele with ClinGen Allele Registry.", + metadata={"submitted_hgvs": hgvs_string}, ) - # Submitted HGVS with no trustworthy response: the truncated tail when the counts line up - # (network drop, service-side omission), or the entire batch when the response count - # violated CAR's one-result-per-input contract and we rejected it above. - unattributed_hgvs = hgvs_list if not aligned else hgvs_list[len(registered_alleles) :] - for hgvs_string in unattributed_hgvs: - for allele_id in hgvs_to_allele_ids[hgvs_string]: - failed_allele_ids.add(allele_id) - _annotate_caid( - annotation_manager, - allele_id, - Disposition.FAILED, - EventReason.API_ERROR, - error_message="Failed to register allele with ClinGen Allele Registry.", - metadata={"submitted_hgvs": hgvs_string}, - ) - annotation_manager.flush() outcome_data = _outcome_data() @@ -477,12 +498,14 @@ async def submit_score_set_mappings_to_ldh(ctx: dict, job_id: int, job_manager: correlation_id = job.job_params["correlation_id"] # type: ignore # Setup initial context and progress - job_manager.save_to_context({ - "application": "mavedb-worker", - "function": "submit_score_set_mappings_to_ldh", - "resource": score_set.urn, - "correlation_id": correlation_id, - }) + job_manager.save_to_context( + { + "application": "mavedb-worker", + "function": "submit_score_set_mappings_to_ldh", + "resource": score_set.urn, + "correlation_id": correlation_id, + } + ) job_manager.update_progress(0, 100, "Starting LDH mapped resource submission.") logger.info(msg="Started LDH mapped resource submission", extra=job_manager.logging_context()) @@ -496,8 +519,7 @@ async def submit_score_set_mappings_to_ldh(ctx: dict, job_id: int, job_manager: # MaveDB score to its canonical mapped variant, not to every equivalent allele (unlike CAR, # which registers a CAID per allele). variant_objects: Sequence[tuple[Variant, MappingRecord, AlleleModel]] = ( - job_manager.db - .execute( + job_manager.db.execute( select(Variant, MappingRecord, AlleleModel) .join(MappingRecord, MappingRecord.variant_id == Variant.id) .join(MappingRecordAllele, MappingRecordAllele.mapping_record_id == MappingRecord.id) @@ -560,10 +582,12 @@ async def submit_score_set_mappings_to_ldh(ctx: dict, job_id: int, job_manager: loop = asyncio.get_running_loop() submission_successes, submission_failures = await loop.run_in_executor(ctx["pool"], blocking) job_manager.update_progress(90, 100, "Finalizing LDH mapped resource submission.") - job_manager.save_to_context({ - "ldh_submission_successes": len(submission_successes), - "ldh_submission_failures": len(submission_failures), - }) + job_manager.save_to_context( + { + "ldh_submission_successes": len(submission_successes), + "ldh_submission_failures": len(submission_failures), + } + ) # See https://reg.clinicalgenome.org/doc/AlleleRegistry_1.01.xx_api_v1.pdf annotation_manager = AnnotationStatusManager( diff --git a/tests/worker/jobs/external_services/test_clingen.py b/tests/worker/jobs/external_services/test_clingen.py index 3b44a35c5..5431227d6 100644 --- a/tests/worker/jobs/external_services/test_clingen.py +++ b/tests/worker/jobs/external_services/test_clingen.py @@ -917,6 +917,171 @@ async def test_submit_score_set_mappings_to_car_force_reregister_caid_conflict( assert event.event_metadata["clingen_allele_id"] == "CA_STORED" assert event.event_metadata["conflicting_caid"] == "CA_DIFFERENT" + async def _add_derived_alleles_with_distinct_hgvs(self, session, count): + """Attach ``count`` extra alleles to the score set, each carrying a distinct HGVS so the CAR + job produces ``count + 1`` distinct submission lines (one per HGVS). Returns every allele.""" + authoritative_allele = session.scalars(select(Allele)).one() + a_link = session.scalars( + select(MappingRecordAllele).where(MappingRecordAllele.allele_id == authoritative_allele.id) + ).first() + + alleles = [authoritative_allele] + for i in range(count): + post_mapped = deepcopy(authoritative_allele.post_mapped) + post_mapped["expressions"][0]["value"] = f"NC_000001.11:g.{1000 + i}A>T" + derived = Allele( + vrs_digest=f"{authoritative_allele.vrs_digest}-d{i}", + level=authoritative_allele.level, + post_mapped=post_mapped, + ) + session.add(derived) + session.flush() + session.add( + MappingRecordAllele( + mapping_record_id=a_link.mapping_record_id, + allele_id=derived.id, + is_authoritative=False, + ) + ) + alleles.append(derived) + + session.flush() + return alleles + + async def test_submit_score_set_mappings_to_car_batches_submissions( + self, + mock_worker_ctx, + session, + with_submit_score_set_mappings_to_car_job, + submit_score_set_mappings_to_car_sample_job_run, + mock_s3_client, + sample_score_dataframe, + sample_count_dataframe, + with_dummy_setup_jobs, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ): + """A large allele set is dispatched in fixed-size chunks rather than one PUT: 4 distinct HGVS + with a batch size of 2 yields two dispatch calls, and every allele is still registered.""" + await create_mappings_in_score_set( + session, + mock_s3_client, + mock_worker_ctx, + sample_score_dataframe, + sample_count_dataframe, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ) + alleles = await self._add_derived_alleles_with_distinct_hgvs(session, count=3) + + dispatched_batches = [] + + def fake_dispatch(hgvs_list): + dispatched_batches.append(list(hgvs_list)) + return [{"@id": f"CA/{hgvs}", "type": "nucleotide", "genomicAlleles": []} for hgvs in hgvs_list] + + with ( + patch( + "mavedb.worker.jobs.external_services.clingen.ClinGenAlleleRegistryService.dispatch_submissions", + side_effect=fake_dispatch, + ), + patch("mavedb.worker.jobs.external_services.clingen.DEFAULT_CAR_SUBMISSION_BATCH_SIZE", 2), + patch("mavedb.worker.jobs.external_services.clingen.CAR_SUBMISSION_ENDPOINT", "http://fake-endpoint"), + patch("mavedb.worker.jobs.external_services.clingen.CLIN_GEN_SUBMISSION_ENABLED", True), + ): + result = await submit_score_set_mappings_to_car( + mock_worker_ctx, + submit_score_set_mappings_to_car_sample_job_run.id, + JobManager(session, mock_worker_ctx["redis"], submit_score_set_mappings_to_car_sample_job_run.id), + ) + + assert result.status == JobStatus.SUCCEEDED + + # 4 distinct HGVS at batch size 2 → two dispatch calls, neither exceeding the batch size, and + # together covering every submitted line exactly once. + assert len(dispatched_batches) == 2 + assert all(len(batch) <= 2 for batch in dispatched_batches) + assert sum(len(batch) for batch in dispatched_batches) == 4 + + # Every allele across both batches is registered. + for allele in alleles: + session.refresh(allele) + assert allele.clingen_allele_id is not None + + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") + ).all() + assert len(events) == 4 + assert all(event.disposition == "present" for event in events) + + async def test_submit_score_set_mappings_to_car_batch_failure_isolated( + self, + mock_worker_ctx, + session, + with_submit_score_set_mappings_to_car_job, + submit_score_set_mappings_to_car_sample_job_run, + mock_s3_client, + sample_score_dataframe, + sample_count_dataframe, + with_dummy_setup_jobs, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ): + """A failed batch fails only its own alleles: the first chunk returns nothing (request failure) + while the second registers normally. Per-batch reconciliation keeps the two independent.""" + await create_mappings_in_score_set( + session, + mock_s3_client, + mock_worker_ctx, + sample_score_dataframe, + sample_count_dataframe, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ) + await self._add_derived_alleles_with_distinct_hgvs(session, count=3) + + call_count = {"n": 0} + + def fake_dispatch(hgvs_list): + call_count["n"] += 1 + # First batch mimics a request failure (dispatch_submissions returns [] on error); the rest + # register normally. + if call_count["n"] == 1: + return [] + return [{"@id": f"CA/{hgvs}", "type": "nucleotide", "genomicAlleles": []} for hgvs in hgvs_list] + + with ( + patch( + "mavedb.worker.jobs.external_services.clingen.ClinGenAlleleRegistryService.dispatch_submissions", + side_effect=fake_dispatch, + ), + patch("mavedb.worker.jobs.external_services.clingen.DEFAULT_CAR_SUBMISSION_BATCH_SIZE", 2), + patch("mavedb.worker.jobs.external_services.clingen.CAR_SUBMISSION_ENDPOINT", "http://fake-endpoint"), + patch("mavedb.worker.jobs.external_services.clingen.CLIN_GEN_SUBMISSION_ENABLED", True), + ): + result = await submit_score_set_mappings_to_car( + mock_worker_ctx, + submit_score_set_mappings_to_car_sample_job_run.id, + JobManager(session, mock_worker_ctx["redis"], submit_score_set_mappings_to_car_sample_job_run.id), + ) + + # The good batch links alleles, so the job overall succeeds despite the failed batch. + assert result.status == JobStatus.SUCCEEDED + assert call_count["n"] == 2 + + # Exactly the second batch's two alleles are registered; the first batch's two are not. + registered = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(registered) == 2 + + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") + ).all() + assert len(events) == 4 + assert sum(event.disposition == "present" for event in events) == 2 + failed = [event for event in events if event.disposition == "failed"] + assert len(failed) == 2 + assert all(event.reason == "api_error" for event in failed) + @pytest.mark.integration @pytest.mark.asyncio From 3b9f7da6c5d54aa320f934ab833e59e57e6a6d08 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 15 Jul 2026 11:22:33 -0700 Subject: [PATCH 079/252] feat(worker): stop linking protein-level alleles to ClinVar and gnomAD ClinVar and gnomAD are nucleotide-level resources keyed on genomic/coding changes. A protein allele's clinical calls and population frequencies are carried by its underlying g./c. siblings, so linking them at the protein level is a category error now that projection pairing exposes those siblings. Exclude protein-level alleles from both linking jobs. - Carry the allele's sequence level on ScoreSetAlleleRow so nucleotide-only jobs can filter without a second query - Add EventReason.PROTEIN_LEVEL_ALLELE and record protein alleles as a not-applicable structural gap (mirrors the multi-variant-CAID skip) - refresh_clinvar_controls: skip protein alleles first in the loop, before any ClinGen resolution - link_gnomad_variants: drop protein alleles from the Athena query set and skip them in the annotation loop, sparing the round-trip --- src/mavedb/lib/clingen/alleles.py | 10 ++- src/mavedb/models/enums/event_reason.py | 3 + .../worker/jobs/external_services/clinvar.py | 63 ++++++++++++------- .../worker/jobs/external_services/gnomad.py | 37 ++++++++--- .../jobs/external_services/test_clinvar.py | 44 +++++++++++++ .../jobs/external_services/test_gnomad.py | 41 ++++++++++++ 6 files changed, 165 insertions(+), 33 deletions(-) diff --git a/src/mavedb/lib/clingen/alleles.py b/src/mavedb/lib/clingen/alleles.py index 58cb6d7d0..562dab94e 100644 --- a/src/mavedb/lib/clingen/alleles.py +++ b/src/mavedb/lib/clingen/alleles.py @@ -24,8 +24,10 @@ class ScoreSetAlleleRow(NamedTuple): per allele. ``hgvs_g``/``hgvs_c``/``hgvs_p`` are allele-level (stable by construction), carried here so the - VEP job can build its HGVS payload without a second query. They are optional with a ``None`` - default so payloads keying only on the CAID (gnomAD/ClinVar) need not name them. + VEP job can build its HGVS payload without a second query. ``level`` is the allele's sequence + level (``protein``/``cdna``/``genomic``), carried so jobs annotating just nucleotide alleles can + skip protein alleles without a second query. All are optional with a ``None`` default so payloads keying + only on the CAID need not name them; real DB rows always populate them. """ allele_id: int @@ -35,6 +37,7 @@ class ScoreSetAlleleRow(NamedTuple): hgvs_g: str | None = None hgvs_c: str | None = None hgvs_p: str | None = None + level: str | None = None def get_alleles_for_score_set(db: Session, score_set_id: int) -> list[ScoreSetAlleleRow]: @@ -56,6 +59,7 @@ def get_alleles_for_score_set(db: Session, score_set_id: int) -> list[ScoreSetAl Allele.hgvs_g, Allele.hgvs_c, Allele.hgvs_p, + Allele.level, ) .join(MappingRecordAllele, MappingRecordAllele.allele_id == Allele.id) .join(MappingRecord, MappingRecord.id == MappingRecordAllele.mapping_record_id) @@ -67,7 +71,7 @@ def get_alleles_for_score_set(db: Session, score_set_id: int) -> list[ScoreSetAl ).all() return [ - ScoreSetAlleleRow(r.id, r.post_mapped, r.clingen_allele_id, r.variant_id, r.hgvs_g, r.hgvs_c, r.hgvs_p) + ScoreSetAlleleRow(r.id, r.post_mapped, r.clingen_allele_id, r.variant_id, r.hgvs_g, r.hgvs_c, r.hgvs_p, r.level) for r in rows ] diff --git a/src/mavedb/models/enums/event_reason.py b/src/mavedb/models/enums/event_reason.py index f7fd4b56b..35aa23af4 100644 --- a/src/mavedb/models/enums/event_reason.py +++ b/src/mavedb/models/enums/event_reason.py @@ -30,6 +30,9 @@ class EventReason(str, Enum): NO_CAID = "no_caid" # no ClinGen allele id to key on (gnomAD, ClinVar) NO_HGVS = "no_hgvs" # no HGVS to submit/resolve (CAR, VEP) MULTI_VARIANT_CAID = "multi_variant_caid" # cis-block CAID cannot be used (ClinVar) + PROTEIN_LEVEL_ALLELE = ( + "protein_level_allele" # protein allele excluded from nucleotide-level annotation (ClinVar, gnomAD) + ) NO_ASSAY_LEVEL_HGVS = "no_assay_level_hgvs" # no assay-level HGVS to translate (RT) # failed — errored diff --git a/src/mavedb/worker/jobs/external_services/clinvar.py b/src/mavedb/worker/jobs/external_services/clinvar.py index c3fcf2f2c..557919299 100644 --- a/src/mavedb/worker/jobs/external_services/clinvar.py +++ b/src/mavedb/worker/jobs/external_services/clinvar.py @@ -30,6 +30,7 @@ from mavedb.models.enums.disposition import Disposition from mavedb.models.enums.event_reason import EventReason from mavedb.models.enums.job_pipeline import FailureCategory +from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.models.score_set import ScoreSet from mavedb.worker.jobs.utils.setup import validate_job_params from mavedb.worker.lib.decorators.pipeline_management import with_pipeline_management @@ -120,34 +121,38 @@ async def refresh_clinvar_controls(ctx: dict, job_id: int, job_manager: JobManag versions = _generate_clinvar_versions() - job_manager.save_to_context({ - "application": "mavedb-worker", - "function": "refresh_clinvar_controls", - "resource": score_set.urn, - "correlation_id": correlation_id, - "versions": versions, - "total_versions": len(versions), - "force": force, - }) + job_manager.save_to_context( + { + "application": "mavedb-worker", + "function": "refresh_clinvar_controls", + "resource": score_set.urn, + "correlation_id": correlation_id, + "versions": versions, + "total_versions": len(versions), + "force": force, + } + ) job_manager.update_progress(0, 100, f"Starting ClinVar refresh across {len(versions)} versions.") logger.info(f"Starting ClinVar refresh across {len(versions)} versions", extra=job_manager.logging_context()) - # One work-unit per allele (payload = CAID; alleles without one are dropped). Covers ALL the score - # set's alleles — authoritative and RT-derived — since the genomic allele ClinVar keys on is often - # the RT-derived one. Events are allele-keyed, so every allele is recorded per release (no fan-out). - allele_data = group_alleles_for_annotation( - get_alleles_for_score_set(job_manager.db, score_set.id), - payload=lambda row: row.clingen_allele_id, - ) + # One work-unit per allele (payload = CAID; alleles without one are dropped). Covers the score + # set's alleles — authoritative and RT-derived — since the genomic allele ClinVar keys + # on is often the RT-derived one. Events are allele-keyed, so every allele is recorded per release + # (no fan-out). + allele_rows = get_alleles_for_score_set(job_manager.db, score_set.id) + allele_data = group_alleles_for_annotation(allele_rows, payload=lambda row: row.clingen_allele_id) + allele_levels = {row.allele_id: row.level for row in allele_rows} job_manager.save_to_context({"num_alleles_with_caids": len(allele_data)}) # Link counts accumulate across all versions (an allele may link in every release it appears in). - annotation_counts: Counter[str] = Counter({ - "created_link_count": 0, - "preexisting_link_count": 0, - "skipped_link_count": 0, - "failed_link_count": 0, - }) + annotation_counts: Counter[str] = Counter( + { + "created_link_count": 0, + "preexisting_link_count": 0, + "skipped_link_count": 0, + "failed_link_count": 0, + } + ) if not allele_data: logger.warning( @@ -204,6 +209,20 @@ def alleles_linked_at_version(clinvar_version: str) -> set[int]: job_manager.db, job_run_id=job_manager.job_id, score_set_id=score_set.id ) for allele_id, caid in allele_data.items(): + # ClinVar is a nucleotide-level database, skip any protein level alleles. + if allele_levels.get(allele_id) == SequenceLevel.protein.value: + annotation_counts["skipped_link_count"] += 1 + _annotate_clinvar( + annotation_manager, + allele_id, + Disposition.NOT_APPLICABLE, + EventReason.PROTEIN_LEVEL_ALLELE, + source_version=clinvar_version, + error_message="Protein-level alleles are not linked to ClinVar; see nucleotide siblings.", + metadata={"clingen_allele_id": caid}, + ) + continue + if allele_id in already_linked: annotation_counts["preexisting_link_count"] += 1 _annotate_clinvar( diff --git a/src/mavedb/worker/jobs/external_services/gnomad.py b/src/mavedb/worker/jobs/external_services/gnomad.py index 3b7a48d16..31e541568 100644 --- a/src/mavedb/worker/jobs/external_services/gnomad.py +++ b/src/mavedb/worker/jobs/external_services/gnomad.py @@ -24,6 +24,7 @@ from mavedb.models.enums.annotation_type import AnnotationType from mavedb.models.enums.disposition import Disposition from mavedb.models.enums.event_reason import EventReason +from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.models.gnomad_allele_link import GnomadAlleleLink from mavedb.models.gnomad_variant import GnomADVariant from mavedb.models.score_set import ScoreSet @@ -111,13 +112,12 @@ async def link_gnomad_variants(ctx: dict, job_id: int, job_manager: JobManager) job_manager.update_progress(0, 100, "Starting gnomAD mapped resource linkage.") logger.info(msg="Started gnomAD mapped resource linkage", extra=job_manager.logging_context()) - # One work-unit per allele (payload = CAID; alleles without one are skipped). Covers ALL the score - # set's alleles — authoritative and RT-derived — since the genomic allele gnomAD knows is often the - # RT-derived one. Events are allele-keyed, so every linked allele is recorded. - allele_data = group_alleles_for_annotation( - get_alleles_for_score_set(job_manager.db, score_set.id), - payload=lambda row: row.clingen_allele_id, - ) + # One work-unit per allele (payload = CAID; alleles without one are skipped). Covers the score + # set's alleles — authoritative and RT-derived — since the genomic allele gnomAD knows + # is often the RT-derived one. Events are allele-keyed, so every linked allele is recorded. + allele_rows = get_alleles_for_score_set(job_manager.db, score_set.id) + allele_data = group_alleles_for_annotation(allele_rows, payload=lambda row: row.clingen_allele_id) + allele_levels = {row.allele_id: row.level for row in allele_rows} annotation_counts: Counter[str] = Counter( { @@ -159,7 +159,15 @@ def alleles_linked_at_current_version(allele_ids: set[int]) -> set[int]: # Skip alleles already linked at the current version (they can't change). force re-fetches # all; the linker still supersedes only on change, so a forced no-op writes nothing. already_current = set() if force else alleles_linked_at_current_version(set(allele_data.keys())) - variant_caids = sorted({allele_data[aid] for aid in allele_data if aid not in already_current}) + # Never query Athena for protein alleles — gnomAD is nucleotide-level, so they can't match. They + # are still recorded as not-applicable in the loop below; this only spares them the round-trip. + variant_caids = sorted( + { + allele_data[aid] + for aid in allele_data + if aid not in already_current and allele_levels.get(aid) != SequenceLevel.protein.value + } + ) job_manager.save_to_context( { @@ -195,6 +203,19 @@ def alleles_linked_at_current_version(allele_ids: set[int]) -> set[int]: job_manager.db, job_run_id=job_manager.job_id, score_set_id=score_set.id ) for allele_id, caid in allele_data.items(): + # gnomAD is a nucleotide-level, genomic-coordinate resource. Skip any protein alleles. + if allele_levels.get(allele_id) == SequenceLevel.protein.value: + annotation_counts["skipped_allele_count"] += 1 + _annotate_gnomad( + annotation_manager, + allele_id, + Disposition.NOT_APPLICABLE, + EventReason.PROTEIN_LEVEL_ALLELE, + error_message="Protein-level alleles are not linked to gnomAD; see nucleotide siblings.", + metadata={"clingen_allele_id": caid}, + ) + continue + verdict = verdicts.get(allele_id) if verdict is GnomadLinkVerdict.CREATED: annotation_counts["created_allele_count"] += 1 diff --git a/tests/worker/jobs/external_services/test_clinvar.py b/tests/worker/jobs/external_services/test_clinvar.py index 47be57383..f7698d737 100644 --- a/tests/worker/jobs/external_services/test_clinvar.py +++ b/tests/worker/jobs/external_services/test_clinvar.py @@ -132,6 +132,50 @@ async def test_multi_variant_caid_skipped( assert event.reason == EventReason.MULTI_VARIANT_CAID assert event.allele_id == allele.id and event.variant_id is None + async def test_protein_level_allele_skipped( + self, + mock_worker_ctx, + session, + with_refresh_clinvar_controls_job, + sample_refresh_clinvar_controls_job_run, + setup_sample_alleles_with_caid, + ): + """A protein-level allele is never linked to ClinVar (a nucleotide-level DB): its clinical + calls come from its g./c. siblings. Skipped as not-applicable before any resolution runs.""" + _, allele = setup_sample_alleles_with_caid + allele.level = "protein" + session.commit() + + with ( + patch( + "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", + return_value="VCV000000123", + ) as resolve_spy, + patch( + "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", + return_value=MOCK_CLINVAR_DATA, + ), + ): + result = await refresh_clinvar_controls( + mock_worker_ctx, + sample_refresh_clinvar_controls_job_run.id, + JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), + ) + + assert result.status == JobStatus.SUCCEEDED + assert session.scalars(select(ClinvarAlleleLink)).all() == [] + assert session.scalars(select(ClinvarControl)).all() == [] + # Short-circuited before touching ClinGen — a protein allele can never resolve to a ClinVar id. + resolve_spy.assert_not_awaited() + + # Allele-keyed not-applicable event: ClinVar is nucleotide-level, so a protein allele is a + # structural gap (like a multi-variant CAID), not a statement about the source. + event = session.scalars(select(AnnotationEvent)).one() + assert event.annotation_type == AnnotationType.CLINVAR_CONTROL + assert event.disposition == Disposition.NOT_APPLICABLE + assert event.reason == EventReason.PROTEIN_LEVEL_ALLELE + assert event.allele_id == allele.id and event.variant_id is None + async def test_no_associated_clinvar_allele_id_skipped( self, mock_worker_ctx, diff --git a/tests/worker/jobs/external_services/test_gnomad.py b/tests/worker/jobs/external_services/test_gnomad.py index 0b14fda21..03a3cefac 100644 --- a/tests/worker/jobs/external_services/test_gnomad.py +++ b/tests/worker/jobs/external_services/test_gnomad.py @@ -11,6 +11,9 @@ from mavedb.lib import gnomad as gnomad_lib from mavedb.lib.gnomad import GNOMAD_DATA_VERSION from mavedb.lib.types.workflow import JobExecutionOutcome +from mavedb.models.enums.annotation_type import AnnotationType +from mavedb.models.enums.disposition import Disposition +from mavedb.models.enums.event_reason import EventReason from mavedb.models.enums.job_pipeline import JobStatus, PipelineStatus from mavedb.models.gnomad_allele_link import GnomadAlleleLink from mavedb.models.gnomad_variant import GnomADVariant @@ -72,6 +75,44 @@ async def test_link_gnomad_variants_no_gnomad_matches( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED + async def test_link_gnomad_variants_protein_level_allele_skipped( + self, + session, + with_populated_domain_data, + with_gnomad_linking_job, + mock_worker_ctx, + sample_link_gnomad_variants_run, + setup_sample_alleles_with_caid, + ): + """A protein-level allele is never linked to gnomAD (a nucleotide-level, genomic-coordinate + resource): its frequencies come from its g./c. siblings. Skipped as not-applicable, and it + never reaches the Athena query — sparing the round-trip.""" + _, allele = setup_sample_alleles_with_caid + allele.level = "protein" + session.commit() + + with patch( + "mavedb.worker.jobs.external_services.gnomad.gnomad_variant_data_for_caids", + ) as fetch_spy: + result = await link_gnomad_variants( + mock_worker_ctx, + 1, + JobManager(session, mock_worker_ctx["redis"], sample_link_gnomad_variants_run.id), + ) + + assert result.status == JobStatus.SUCCEEDED + assert session.scalars(select(GnomadAlleleLink)).all() == [] + # The protein allele's CAID was dropped from the query set, so Athena is never queried. + fetch_spy.assert_not_called() + + # Allele-keyed not-applicable event: gnomAD is nucleotide-level, so a protein allele is a + # structural gap, not a statement about the source. + event = session.scalars(select(AnnotationEvent)).one() + assert event.annotation_type == AnnotationType.GNOMAD_ALLELE_FREQUENCY + assert event.disposition == Disposition.NOT_APPLICABLE + assert event.reason == EventReason.PROTEIN_LEVEL_ALLELE + assert event.allele_id == allele.id and event.variant_id is None + async def test_link_gnomad_variants_call_linking_method( self, session, From 4377e72d99070e3b5639622bc10a7aee9ad3311a Mon Sep 17 00:00:00 2001 From: David Reinhart Date: Tue, 7 Jul 2026 11:17:34 -0700 Subject: [PATCH 080/252] feat(uniprot): support Ensembl protein accessions in UniProt ID mapping infer_db_name_from_sequence_accession now recognizes ENSP accessions and maps them to the Ensembl_Protein source, so accession-based target genes with Ensembl protein IDs resolve to a UniProt ID instead of raising NotImplementedError. Also make validate_ensembl_identifier accept versioned Ensembl stable IDs (e.g. ENSP00000369497.3) by stripping the trailing . before the idutils.is_ensembl check, matching how validate_refseq_identifier already handles versioned RefSeq accessions. Related to varianteffect/dcd_mapping2#110 --- src/mavedb/lib/uniprot/utils.py | 14 +++++++++++--- src/mavedb/lib/validation/identifier.py | 8 +++++++- tests/lib/uniprot/test_utils.py | 8 ++++++++ tests/validation/test_identifier.py | 4 ++++ 4 files changed, 30 insertions(+), 4 deletions(-) diff --git a/src/mavedb/lib/uniprot/utils.py b/src/mavedb/lib/uniprot/utils.py index 7f87524b2..11c13a8f4 100644 --- a/src/mavedb/lib/uniprot/utils.py +++ b/src/mavedb/lib/uniprot/utils.py @@ -1,11 +1,11 @@ from typing import Literal, Union -from mavedb.lib.validation.identifier import validate_refseq_identifier +from mavedb.lib.validation.identifier import validate_ensembl_identifier, validate_refseq_identifier def infer_db_name_from_sequence_accession( sequence_accession: str, -) -> Union[Literal["RefSeq_Nucleotide", "RefSeq_Protein"]]: +) -> Union[Literal["RefSeq_Nucleotide", "RefSeq_Protein", "Ensembl_Protein"]]: """ Infers the database name from a sequence accession. @@ -15,6 +15,12 @@ def infer_db_name_from_sequence_accession( Returns: str: The inferred database name. """ + # Ensembl protein accessions must be handled before validate_refseq_identifier, + # which raises on any non-RefSeq accession. + if sequence_accession.startswith("ENSP"): + validate_ensembl_identifier(sequence_accession) + return "Ensembl_Protein" + validate_refseq_identifier(sequence_accession) if sequence_accession.startswith("NM_"): @@ -22,4 +28,6 @@ def infer_db_name_from_sequence_accession( if sequence_accession.startswith("NP_"): return "RefSeq_Protein" - raise NotImplementedError("Only RefSeq NM and NP identifiers are currently supported for inference.") + raise NotImplementedError( + "Only RefSeq NM/NP and Ensembl protein (ENSP) identifiers are currently supported for inference." + ) diff --git a/src/mavedb/lib/validation/identifier.py b/src/mavedb/lib/validation/identifier.py index e6e2dcde7..077b13fc0 100644 --- a/src/mavedb/lib/validation/identifier.py +++ b/src/mavedb/lib/validation/identifier.py @@ -1,3 +1,5 @@ +import re + import idutils from mavedb.lib.validation.constants.identifier import valid_dbnames @@ -52,6 +54,9 @@ def validate_ensembl_identifier(identifier: str): """ Validates whether the identifier is a valid Ensembl identifier. + Ensembl stable IDs may carry a version suffix (e.g. ``ENSP00000369497.3``), which + ``idutils.is_ensembl`` does not accept; strip it before validating the stable ID. + Parameters __________ identifier : str @@ -62,7 +67,8 @@ def validate_ensembl_identifier(identifier: str): ValidationError If the identifier is not a valid Ensembl identifier. """ - if not idutils.is_ensembl(identifier): + base_identifier = re.sub(r"\.\d+$", "", identifier) + if not idutils.is_ensembl(base_identifier): raise ValidationError(f"'{identifier}' is not a valid Ensembl accession.") diff --git a/tests/lib/uniprot/test_utils.py b/tests/lib/uniprot/test_utils.py index 7afa278c2..5ac773e38 100644 --- a/tests/lib/uniprot/test_utils.py +++ b/tests/lib/uniprot/test_utils.py @@ -21,6 +21,14 @@ def test_infer_db_name_from_sequence_accession_np(): assert result == "RefSeq_Protein" +# Both versioned and unversioned ENSP accessions must route to Ensembl_Protein; the versioned case +# guards that validate_ensembl_identifier's version handling continues to accept it. +@pytest.mark.parametrize("ensembl_protein_accession", ["ENSP00000418960", "ENSP00000418960.3"]) +def test_infer_db_name_from_sequence_accession_ensp(ensembl_protein_accession): + result = infer_db_name_from_sequence_accession(ensembl_protein_accession) + assert result == "Ensembl_Protein" + + @pytest.mark.parametrize("invalid_accession", ["XP_000000", VALID_CHR_ACCESSION]) def test_infer_db_name_from_sequence_accession_invalid(invalid_accession): with pytest.raises(NotImplementedError): diff --git a/tests/validation/test_identifier.py b/tests/validation/test_identifier.py index 005278307..9cbd16b0d 100644 --- a/tests/validation/test_identifier.py +++ b/tests/validation/test_identifier.py @@ -63,6 +63,10 @@ def test_ve_invalid_list(self): def test_passes_valid_id(self): validate_ensembl_identifier("ENSG00000143384") + def test_passes_valid_versioned_id(self): + validate_ensembl_identifier("ENSP00000369497.3") + validate_ensembl_list(["ENSG00000139618.15", "ENST00000380152.8"]) + class TestRefSeqValidators(TestCase): """ From f1655ee5868b584ed446fc66cf05b267cb5deaa7 Mon Sep 17 00:00:00 2001 From: David Reinhart Date: Wed, 8 Jul 2026 11:49:19 -0700 Subject: [PATCH 081/252] Add Ensembl transcript (ENST) support to sequence accession inference Extend infer_db_name_from_sequence_accession to recognize ENST identifiers and return "Ensembl_Transcript" as the database name. Restructure control flow to validate each accession type only within its matching branch rather than calling validate_refseq_identifier unconditionally for non-Ensembl accessions. --- src/mavedb/lib/uniprot/utils.py | 18 +++++++++--------- tests/lib/uniprot/test_utils.py | 18 +++++++++++------- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/src/mavedb/lib/uniprot/utils.py b/src/mavedb/lib/uniprot/utils.py index 11c13a8f4..ec5a8a185 100644 --- a/src/mavedb/lib/uniprot/utils.py +++ b/src/mavedb/lib/uniprot/utils.py @@ -5,7 +5,7 @@ def infer_db_name_from_sequence_accession( sequence_accession: str, -) -> Union[Literal["RefSeq_Nucleotide", "RefSeq_Protein", "Ensembl_Protein"]]: +) -> Union[Literal["RefSeq_Nucleotide", "RefSeq_Protein", "Ensembl_Protein", "Ensembl_Transcript"]]: """ Infers the database name from a sequence accession. @@ -15,19 +15,19 @@ def infer_db_name_from_sequence_accession( Returns: str: The inferred database name. """ - # Ensembl protein accessions must be handled before validate_refseq_identifier, - # which raises on any non-RefSeq accession. if sequence_accession.startswith("ENSP"): validate_ensembl_identifier(sequence_accession) return "Ensembl_Protein" - - validate_refseq_identifier(sequence_accession) - - if sequence_accession.startswith("NM_"): + elif sequence_accession.startswith("ENST"): + validate_ensembl_identifier(sequence_accession) + return "Ensembl_Transcript" + elif sequence_accession.startswith("NM_"): + validate_refseq_identifier(sequence_accession) return "RefSeq_Nucleotide" - if sequence_accession.startswith("NP_"): + elif sequence_accession.startswith("NP_"): + validate_refseq_identifier(sequence_accession) return "RefSeq_Protein" raise NotImplementedError( - "Only RefSeq NM/NP and Ensembl protein (ENSP) identifiers are currently supported for inference." + "Only RefSeq (NM_/NP_) and Ensembl (ENSP/ENST) identifiers are currently supported for inference." ) diff --git a/tests/lib/uniprot/test_utils.py b/tests/lib/uniprot/test_utils.py index 5ac773e38..5d51c2afe 100644 --- a/tests/lib/uniprot/test_utils.py +++ b/tests/lib/uniprot/test_utils.py @@ -1,12 +1,10 @@ import pytest -from mavedb.lib.validation.exceptions import ValidationError from mavedb.lib.uniprot.utils import infer_db_name_from_sequence_accession - from tests.helpers.constants import ( + VALID_CHR_ACCESSION, VALID_NT_ACCESSION, VALID_PRO_ACCESSION, - VALID_CHR_ACCESSION, VALID_UNIPROT_ACCESSION, ) @@ -21,14 +19,20 @@ def test_infer_db_name_from_sequence_accession_np(): assert result == "RefSeq_Protein" -# Both versioned and unversioned ENSP accessions must route to Ensembl_Protein; the versioned case -# guards that validate_ensembl_identifier's version handling continues to accept it. +# Both versioned and unversioned ENSP and ENST accessions must route to Ensembl_Protein and Ensembl_Transcript, respectively; +# the versioned case guards that validate_ensembl_identifier's version handling continues to accept it. @pytest.mark.parametrize("ensembl_protein_accession", ["ENSP00000418960", "ENSP00000418960.3"]) def test_infer_db_name_from_sequence_accession_ensp(ensembl_protein_accession): result = infer_db_name_from_sequence_accession(ensembl_protein_accession) assert result == "Ensembl_Protein" +@pytest.mark.parametrize("ensembl_transcript_accession", ["ENST00000418960", "ENST00000418960.3"]) +def test_infer_db_name_from_sequence_accession_enst(ensembl_transcript_accession): + result = infer_db_name_from_sequence_accession(ensembl_transcript_accession) + assert result == "Ensembl_Transcript" + + @pytest.mark.parametrize("invalid_accession", ["XP_000000", VALID_CHR_ACCESSION]) def test_infer_db_name_from_sequence_accession_invalid(invalid_accession): with pytest.raises(NotImplementedError): @@ -36,10 +40,10 @@ def test_infer_db_name_from_sequence_accession_invalid(invalid_accession): def test_infer_db_name_from_non_refseq_accession(): - with pytest.raises(ValidationError): + with pytest.raises(NotImplementedError): infer_db_name_from_sequence_accession(VALID_UNIPROT_ACCESSION) def test_infer_db_name_from_sequence_accession_empty_string(): - with pytest.raises(ValidationError): + with pytest.raises(NotImplementedError): infer_db_name_from_sequence_accession("") From b1adc4fcb9173ab3c4b11b4fc3c9544b9e4c19ae Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Thu, 9 Jul 2026 10:35:24 -0700 Subject: [PATCH 082/252] chore(export): add changelog to public data archive and document release process MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add CHANGELOG.md to scripts/resources, bundled into every generated archive so each downloaded version carries its own history; documents versions 1–4 (pre-changelog) and v5 in detail - Update export script to include CHANGELOG.md alongside the README and license in the zip output - Add a release-procedure note to the module docstring pointing maintainers to the changelog and deployment docs before each Zenodo publish --- src/mavedb/scripts/export_public_data.py | 7 ++- src/mavedb/scripts/resources/CHANGELOG.md | 62 +++++++++++++++++++++++ src/mavedb/scripts/resources/README.md | 1 + 3 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 src/mavedb/scripts/resources/CHANGELOG.md diff --git a/src/mavedb/scripts/export_public_data.py b/src/mavedb/scripts/export_public_data.py index 4ced338a8..cedc66608 100644 --- a/src/mavedb/scripts/export_public_data.py +++ b/src/mavedb/scripts/export_public_data.py @@ -11,6 +11,10 @@ Unpublished data and data sets licensed other than under the Creative Commons Zero license are not included in the dump, and user details are limited to ORCID IDs and names of contributors to published data sets. + +RELEASING A NEW VERSION: Before publishing a new version of this archive to Zenodo, add an entry describing what +changed to `src/mavedb/scripts/resources/CHANGELOG.md` (it is bundled into the archive). The full release procedure +is documented in `deployment/docs/zenodo-release.md`. """ import json @@ -129,10 +133,11 @@ def export_public_data(db: Session): # Write metadata for all data sets to a single JSON file. zipfile.writestr("main.json", json.dumps(jsonable_encoder(json_data))) - # Copy the CC0 license and README. + # Copy the CC0 license, README, and changelog. resources_dir = os.path.join(os.path.dirname(__file__), "resources") zipfile.write(os.path.join(resources_dir, "CC0_license.txt"), "LICENSE.txt") zipfile.write(os.path.join(resources_dir, "README.md"), "README.md") + zipfile.write(os.path.join(resources_dir, "CHANGELOG.md"), "CHANGELOG.md") # Write score and count files for each score set. num_score_sets = len(score_set_ids) diff --git a/src/mavedb/scripts/resources/CHANGELOG.md b/src/mavedb/scripts/resources/CHANGELOG.md new file mode 100644 index 000000000..3f441fc0c --- /dev/null +++ b/src/mavedb/scripts/resources/CHANGELOG.md @@ -0,0 +1,62 @@ +# MaveDB Public Data Dump — Changelog + +This file records what changed between versions of the MaveDB bulk data archive +published on [Zenodo](https://doi.org/10.5281/zenodo.11201736). It ships inside +every archive so the version you downloaded always carries its own history. + +Versions are the Zenodo archive versions (v1, v2, …), not MaveDB API or website +release numbers. The `asOf` timestamp in `main.json` records the exact moment a +given archive was generated. + +--- + +## [5] — 2026-06-25 + +Version 5 is the first version tracked in this changelog. Versions 1–4 contained +metadata (`main.json`) plus per-score-set score and count CSVs; they predate this +file and are not itemized here. Version 5 adds variant-mapping and annotation +outputs derived from the MaveDB variant mapping pipeline. + +### Added + +- **Annotation CSVs** — `csv/{urn}.annotations.csv` for every score set that has + completed the mapping pipeline. Joins external-database annotations (Ensembl VEP + functional consequence, gnomAD v4.1 allele frequency, ClinGen Allele Registry ID) + with post-mapped HGVS (`g.`/`c.`/`p.` and assay-level) and the GA4GH VRS digest. +- **Mapped-variant JSON** — `mapped/{urn}.mapped-variants.json` for every mapped + score set. Each record carries the pre-mapped and post-mapped VRS alleles, the + VRS schema version, the mapping API version, and the ClinGen allele ID, mirroring + `GET /api/v1/score-sets/{urn}/mapped-variants`. +- **VA-Spec annotation NDJSON** — `va/{urn}.va.ndjson` for every mapped score set. + One line per current mapped variant carrying its highest materialized GA4GH + VA-Spec layer (study result → functional-impact statement → pathogenicity + statement), mirroring the `annotated-variants` streaming endpoints. +- **ClinVar release columns** in the annotation CSVs, covering the `2015_02` + through `2026_01` annual snapshots. + +### Changed + +- **CSV columns are now namespaced.** Every column carries a source prefix + separated by a dot: score columns are `scores.*` (e.g. `scores.score`), counts + are `counts.*`, mapping-pipeline columns are `mavedb.*`, and external annotations + are `vep.*` / `gnomad.*` / `clingen.*`. Core identifier columns (`accession`, + `hgvs_nt`, `hgvs_pro`, `hgvs_splice`) remain unprefixed. **Consumers that parsed + earlier archives by bare column name (e.g. `score`) must update to the namespaced + names.** +- **Cis-phased (haplotype) variants** are now combined into a single HGVS + expression in the CSV outputs where a combined expression can be derived. +- Mapped-variant and VA-Spec outputs include only the **current** mapping for each + variant; superseded mappings are excluded. + +### Fixed + +- VA-Spec annotation output is now valid and round-trippable (it deserializes back + into GA4GH VA-Spec objects without loss). + +### Notes + +- Annotation, mapped-variant, and VA-Spec files are present **only** for score sets + that have completed the mapping pipeline. Unmapped score sets ship scores/counts + CSVs only, as before. +- See `README.md` in this archive for the full file layout, column definitions, and + caveats. diff --git a/src/mavedb/scripts/resources/README.md b/src/mavedb/scripts/resources/README.md index 31ec4e26c..44b8eadf2 100644 --- a/src/mavedb/scripts/resources/README.md +++ b/src/mavedb/scripts/resources/README.md @@ -30,6 +30,7 @@ Unpublished data, private datasets, and datasets published under other licenses ``` mavedb-dump.YYYYMMDDHHMMSS.zip ├── README.md # This file +├── CHANGELOG.md # What changed between archive versions ├── LICENSE.txt # Creative Commons CC0 1.0 license text ├── main.json # Metadata for all included datasets ├── csv/ From 947b7449c8d62d06e114b6bcddd4516fcdb32181 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Thu, 9 Jul 2026 11:23:16 -0700 Subject: [PATCH 083/252] feat(worker): detect stuck RUNNING jobs via progress heartbeat Replace the pure wall-clock RUNNING-stall check with a progress heartbeat, so a healthy long-running job (e.g. large VEP fan-out) is no longer killed while a genuinely wedged one is still caught. Make the DB-driven sweeper authoritative over termination and keep ARQ's timeout only as a last resort. - Add JobRun.progress_updated_at (onupdate=now()) as a zero-code liveness heartbeat, bumped on every committing progress checkpoint; index (status, progress_updated_at) for the sweeper query - Detect stalled RUNNING jobs on stale heartbeat (PROGRESS_STALL_MINUTES = 20) OR a wall-clock backstop (RUNNING_TIMEOUT_MINUTES = 450), and recover them abort-first: abort the current attempt and only re-drive once it is confirmed dead or absent, eliminating the double-run where a retry ran alongside the still-live original - Handle asyncio.CancelledError in with_job_management: mark the row FAILED/TIMEOUT then re-raise, so an ARQ timeout or a stall-recovery abort no longer orphans the row as RUNNING and Job.abort() can confirm - Raise ARQ job_timeout to 8h and enable allow_abort_jobs; tighten the cleanup cron to every 10 minutes - Cover new behavior with unit and integration tests --- ...104_add_progress_updated_at_to_job_runs.py | 44 +++++ src/mavedb/models/job_run.py | 10 ++ src/mavedb/worker/jobs/registry.py | 6 +- src/mavedb/worker/jobs/system/cleanup.py | 140 +++++++++++++-- .../worker/lib/decorators/job_management.py | 40 ++++- src/mavedb/worker/settings/worker.py | 12 +- tests/worker/jobs/system/test_cleanup.py | 167 +++++++++++++++++- .../lib/decorators/test_job_management.py | 63 ++++++- tests/worker/lib/managers/test_job_manager.py | 23 +++ 9 files changed, 472 insertions(+), 33 deletions(-) create mode 100644 alembic/versions/a7f3c2e9b104_add_progress_updated_at_to_job_runs.py diff --git a/alembic/versions/a7f3c2e9b104_add_progress_updated_at_to_job_runs.py b/alembic/versions/a7f3c2e9b104_add_progress_updated_at_to_job_runs.py new file mode 100644 index 000000000..d04ceab9b --- /dev/null +++ b/alembic/versions/a7f3c2e9b104_add_progress_updated_at_to_job_runs.py @@ -0,0 +1,44 @@ +"""Add progress_updated_at heartbeat column to job_runs + +Revision ID: a7f3c2e9b104 +Revises: 4dbf24ed1857 +Create Date: 2026-07-09 00:00:00.000000 + +""" + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "a7f3c2e9b104" +down_revision = "4dbf24ed1857" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column( + "job_runs", + sa.Column( + "progress_updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + ) + op.create_index( + "ix_job_runs_status_progress_updated", + "job_runs", + ["status", "progress_updated_at"], + unique=False, + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index("ix_job_runs_status_progress_updated", table_name="job_runs") + op.drop_column("job_runs", "progress_updated_at") + # ### end Alembic commands ### diff --git a/src/mavedb/models/job_run.py b/src/mavedb/models/job_run.py index 877eeab02..b079a1757 100644 --- a/src/mavedb/models/job_run.py +++ b/src/mavedb/models/job_run.py @@ -70,6 +70,14 @@ class JobRun(Base): progress_total: Mapped[Optional[int]] = mapped_column(Integer, nullable=True) progress_message: Mapped[Optional[str]] = mapped_column(String(500), nullable=True) + # Liveness heartbeat. `onupdate=func.now()` bumps this on *any* commit that dirties the row, + # so every progress checkpoint (and every lifecycle transition) refreshes it with zero per-job + # code. The stalled-job sweeper treats a RUNNING job whose heartbeat has gone stale as wedged, + # which distinguishes a healthy long-running job (still committing progress) from a dead one. + progress_updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now(), onupdate=func.now() + ) + # Correlation for tracing correlation_id: Mapped[Optional[str]] = mapped_column(String(255), nullable=True) @@ -98,6 +106,8 @@ class JobRun(Base): Index("ix_job_runs_created_at", "created_at"), Index("ix_job_runs_correlation_id", "correlation_id"), Index("ix_job_runs_status_scheduled", "status", "scheduled_at"), + # Supports the stalled-job sweeper's RUNNING heartbeat query (status + heartbeat staleness). + Index("ix_job_runs_status_progress_updated", "status", "progress_updated_at"), CheckConstraint( "status IN ('pending', 'queued', 'running', 'succeeded', 'failed', 'errored', 'cancelled', 'skipped')", name="ck_job_runs_status_valid", diff --git a/src/mavedb/worker/jobs/registry.py b/src/mavedb/worker/jobs/registry.py index 94cd76e32..ba9754a05 100644 --- a/src/mavedb/worker/jobs/registry.py +++ b/src/mavedb/worker/jobs/registry.py @@ -71,8 +71,10 @@ cron( cleanup_stalled_jobs, name="cleanup_stalled_jobs_cron", - minute={15, 45}, # Run at :15 and :45 past each hour (every 30 minutes) - keep_result=timedelta(minutes=25).total_seconds(), + # Every 10 minutes. Paired with the 20-min progress-stall window this gives an effective + # stall-detection latency of ~20-30 min (vs. up to 50 min on the old 30-min cadence). + minute={5, 15, 25, 35, 45, 55}, + keep_result=timedelta(minutes=9).total_seconds(), ), ] diff --git a/src/mavedb/worker/jobs/system/cleanup.py b/src/mavedb/worker/jobs/system/cleanup.py index 89089828c..4a2dc975e 100644 --- a/src/mavedb/worker/jobs/system/cleanup.py +++ b/src/mavedb/worker/jobs/system/cleanup.py @@ -13,6 +13,7 @@ The cleanup job acts as a safety net to ensure jobs don't remain in limbo forever. """ +import asyncio import logging from datetime import datetime, timedelta, timezone @@ -37,14 +38,30 @@ logger = logging.getLogger(__name__) # Timeout thresholds for detecting stalled jobs (in minutes). -# RUNNING_TIMEOUT_MINUTES must stay below ArqWorkerSettings.job_timeout (currently 2 hours) -# to avoid marking legitimately running jobs as stalled. -RUNNING_TIMEOUT_MINUTES = 150 # RUNNING jobs should complete within 150 min (30 min buffer under ARQ timeout) +# +# RUNNING jobs are evaluated on two signals: +# 1. PROGRESS_STALL_MINUTES — the primary signal. A healthy job checkpoints progress frequently +# (JobRun.progress_updated_at is bumped on every committing progress update via onupdate=now()). +# If the heartbeat has not advanced in this window the job is considered wedged, regardless of +# how long it has legitimately been running. This is what lets a multi-hour VEP fan-out run to +# completion without being killed, while still catching a job that has genuinely hung. +# 2. RUNNING_TIMEOUT_MINUTES — a wall-clock backstop for jobs that never emit progress (or whose +# heartbeat column is somehow stuck). Sits ~30 min under ArqWorkerSettings.job_timeout (8h) so +# the DB-driven sweeper, not ARQ's hard kill, is what terminates a job — keeping DB state and +# actual work in sync. +PROGRESS_STALL_MINUTES = 20 # RUNNING jobs should checkpoint progress at least this often +RUNNING_TIMEOUT_MINUTES = 450 # Wall-clock backstop (7.5h), 30 min under the 8h ARQ ceiling PENDING_TIMEOUT_MINUTES = 5 # PENDING jobs which are actionable within pipelines should be enqueued within 5 minutes PIPELINE_STUCK_TIMEOUT_MINUTES = ( 5 # Pipelines in non-terminal states with no active jobs should resolve within 5 minutes ) +# How long the sweeper waits for an aborted RUNNING job to actually die before giving up. A job +# cancelled at an await point dies within a poll cycle (sub-second); a job wedged in a blocking sync +# call never honors the abort and will hit this timeout — in which case we do NOT re-drive it (see +# _handle_stalled_running_job) to avoid running it twice. +ABORT_CONFIRM_TIMEOUT_SECONDS = 15 + async def _handle_stalled_job_retry( job: JobRun, @@ -193,19 +210,90 @@ async def _handle_stalled_job_retry( return False +async def _running_job_confirmed_dead(arq_id: str, redis: ArqRedis) -> bool: + """Abort a RUNNING job's current ARQ attempt and report whether it is safe to re-drive. + + RUNNING recovery differs from QUEUED/PENDING recovery: a RUNNING job may still have a live + coroutine holding a worker slot. Re-enqueuing without cancelling it first may mean two + coroutines are running the same job simultaneously, which is unsafe. + + We therefore issue Job.abort() on the current attempt and interpret the outcome: + + - abort() returns True -> the coroutine was alive and has now been cancelled + recorded a + terminal result. Safe to re-drive. + - abort() returns False -> the job is absent from ARQ (crashed worker / never enqueued / already + finished). Nothing is running. Safe to re-drive — this is the common crash-recovery path. + - abort() raises TimeoutError -> the job is still present in ARQ and did not die within the + window: a coroutine wedged in a blocking sync call that cannot honor cooperative cancellation. + NOT safe to re-drive; leave it for the next sweep or ARQ's own ceiling. + - any other error -> treat as not-confirmably-alive and allow re-drive; a genuinely live job + would have either confirmed the abort or timed out, not errored here. + + Returns True when it is safe to re-drive the job, False when the job may still be running. + """ + try: + await ArqJob(arq_id, redis).abort(timeout=ABORT_CONFIRM_TIMEOUT_SECONDS) + return True + except asyncio.TimeoutError: + logger.warning( + f"Abort of RUNNING job attempt {arq_id} did not confirm within " + f"{ABORT_CONFIRM_TIMEOUT_SECONDS}s; job may be wedged in a blocking call — not re-driving" + ) + return False + except Exception as e: + logger.warning(f"Abort of RUNNING job attempt {arq_id} raised {e!r}; treating as not running and re-driving") + return True + + +async def _handle_stalled_running_job( + job: JobRun, + manager: JobManager, + redis: ArqRedis, + stall_reason: str, + db: Session, +) -> bool: + """Recover a stalled RUNNING job via abort-first ordering. + + Aborts the current attempt (urn#N) BEFORE any retry bookkeeping, and only proceeds to re-drive + (which increments retry_count and enqueues urn#N+1) once the abort confirms the original is dead + or absent. If the original refuses to die, we leave it alone rather than risk a double-run. + + Returns True if the sweeper acted on the job (re-drove or permanently failed it), False if it was + left in place because the abort could not be confirmed (so it should not be counted as cleaned). + """ + arq_id = arq_job_id(job) + + # Could not confirm the coroutine is dead. Leave the job RUNNING; the next sweep will retry + # the abort, and ARQ's job_timeout ceiling remains as a final backstop. + if not await _running_job_confirmed_dead(arq_id, redis): + logger.warning( + f"Leaving stalled RUNNING job {job.urn} in place — abort unconfirmed, re-driving would risk a double-run", + extra=manager.logging_context(), + ) + return False + + # Abort confirmed dead (or the job was absent from ARQ). Refresh our view — if the aborted + # coroutine's own lifecycle handler already marked the row terminal, we want to see that — then + # re-drive through the shared retry handler, which preserves pipeline dependency semantics. + db.refresh(job) + await _handle_stalled_job_retry(job, manager, redis, stall_reason, db) + return True + + @with_guaranteed_job_run_record("cron_job") @with_job_management async def cleanup_stalled_jobs(ctx: dict, job_id: int, job_manager: JobManager) -> JobExecutionOutcome: """Detect and handle jobs that have stalled in intermediate states. - This job runs periodically (every 15 minutes) to find jobs that have been - stuck in QUEUED, RUNNING, or PENDING states beyond reasonable timeouts - and handles them appropriately. + This job runs periodically to find jobs that have been stuck in QUEUED, RUNNING, or PENDING + states beyond reasonable timeouts and handles them appropriately. Stalled job detection criteria: - QUEUED: Present in DB as QUEUED but absent from ARQ's Redis queue (process crashed between prepare_queue and redis.enqueue_job) - - RUNNING: Started > 60 minutes ago but not finished (worker likely crashed) + - RUNNING: Progress heartbeat idle > PROGRESS_STALL_MINUTES, or wall-clock runtime past the + RUNNING_TIMEOUT_MINUTES backstop, and not finished. Recovered abort-first (see + _handle_stalled_running_job) so a wedged attempt is never re-driven alongside its own retry. - PENDING: Created > 5 minutes ago in a pipeline and currently runnable (coordination failure) - Pipeline stuck: Non-terminal pipeline with no active jobs older than 5 minutes @@ -230,7 +318,8 @@ async def cleanup_stalled_jobs(ctx: dict, job_id: int, job_manager: JobManager) Job stalled in RUNNING (worker crash): - Worker started job, marked it RUNNING, then crashed - - After 60 minutes (longer than ARQ timeout), janitor detects and retries + - Once the progress heartbeat goes stale (or the wall-clock backstop is hit), the janitor + aborts the dead attempt and re-drives it """ job_manager.save_to_context( { @@ -305,14 +394,19 @@ async def cleanup_stalled_jobs(ctx: dict, job_id: int, job_manager: JobManager) job_manager.save_to_context({"cleaned_queued_jobs": queued_jobs}) logger.debug("Completed cleaning stalled QUEUED jobs.", extra=job_manager.logging_context()) - # Find RUNNING jobs that have been running too long OR have missing started_at - # These likely indicate worker crashes (worker died mid-execution) or data inconsistencies + # Find RUNNING jobs that are wedged. A job is a candidate when EITHER its progress heartbeat has + # gone stale (the primary, runtime-agnostic signal) OR it has blown past the wall-clock backstop + # (or is missing started_at entirely — a data inconsistency). Healthy long-running jobs keep + # bumping progress_updated_at and so are never selected until the backstop. + progress_stall_threshold = now - timedelta(minutes=PROGRESS_STALL_MINUTES) running_threshold = now - timedelta(minutes=RUNNING_TIMEOUT_MINUTES) running_jobs = job_manager.db.scalars( select(JobRun).where( JobRun.status == JobStatus.RUNNING, - (JobRun.started_at < running_threshold) | (JobRun.started_at.is_(None)), JobRun.finished_at.is_(None), + (JobRun.progress_updated_at < progress_stall_threshold) + | (JobRun.started_at < running_threshold) + | (JobRun.started_at.is_(None)), ) ).all() @@ -333,18 +427,26 @@ async def cleanup_stalled_jobs(ctx: dict, job_id: int, job_manager: JobManager) continue elapsed_minutes = (now - job.started_at).total_seconds() / 60 + heartbeat_stale = job.progress_updated_at < progress_stall_threshold + if heartbeat_stale: + heartbeat_age = (now - job.progress_updated_at).total_seconds() / 60 + stall_reason = ( + f"Job stalled in RUNNING state: progress heartbeat idle for {heartbeat_age:.1f} minutes " + f"(running for {elapsed_minutes:.1f} minutes)" + ) + else: + stall_reason = ( + f"Job stalled in RUNNING state for {elapsed_minutes:.1f} minutes " + f"(exceeded {RUNNING_TIMEOUT_MINUTES} min backstop; likely worker crash)" + ) - logger.warning( - f"Detected stalled RUNNING job {job.urn} " - f"(started {job.started_at}, running for {elapsed_minutes:.1f} minutes)", - extra=manager.logging_context(), - ) + logger.warning(f"Detected stalled RUNNING job {job.urn}: {stall_reason}", extra=manager.logging_context()) - stall_reason = f"Job stalled in RUNNING state for {elapsed_minutes:.1f} minutes (likely worker crash)" - await _handle_stalled_job_retry(job, manager, job_manager.redis, stall_reason, job_manager.db) + acted = await _handle_stalled_running_job(job, manager, job_manager.redis, stall_reason, job_manager.db) manager.db.commit() - cleaned_jobs["running"].append(job.urn) + if acted: + cleaned_jobs["running"].append(job.urn) job_manager.save_to_context({"cleaned_running_jobs": running_jobs}) logger.debug("Completed cleaning stalled RUNNING jobs.", extra=job_manager.logging_context()) diff --git a/src/mavedb/worker/lib/decorators/job_management.py b/src/mavedb/worker/lib/decorators/job_management.py index 58aad427c..0dcb463ad 100644 --- a/src/mavedb/worker/lib/decorators/job_management.py +++ b/src/mavedb/worker/lib/decorators/job_management.py @@ -5,6 +5,7 @@ Includes JobManager injection for advanced operations and robust error handling. """ +import asyncio import functools import inspect import logging @@ -15,7 +16,7 @@ from mavedb.lib.slack import send_slack_error, send_slack_job_error, send_slack_job_failure from mavedb.lib.types.workflow import JobExecutionOutcome -from mavedb.models.enums.job_pipeline import JobStatus +from mavedb.models.enums.job_pipeline import FailureCategory, JobStatus from mavedb.worker.lib.decorators.utils import ensure_ctx, ensure_job_id, ensure_session_ctx, is_test_mode from mavedb.worker.lib.managers import JobManager from mavedb.worker.lib.managers.constants import TERMINAL_JOB_STATUSES @@ -143,6 +144,43 @@ async def _execute_managed_job(func: Callable[..., Awaitable[JobExecutionOutcome return result + # The coroutine is being cancelled — either ARQ hit job_timeout, or cleanup_stalled_jobs + # aborted this attempt (Job.abort()) as part of stall recovery. CancelledError is a + # BaseException, so the `except Exception` handler below never sees it; without this branch + # the JobRun row would be orphaned as RUNNING forever while the coroutine dies. + # + # We mark the row terminally (FAILED / TIMEOUT) so DB state matches reality, then RE-RAISE. + # Re-raising is essential: it lets ARQ record the cancellation as the job's result, which is + # what makes a concurrent Job.abort() confirm the job actually died. + except asyncio.CancelledError: + logger.warning(f"Job {job_id} cancelled (timeout or stall-recovery abort); marking FAILED before re-raising") + try: + db_session.rollback() + if job_manager is not None: + job_manager.fail_job( + result=JobExecutionOutcome.failed( + reason="Job cancelled due to timeout, stall-recovery, or internal ARQ abort", + data={"reason": "cancelled"}, + failure_category=FailureCategory.TIMEOUT, + ) + ) + db_session.commit() + job = job_manager.get_job() + send_slack_job_failure( + job_urn=job.urn, + job_function=job.job_function, + reason="Job cancelled due to timeout.", + failure_category=str(FailureCategory.TIMEOUT), + retry_count=job.retry_count, + max_retries=job.max_retries, + will_retry=False, + ) + + except Exception as inner_e: + logger.critical(f"Failed to mark cancelled job {job_id} as failed: {inner_e}") + send_slack_error(inner_e) + raise + except Exception as e: # Prioritize salvaging lifecycle state will_retry = False diff --git a/src/mavedb/worker/settings/worker.py b/src/mavedb/worker/settings/worker.py index 6a558e6b6..3b0ab4e89 100644 --- a/src/mavedb/worker/settings/worker.py +++ b/src/mavedb/worker/settings/worker.py @@ -25,7 +25,13 @@ # driver, enabling incremental migration of job functions without touching # the FastAPI layer. Once all jobs use async sessions, raise MAX_JOBS to 10+. MAX_JOBS = 2 -JOB_TIMEOUT_SECONDS = 3 * 60 * 60 # 3 hours — matches RUNNING_TIMEOUT_MINUTES (150 min) with buffer +# ARQ's hard coroutine-kill ceiling. Kept deliberately high (8h) so ARQ is the *last* resort: +# our own DB-driven sweeper (cleanup_stalled_jobs) should detect and recover stalls long before +# this fires, keeping the JobRun state machine in sync with reality. VEP fan-out over large allele +# sets can legitimately run for hours, so a low ceiling would kill healthy work. +# cleanup's RUNNING_TIMEOUT_MINUTES backstop sits ~30 min under this to try and sweep these jobs before +# ARQ's hard timeout fires. +JOB_TIMEOUT_SECONDS = 8 * 60 * 60 # 8 hours class ArqWorkerSettings: @@ -44,3 +50,7 @@ class ArqWorkerSettings: max_jobs = MAX_JOBS job_timeout = JOB_TIMEOUT_SECONDS + # Required for cleanup_stalled_jobs to abort a wedged RUNNING job's coroutine (via Job.abort()) + # before re-driving it. Without this, ARQ ignores abort requests and the sweeper cannot safely + # confirm a stalled job is dead before retrying it. + allow_abort_jobs = True diff --git a/tests/worker/jobs/system/test_cleanup.py b/tests/worker/jobs/system/test_cleanup.py index 522b6167d..104c3af6d 100644 --- a/tests/worker/jobs/system/test_cleanup.py +++ b/tests/worker/jobs/system/test_cleanup.py @@ -12,6 +12,7 @@ pytest.importorskip("arq") # Skip tests if arq is not installed +import asyncio from datetime import datetime, timedelta, timezone from unittest.mock import AsyncMock, patch @@ -29,6 +30,7 @@ from mavedb.worker.jobs.system.cleanup import ( PENDING_TIMEOUT_MINUTES, PIPELINE_STUCK_TIMEOUT_MINUTES, + PROGRESS_STALL_MINUTES, RUNNING_TIMEOUT_MINUTES, cleanup_stalled_jobs, ) @@ -40,17 +42,55 @@ @pytest.fixture def mock_arq_job_not_found(): - """Mock ArqJob.status() to return not_found, simulating a crashed-enqueue QUEUED job.""" + """Mock ArqJob for a crashed-enqueue QUEUED job: status() is not_found, and abort() confirms dead.""" with patch("mavedb.worker.jobs.system.cleanup.ArqJob") as mock_arq_job: mock_arq_job.return_value.status = AsyncMock(return_value=ArqJobStatus.not_found) + mock_arq_job.return_value.abort = AsyncMock(return_value=True) yield mock_arq_job @pytest.fixture def mock_arq_job_in_redis(): - """Mock ArqJob.status() to return queued, simulating a legitimately queued job in ARQ Redis.""" + """Mock ArqJob for a legitimately queued job: status() is queued, and abort() confirms dead.""" with patch("mavedb.worker.jobs.system.cleanup.ArqJob") as mock_arq_job: mock_arq_job.return_value.status = AsyncMock(return_value=ArqJobStatus.queued) + mock_arq_job.return_value.abort = AsyncMock(return_value=True) + yield mock_arq_job + + +@pytest.fixture +def mock_arq_abort_confirmed(): + """Mock ArqJob.abort() to return True — the stalled RUNNING attempt was alive and is now dead. + + This is the signal that makes it safe for the sweeper to re-drive a stalled RUNNING job. + """ + with patch("mavedb.worker.jobs.system.cleanup.ArqJob") as mock_arq_job: + mock_arq_job.return_value.status = AsyncMock(return_value=ArqJobStatus.not_found) + mock_arq_job.return_value.abort = AsyncMock(return_value=True) + yield mock_arq_job + + +@pytest.fixture +def mock_arq_abort_absent(): + """Mock ArqJob.abort() to return False — the job is absent from ARQ (crashed / never enqueued). + + Nothing is running, so it is still safe to re-drive. + """ + with patch("mavedb.worker.jobs.system.cleanup.ArqJob") as mock_arq_job: + mock_arq_job.return_value.status = AsyncMock(return_value=ArqJobStatus.not_found) + mock_arq_job.return_value.abort = AsyncMock(return_value=False) + yield mock_arq_job + + +@pytest.fixture +def mock_arq_abort_wedged(): + """Mock ArqJob.abort() to time out — the coroutine is wedged in a blocking call and won't die. + + The sweeper must NOT re-drive such a job (it could still be running), to avoid a double-run. + """ + with patch("mavedb.worker.jobs.system.cleanup.ArqJob") as mock_arq_job: + mock_arq_job.return_value.status = AsyncMock(return_value=ArqJobStatus.in_progress) + mock_arq_job.return_value.abort = AsyncMock(side_effect=asyncio.TimeoutError()) yield mock_arq_job @@ -148,7 +188,7 @@ async def test_cleanup_stalled_queued_job_max_retries_reached( assert "stalled" in stalled_job.error_message.lower() async def test_cleanup_stalled_running_job_with_retries( - self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job + self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job, mock_arq_abort_confirmed ): """Test cleanup of a stalled RUNNING job with retries remaining.""" # Create a stalled RUNNING job in the database @@ -184,7 +224,7 @@ async def test_cleanup_stalled_running_job_with_retries( assert stalled_job.finished_at is None async def test_cleanup_stalled_running_job_max_retries_reached( - self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job + self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job, mock_arq_abort_confirmed ): """Test cleanup of a stalled RUNNING job with max retries reached.""" # Create a stalled RUNNING job with max retries @@ -257,6 +297,115 @@ async def test_cleanup_stalled_running_job_missing_started_at( assert stalled_job.status == JobStatus.RUNNING assert stalled_job.retry_count == 0 + async def test_cleanup_detects_running_job_by_stale_heartbeat( + self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job, mock_arq_abort_confirmed + ): + """A RUNNING job with a recent started_at but a stale progress heartbeat is detected and re-driven. + + This is the primary staleness signal: staleness is judged by progress_updated_at, not total runtime. + """ + stalled_job = JobRun( + job_type="test_job", + job_function="test_function", + status=JobStatus.RUNNING, + created_at=datetime.now(timezone.utc) - timedelta(minutes=40), + started_at=datetime.now(timezone.utc) - timedelta(minutes=30), # well within the wall-clock backstop + progress_updated_at=datetime.now(timezone.utc) - timedelta(minutes=PROGRESS_STALL_MINUTES + 5), + finished_at=None, + max_retries=3, + retry_count=0, + job_params={}, + ) + session.add(stalled_job) + session.commit() + + result = await cleanup_stalled_jobs( + mock_worker_ctx, None, JobManager(session, mock_worker_ctx["redis"], sample_cleanup_job_run.id) + ) + + # Abort of the wedged attempt was confirmed, so the job was re-driven. + mock_arq_abort_confirmed.return_value.abort.assert_awaited_once() + mock_worker_ctx["redis"].enqueue_job.assert_called_once() + + assert result.data["total_cleaned"] == 1 + assert stalled_job.urn in result.data["running_jobs"] + + session.refresh(stalled_job) + assert stalled_job.status == JobStatus.QUEUED + assert stalled_job.retry_count == 1 + + async def test_cleanup_leaves_running_job_with_fresh_heartbeat( + self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job + ): + """A RUNNING job that has been running a long time but is still checkpointing progress is left alone. + + This is the VEP fan-out case: legitimate multi-hour runtime, well past the old 150-min limit, + but under the wall-clock backstop and with a fresh heartbeat — must NOT be treated as stalled. + """ + healthy_job = JobRun( + job_type="test_job", + job_function="test_function", + status=JobStatus.RUNNING, + created_at=datetime.now(timezone.utc) - timedelta(minutes=310), + started_at=datetime.now(timezone.utc) - timedelta(minutes=300), # 5h — past old limit, under backstop + progress_updated_at=datetime.now(timezone.utc) - timedelta(minutes=2), # fresh heartbeat + finished_at=None, + max_retries=3, + retry_count=0, + job_params={}, + ) + session.add(healthy_job) + session.commit() + + result = await cleanup_stalled_jobs( + mock_worker_ctx, None, JobManager(session, mock_worker_ctx["redis"], sample_cleanup_job_run.id) + ) + + assert result.data["total_cleaned"] == 0 + assert healthy_job.urn not in result.data["running_jobs"] + + session.refresh(healthy_job) + assert healthy_job.status == JobStatus.RUNNING + assert healthy_job.retry_count == 0 + + async def test_cleanup_does_not_redrive_wedged_running_job( + self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job, mock_arq_abort_wedged + ): + """A stalled RUNNING job whose abort does not confirm is left in place, not re-driven. + + Re-driving a job that may still be executing (wedged in a blocking sync call) would recreate + the historical double-run, so the sweeper takes no action and leaves it for a later sweep. + """ + wedged_job = JobRun( + job_type="test_job", + job_function="test_function", + status=JobStatus.RUNNING, + created_at=datetime.now(timezone.utc) - timedelta(minutes=40), + started_at=datetime.now(timezone.utc) - timedelta(minutes=30), + progress_updated_at=datetime.now(timezone.utc) - timedelta(minutes=PROGRESS_STALL_MINUTES + 5), + finished_at=None, + max_retries=3, + retry_count=0, + job_params={}, + ) + session.add(wedged_job) + session.commit() + + result = await cleanup_stalled_jobs( + mock_worker_ctx, None, JobManager(session, mock_worker_ctx["redis"], sample_cleanup_job_run.id) + ) + + # Abort was attempted but never confirmed, so no retry was enqueued. + mock_arq_abort_wedged.return_value.abort.assert_awaited_once() + mock_worker_ctx["redis"].enqueue_job.assert_not_called() + + assert result.data["total_cleaned"] == 0 + assert wedged_job.urn not in result.data["running_jobs"] + + session.refresh(wedged_job) + assert wedged_job.status == JobStatus.RUNNING + assert wedged_job.retry_count == 0 + async def test_cleanup_stalled_pending_job_with_retries( self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job ): @@ -477,7 +626,7 @@ async def test_cleanup_stalled_queued_standalone_job_enqueue_failure( assert "Failed to enqueue after stall recovery" in stalled_job.error_message async def test_cleanup_stalled_running_standalone_job_enqueue_failure( - self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job + self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job, mock_arq_abort_confirmed ): """Test that stalled standalone RUNNING job is marked FAILED if ARQ enqueue fails.""" @@ -559,7 +708,7 @@ async def test_cleanup_stalled_queued_pipeline_job_dependencies_satisfied( assert stalled_job.retry_count == 1 async def test_cleanup_stalled_running_pipeline_job_dependencies_satisfied( - self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job + self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job, mock_arq_abort_confirmed ): """Test that stalled pipeline RUNNING job with satisfied dependencies is enqueued.""" # Create a pipeline with all dependencies satisfied @@ -735,7 +884,7 @@ async def test_cleanup_stalled_queued_pipeline_job_dependencies_not_ready( assert stalled_job.retry_count == 1 async def test_cleanup_stalled_running_pipeline_job_dependencies_failed( - self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job + self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job, mock_arq_abort_confirmed ): """Test that stalled pipeline RUNNING job with failed dependencies is skipped.""" # Create a pipeline @@ -867,7 +1016,7 @@ async def test_cleanup_stalled_pending_pipeline_job_dependencies_failed( assert stalled_job.retry_count == 0 async def test_cleanup_stalled_running_pipeline_job_dependencies_not_ready( - self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job + self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job, mock_arq_abort_confirmed ): """Test that stalled pipeline RUNNING job with dependencies not ready is skipped.""" # Create a pipeline @@ -1249,7 +1398,7 @@ async def test_cleanup_sends_slack_when_max_retries_reached_queued_job( assert call_kwargs["max_retries"] == 3 async def test_cleanup_sends_slack_when_max_retries_reached_running_job( - self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job + self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job, mock_arq_abort_confirmed ): """Test that send_slack_job_failure is called when sweeper permanently fails a stalled RUNNING job.""" stalled_job = JobRun( diff --git a/tests/worker/lib/decorators/test_job_management.py b/tests/worker/lib/decorators/test_job_management.py index dd2e10ef4..fbfb370d8 100644 --- a/tests/worker/lib/decorators/test_job_management.py +++ b/tests/worker/lib/decorators/test_job_management.py @@ -16,7 +16,7 @@ from sqlalchemy import select from mavedb.lib.types.workflow import JobExecutionOutcome -from mavedb.models.enums.job_pipeline import JobStatus +from mavedb.models.enums.job_pipeline import FailureCategory, JobStatus from mavedb.models.job_run import JobRun from mavedb.worker.lib.decorators.job_management import with_job_management from mavedb.worker.lib.managers.exceptions import JobStateError @@ -133,6 +133,67 @@ async def sample_error(ctx: dict, job_id: int, job_manager: JobManager): mock_start_job.assert_called_once() mock_error_job.assert_called_once() + async def test_decorator_marks_failed_and_reraises_on_cancellation( + self, session, mock_worker_ctx, mock_job_manager + ): + """A CancelledError (ARQ timeout or stall-recovery abort) must mark the row FAILED then re-raise. + + Re-raising is what lets ARQ record the cancellation as the job result so a concurrent + Job.abort() can confirm the job actually died. Without this handler the row would be + orphaned as RUNNING (CancelledError is a BaseException, not caught by `except Exception`). + """ + + @with_job_management + async def sample_cancel(ctx: dict, job_id: int, job_manager: JobManager): + raise asyncio.CancelledError() + + with ( + patch("mavedb.worker.lib.decorators.job_management.JobManager") as mock_job_manager_class, + patch("mavedb.worker.lib.decorators.job_management.send_slack_job_failure") as mock_send_slack_job_failure, + patch.object(mock_job_manager, "start_job", return_value=None) as mock_start_job, + patch.object(mock_job_manager, "fail_job", return_value=None) as mock_fail_job, + ): + mock_job_manager_class.return_value = mock_job_manager + + with pytest.raises(asyncio.CancelledError): + await sample_cancel(mock_worker_ctx, 999) + + mock_start_job.assert_called_once() + mock_fail_job.assert_called_once() + # The row is failed with a TIMEOUT category. + outcome = mock_fail_job.call_args.kwargs["result"] + assert outcome.status == JobStatus.FAILED + assert outcome.failure_category == FailureCategory.TIMEOUT + # A permanent-failure alert is emitted for the cancellation. + mock_send_slack_job_failure.assert_called_once() + + async def test_decorator_reraises_cancellation_even_if_marking_failed_errors( + self, session, mock_worker_ctx, mock_job_manager + ): + """If marking the cancelled job FAILED itself raises, the decorator still re-raises CancelledError. + + The salvage path must never swallow the cancellation — ARQ needs it to record the result. + """ + + @with_job_management + async def sample_cancel(ctx: dict, job_id: int, job_manager: JobManager): + raise asyncio.CancelledError() + + with ( + patch("mavedb.worker.lib.decorators.job_management.JobManager") as mock_job_manager_class, + patch("mavedb.worker.lib.decorators.job_management.send_slack_error") as mock_send_slack_error, + patch("mavedb.worker.lib.decorators.job_management.send_slack_job_failure"), + patch.object(mock_job_manager, "start_job", return_value=None), + patch.object(mock_job_manager, "fail_job", side_effect=RuntimeError("db down")), + ): + mock_job_manager_class.return_value = mock_job_manager + + with pytest.raises(asyncio.CancelledError): + await sample_cancel(mock_worker_ctx, 999) + + # The inner failure is reported, and the original cancellation still propagates. + mock_send_slack_error.assert_called_once() + async def test_decorator_calls_start_job_and_skip_job_when_wrapped_function_returns_skipped_status( self, session, mock_worker_ctx, mock_job_manager ): diff --git a/tests/worker/lib/managers/test_job_manager.py b/tests/worker/lib/managers/test_job_manager.py index 980a30f02..84f709bf4 100644 --- a/tests/worker/lib/managers/test_job_manager.py +++ b/tests/worker/lib/managers/test_job_manager.py @@ -11,6 +11,7 @@ pytest.importorskip("arq") import re +from datetime import datetime, timedelta, timezone from unittest.mock import Mock, PropertyMock, patch from arq import ArqRedis @@ -1348,6 +1349,28 @@ def test_update_progress_success(self, session, arq_redis, with_populated_job_da assert job.progress_total == 100 assert job.progress_message == "Halfway done" + def test_update_progress_bumps_progress_updated_at_heartbeat( + self, session, arq_redis, with_populated_job_data, sample_job_run + ): + """Committing a progress update refreshes progress_updated_at via onupdate=now(). + + This is the liveness heartbeat the stalled-job sweeper reads: it must advance on every + committing progress checkpoint with no explicit code in the job. + """ + + manager = JobManager(session, arq_redis, sample_job_run.id) + + # Pin the heartbeat to a stale value so the bump is unambiguous. + stale = datetime.now(timezone.utc) - timedelta(hours=1) + job = session.execute(select(JobRun).where(JobRun.id == sample_job_run.id)).scalar_one() + job.progress_updated_at = stale + session.commit() + + manager.update_progress(10, 100, "tick", commit=True) # commit triggers the UPDATE + onupdate + + job = session.execute(select(JobRun).where(JobRun.id == sample_job_run.id)).scalar_one() + assert job.progress_updated_at > stale + def test_update_progress_success_does_not_overwrite_old_message_when_no_new_message_is_provided( self, session, arq_redis, with_populated_job_data, sample_job_run ): From 4e1ddf5211ad9012d5d25f9fa8c01afd08fd13d9 Mon Sep 17 00:00:00 2001 From: David Reinhart Date: Tue, 7 Jul 2026 11:17:34 -0700 Subject: [PATCH 084/252] feat(uniprot): support Ensembl protein accessions in UniProt ID mapping infer_db_name_from_sequence_accession now recognizes ENSP accessions and maps them to the Ensembl_Protein source, so accession-based target genes with Ensembl protein IDs resolve to a UniProt ID instead of raising NotImplementedError. Also make validate_ensembl_identifier accept versioned Ensembl stable IDs (e.g. ENSP00000369497.3) by stripping the trailing . before the idutils.is_ensembl check, matching how validate_refseq_identifier already handles versioned RefSeq accessions. Related to varianteffect/dcd_mapping2#110 --- src/mavedb/lib/uniprot/utils.py | 14 +++++++++++--- src/mavedb/lib/validation/identifier.py | 8 +++++++- tests/lib/uniprot/test_utils.py | 8 ++++++++ tests/validation/test_identifier.py | 4 ++++ 4 files changed, 30 insertions(+), 4 deletions(-) diff --git a/src/mavedb/lib/uniprot/utils.py b/src/mavedb/lib/uniprot/utils.py index 7f87524b2..11c13a8f4 100644 --- a/src/mavedb/lib/uniprot/utils.py +++ b/src/mavedb/lib/uniprot/utils.py @@ -1,11 +1,11 @@ from typing import Literal, Union -from mavedb.lib.validation.identifier import validate_refseq_identifier +from mavedb.lib.validation.identifier import validate_ensembl_identifier, validate_refseq_identifier def infer_db_name_from_sequence_accession( sequence_accession: str, -) -> Union[Literal["RefSeq_Nucleotide", "RefSeq_Protein"]]: +) -> Union[Literal["RefSeq_Nucleotide", "RefSeq_Protein", "Ensembl_Protein"]]: """ Infers the database name from a sequence accession. @@ -15,6 +15,12 @@ def infer_db_name_from_sequence_accession( Returns: str: The inferred database name. """ + # Ensembl protein accessions must be handled before validate_refseq_identifier, + # which raises on any non-RefSeq accession. + if sequence_accession.startswith("ENSP"): + validate_ensembl_identifier(sequence_accession) + return "Ensembl_Protein" + validate_refseq_identifier(sequence_accession) if sequence_accession.startswith("NM_"): @@ -22,4 +28,6 @@ def infer_db_name_from_sequence_accession( if sequence_accession.startswith("NP_"): return "RefSeq_Protein" - raise NotImplementedError("Only RefSeq NM and NP identifiers are currently supported for inference.") + raise NotImplementedError( + "Only RefSeq NM/NP and Ensembl protein (ENSP) identifiers are currently supported for inference." + ) diff --git a/src/mavedb/lib/validation/identifier.py b/src/mavedb/lib/validation/identifier.py index e6e2dcde7..077b13fc0 100644 --- a/src/mavedb/lib/validation/identifier.py +++ b/src/mavedb/lib/validation/identifier.py @@ -1,3 +1,5 @@ +import re + import idutils from mavedb.lib.validation.constants.identifier import valid_dbnames @@ -52,6 +54,9 @@ def validate_ensembl_identifier(identifier: str): """ Validates whether the identifier is a valid Ensembl identifier. + Ensembl stable IDs may carry a version suffix (e.g. ``ENSP00000369497.3``), which + ``idutils.is_ensembl`` does not accept; strip it before validating the stable ID. + Parameters __________ identifier : str @@ -62,7 +67,8 @@ def validate_ensembl_identifier(identifier: str): ValidationError If the identifier is not a valid Ensembl identifier. """ - if not idutils.is_ensembl(identifier): + base_identifier = re.sub(r"\.\d+$", "", identifier) + if not idutils.is_ensembl(base_identifier): raise ValidationError(f"'{identifier}' is not a valid Ensembl accession.") diff --git a/tests/lib/uniprot/test_utils.py b/tests/lib/uniprot/test_utils.py index 7afa278c2..5ac773e38 100644 --- a/tests/lib/uniprot/test_utils.py +++ b/tests/lib/uniprot/test_utils.py @@ -21,6 +21,14 @@ def test_infer_db_name_from_sequence_accession_np(): assert result == "RefSeq_Protein" +# Both versioned and unversioned ENSP accessions must route to Ensembl_Protein; the versioned case +# guards that validate_ensembl_identifier's version handling continues to accept it. +@pytest.mark.parametrize("ensembl_protein_accession", ["ENSP00000418960", "ENSP00000418960.3"]) +def test_infer_db_name_from_sequence_accession_ensp(ensembl_protein_accession): + result = infer_db_name_from_sequence_accession(ensembl_protein_accession) + assert result == "Ensembl_Protein" + + @pytest.mark.parametrize("invalid_accession", ["XP_000000", VALID_CHR_ACCESSION]) def test_infer_db_name_from_sequence_accession_invalid(invalid_accession): with pytest.raises(NotImplementedError): diff --git a/tests/validation/test_identifier.py b/tests/validation/test_identifier.py index 005278307..9cbd16b0d 100644 --- a/tests/validation/test_identifier.py +++ b/tests/validation/test_identifier.py @@ -63,6 +63,10 @@ def test_ve_invalid_list(self): def test_passes_valid_id(self): validate_ensembl_identifier("ENSG00000143384") + def test_passes_valid_versioned_id(self): + validate_ensembl_identifier("ENSP00000369497.3") + validate_ensembl_list(["ENSG00000139618.15", "ENST00000380152.8"]) + class TestRefSeqValidators(TestCase): """ From 97cf2f385c36c79958b3de6fb7e079b96aa43b15 Mon Sep 17 00:00:00 2001 From: David Reinhart Date: Wed, 8 Jul 2026 11:49:19 -0700 Subject: [PATCH 085/252] Add Ensembl transcript (ENST) support to sequence accession inference Extend infer_db_name_from_sequence_accession to recognize ENST identifiers and return "Ensembl_Transcript" as the database name. Restructure control flow to validate each accession type only within its matching branch rather than calling validate_refseq_identifier unconditionally for non-Ensembl accessions. --- src/mavedb/lib/uniprot/utils.py | 18 +++++++++--------- tests/lib/uniprot/test_utils.py | 18 +++++++++++------- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/src/mavedb/lib/uniprot/utils.py b/src/mavedb/lib/uniprot/utils.py index 11c13a8f4..ec5a8a185 100644 --- a/src/mavedb/lib/uniprot/utils.py +++ b/src/mavedb/lib/uniprot/utils.py @@ -5,7 +5,7 @@ def infer_db_name_from_sequence_accession( sequence_accession: str, -) -> Union[Literal["RefSeq_Nucleotide", "RefSeq_Protein", "Ensembl_Protein"]]: +) -> Union[Literal["RefSeq_Nucleotide", "RefSeq_Protein", "Ensembl_Protein", "Ensembl_Transcript"]]: """ Infers the database name from a sequence accession. @@ -15,19 +15,19 @@ def infer_db_name_from_sequence_accession( Returns: str: The inferred database name. """ - # Ensembl protein accessions must be handled before validate_refseq_identifier, - # which raises on any non-RefSeq accession. if sequence_accession.startswith("ENSP"): validate_ensembl_identifier(sequence_accession) return "Ensembl_Protein" - - validate_refseq_identifier(sequence_accession) - - if sequence_accession.startswith("NM_"): + elif sequence_accession.startswith("ENST"): + validate_ensembl_identifier(sequence_accession) + return "Ensembl_Transcript" + elif sequence_accession.startswith("NM_"): + validate_refseq_identifier(sequence_accession) return "RefSeq_Nucleotide" - if sequence_accession.startswith("NP_"): + elif sequence_accession.startswith("NP_"): + validate_refseq_identifier(sequence_accession) return "RefSeq_Protein" raise NotImplementedError( - "Only RefSeq NM/NP and Ensembl protein (ENSP) identifiers are currently supported for inference." + "Only RefSeq (NM_/NP_) and Ensembl (ENSP/ENST) identifiers are currently supported for inference." ) diff --git a/tests/lib/uniprot/test_utils.py b/tests/lib/uniprot/test_utils.py index 5ac773e38..5d51c2afe 100644 --- a/tests/lib/uniprot/test_utils.py +++ b/tests/lib/uniprot/test_utils.py @@ -1,12 +1,10 @@ import pytest -from mavedb.lib.validation.exceptions import ValidationError from mavedb.lib.uniprot.utils import infer_db_name_from_sequence_accession - from tests.helpers.constants import ( + VALID_CHR_ACCESSION, VALID_NT_ACCESSION, VALID_PRO_ACCESSION, - VALID_CHR_ACCESSION, VALID_UNIPROT_ACCESSION, ) @@ -21,14 +19,20 @@ def test_infer_db_name_from_sequence_accession_np(): assert result == "RefSeq_Protein" -# Both versioned and unversioned ENSP accessions must route to Ensembl_Protein; the versioned case -# guards that validate_ensembl_identifier's version handling continues to accept it. +# Both versioned and unversioned ENSP and ENST accessions must route to Ensembl_Protein and Ensembl_Transcript, respectively; +# the versioned case guards that validate_ensembl_identifier's version handling continues to accept it. @pytest.mark.parametrize("ensembl_protein_accession", ["ENSP00000418960", "ENSP00000418960.3"]) def test_infer_db_name_from_sequence_accession_ensp(ensembl_protein_accession): result = infer_db_name_from_sequence_accession(ensembl_protein_accession) assert result == "Ensembl_Protein" +@pytest.mark.parametrize("ensembl_transcript_accession", ["ENST00000418960", "ENST00000418960.3"]) +def test_infer_db_name_from_sequence_accession_enst(ensembl_transcript_accession): + result = infer_db_name_from_sequence_accession(ensembl_transcript_accession) + assert result == "Ensembl_Transcript" + + @pytest.mark.parametrize("invalid_accession", ["XP_000000", VALID_CHR_ACCESSION]) def test_infer_db_name_from_sequence_accession_invalid(invalid_accession): with pytest.raises(NotImplementedError): @@ -36,10 +40,10 @@ def test_infer_db_name_from_sequence_accession_invalid(invalid_accession): def test_infer_db_name_from_non_refseq_accession(): - with pytest.raises(ValidationError): + with pytest.raises(NotImplementedError): infer_db_name_from_sequence_accession(VALID_UNIPROT_ACCESSION) def test_infer_db_name_from_sequence_accession_empty_string(): - with pytest.raises(ValidationError): + with pytest.raises(NotImplementedError): infer_db_name_from_sequence_accession("") From 1bf1c2b640bb18e348b5cc077f6ccdcbe0ed4149 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Thu, 9 Jul 2026 10:35:24 -0700 Subject: [PATCH 086/252] chore(export): add changelog to public data archive and document release process MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add CHANGELOG.md to scripts/resources, bundled into every generated archive so each downloaded version carries its own history; documents versions 1–4 (pre-changelog) and v5 in detail - Update export script to include CHANGELOG.md alongside the README and license in the zip output - Add a release-procedure note to the module docstring pointing maintainers to the changelog and deployment docs before each Zenodo publish --- src/mavedb/scripts/export_public_data.py | 7 ++- src/mavedb/scripts/resources/CHANGELOG.md | 62 +++++++++++++++++++++++ src/mavedb/scripts/resources/README.md | 1 + 3 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 src/mavedb/scripts/resources/CHANGELOG.md diff --git a/src/mavedb/scripts/export_public_data.py b/src/mavedb/scripts/export_public_data.py index 4ced338a8..cedc66608 100644 --- a/src/mavedb/scripts/export_public_data.py +++ b/src/mavedb/scripts/export_public_data.py @@ -11,6 +11,10 @@ Unpublished data and data sets licensed other than under the Creative Commons Zero license are not included in the dump, and user details are limited to ORCID IDs and names of contributors to published data sets. + +RELEASING A NEW VERSION: Before publishing a new version of this archive to Zenodo, add an entry describing what +changed to `src/mavedb/scripts/resources/CHANGELOG.md` (it is bundled into the archive). The full release procedure +is documented in `deployment/docs/zenodo-release.md`. """ import json @@ -129,10 +133,11 @@ def export_public_data(db: Session): # Write metadata for all data sets to a single JSON file. zipfile.writestr("main.json", json.dumps(jsonable_encoder(json_data))) - # Copy the CC0 license and README. + # Copy the CC0 license, README, and changelog. resources_dir = os.path.join(os.path.dirname(__file__), "resources") zipfile.write(os.path.join(resources_dir, "CC0_license.txt"), "LICENSE.txt") zipfile.write(os.path.join(resources_dir, "README.md"), "README.md") + zipfile.write(os.path.join(resources_dir, "CHANGELOG.md"), "CHANGELOG.md") # Write score and count files for each score set. num_score_sets = len(score_set_ids) diff --git a/src/mavedb/scripts/resources/CHANGELOG.md b/src/mavedb/scripts/resources/CHANGELOG.md new file mode 100644 index 000000000..3f441fc0c --- /dev/null +++ b/src/mavedb/scripts/resources/CHANGELOG.md @@ -0,0 +1,62 @@ +# MaveDB Public Data Dump — Changelog + +This file records what changed between versions of the MaveDB bulk data archive +published on [Zenodo](https://doi.org/10.5281/zenodo.11201736). It ships inside +every archive so the version you downloaded always carries its own history. + +Versions are the Zenodo archive versions (v1, v2, …), not MaveDB API or website +release numbers. The `asOf` timestamp in `main.json` records the exact moment a +given archive was generated. + +--- + +## [5] — 2026-06-25 + +Version 5 is the first version tracked in this changelog. Versions 1–4 contained +metadata (`main.json`) plus per-score-set score and count CSVs; they predate this +file and are not itemized here. Version 5 adds variant-mapping and annotation +outputs derived from the MaveDB variant mapping pipeline. + +### Added + +- **Annotation CSVs** — `csv/{urn}.annotations.csv` for every score set that has + completed the mapping pipeline. Joins external-database annotations (Ensembl VEP + functional consequence, gnomAD v4.1 allele frequency, ClinGen Allele Registry ID) + with post-mapped HGVS (`g.`/`c.`/`p.` and assay-level) and the GA4GH VRS digest. +- **Mapped-variant JSON** — `mapped/{urn}.mapped-variants.json` for every mapped + score set. Each record carries the pre-mapped and post-mapped VRS alleles, the + VRS schema version, the mapping API version, and the ClinGen allele ID, mirroring + `GET /api/v1/score-sets/{urn}/mapped-variants`. +- **VA-Spec annotation NDJSON** — `va/{urn}.va.ndjson` for every mapped score set. + One line per current mapped variant carrying its highest materialized GA4GH + VA-Spec layer (study result → functional-impact statement → pathogenicity + statement), mirroring the `annotated-variants` streaming endpoints. +- **ClinVar release columns** in the annotation CSVs, covering the `2015_02` + through `2026_01` annual snapshots. + +### Changed + +- **CSV columns are now namespaced.** Every column carries a source prefix + separated by a dot: score columns are `scores.*` (e.g. `scores.score`), counts + are `counts.*`, mapping-pipeline columns are `mavedb.*`, and external annotations + are `vep.*` / `gnomad.*` / `clingen.*`. Core identifier columns (`accession`, + `hgvs_nt`, `hgvs_pro`, `hgvs_splice`) remain unprefixed. **Consumers that parsed + earlier archives by bare column name (e.g. `score`) must update to the namespaced + names.** +- **Cis-phased (haplotype) variants** are now combined into a single HGVS + expression in the CSV outputs where a combined expression can be derived. +- Mapped-variant and VA-Spec outputs include only the **current** mapping for each + variant; superseded mappings are excluded. + +### Fixed + +- VA-Spec annotation output is now valid and round-trippable (it deserializes back + into GA4GH VA-Spec objects without loss). + +### Notes + +- Annotation, mapped-variant, and VA-Spec files are present **only** for score sets + that have completed the mapping pipeline. Unmapped score sets ship scores/counts + CSVs only, as before. +- See `README.md` in this archive for the full file layout, column definitions, and + caveats. diff --git a/src/mavedb/scripts/resources/README.md b/src/mavedb/scripts/resources/README.md index 31ec4e26c..44b8eadf2 100644 --- a/src/mavedb/scripts/resources/README.md +++ b/src/mavedb/scripts/resources/README.md @@ -30,6 +30,7 @@ Unpublished data, private datasets, and datasets published under other licenses ``` mavedb-dump.YYYYMMDDHHMMSS.zip ├── README.md # This file +├── CHANGELOG.md # What changed between archive versions ├── LICENSE.txt # Creative Commons CC0 1.0 license text ├── main.json # Metadata for all included datasets ├── csv/ From 676560010d88b7543d53258b43d50c34f1e28b98 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Thu, 9 Jul 2026 11:23:16 -0700 Subject: [PATCH 087/252] feat(worker): detect stuck RUNNING jobs via progress heartbeat Replace the pure wall-clock RUNNING-stall check with a progress heartbeat, so a healthy long-running job (e.g. large VEP fan-out) is no longer killed while a genuinely wedged one is still caught. Make the DB-driven sweeper authoritative over termination and keep ARQ's timeout only as a last resort. - Add JobRun.progress_updated_at (onupdate=now()) as a zero-code liveness heartbeat, bumped on every committing progress checkpoint; index (status, progress_updated_at) for the sweeper query - Detect stalled RUNNING jobs on stale heartbeat (PROGRESS_STALL_MINUTES = 20) OR a wall-clock backstop (RUNNING_TIMEOUT_MINUTES = 450), and recover them abort-first: abort the current attempt and only re-drive once it is confirmed dead or absent, eliminating the double-run where a retry ran alongside the still-live original - Handle asyncio.CancelledError in with_job_management: mark the row FAILED/TIMEOUT then re-raise, so an ARQ timeout or a stall-recovery abort no longer orphans the row as RUNNING and Job.abort() can confirm - Raise ARQ job_timeout to 8h and enable allow_abort_jobs; tighten the cleanup cron to every 10 minutes - Cover new behavior with unit and integration tests --- ...104_add_progress_updated_at_to_job_runs.py | 44 +++++ src/mavedb/models/job_run.py | 10 ++ src/mavedb/worker/jobs/registry.py | 6 +- src/mavedb/worker/jobs/system/cleanup.py | 140 +++++++++++++-- .../worker/lib/decorators/job_management.py | 40 ++++- src/mavedb/worker/settings/worker.py | 12 +- tests/worker/jobs/system/test_cleanup.py | 167 +++++++++++++++++- .../lib/decorators/test_job_management.py | 63 ++++++- tests/worker/lib/managers/test_job_manager.py | 23 +++ 9 files changed, 472 insertions(+), 33 deletions(-) create mode 100644 alembic/versions/a7f3c2e9b104_add_progress_updated_at_to_job_runs.py diff --git a/alembic/versions/a7f3c2e9b104_add_progress_updated_at_to_job_runs.py b/alembic/versions/a7f3c2e9b104_add_progress_updated_at_to_job_runs.py new file mode 100644 index 000000000..d04ceab9b --- /dev/null +++ b/alembic/versions/a7f3c2e9b104_add_progress_updated_at_to_job_runs.py @@ -0,0 +1,44 @@ +"""Add progress_updated_at heartbeat column to job_runs + +Revision ID: a7f3c2e9b104 +Revises: 4dbf24ed1857 +Create Date: 2026-07-09 00:00:00.000000 + +""" + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "a7f3c2e9b104" +down_revision = "4dbf24ed1857" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column( + "job_runs", + sa.Column( + "progress_updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + ) + op.create_index( + "ix_job_runs_status_progress_updated", + "job_runs", + ["status", "progress_updated_at"], + unique=False, + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index("ix_job_runs_status_progress_updated", table_name="job_runs") + op.drop_column("job_runs", "progress_updated_at") + # ### end Alembic commands ### diff --git a/src/mavedb/models/job_run.py b/src/mavedb/models/job_run.py index 877eeab02..b079a1757 100644 --- a/src/mavedb/models/job_run.py +++ b/src/mavedb/models/job_run.py @@ -70,6 +70,14 @@ class JobRun(Base): progress_total: Mapped[Optional[int]] = mapped_column(Integer, nullable=True) progress_message: Mapped[Optional[str]] = mapped_column(String(500), nullable=True) + # Liveness heartbeat. `onupdate=func.now()` bumps this on *any* commit that dirties the row, + # so every progress checkpoint (and every lifecycle transition) refreshes it with zero per-job + # code. The stalled-job sweeper treats a RUNNING job whose heartbeat has gone stale as wedged, + # which distinguishes a healthy long-running job (still committing progress) from a dead one. + progress_updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now(), onupdate=func.now() + ) + # Correlation for tracing correlation_id: Mapped[Optional[str]] = mapped_column(String(255), nullable=True) @@ -98,6 +106,8 @@ class JobRun(Base): Index("ix_job_runs_created_at", "created_at"), Index("ix_job_runs_correlation_id", "correlation_id"), Index("ix_job_runs_status_scheduled", "status", "scheduled_at"), + # Supports the stalled-job sweeper's RUNNING heartbeat query (status + heartbeat staleness). + Index("ix_job_runs_status_progress_updated", "status", "progress_updated_at"), CheckConstraint( "status IN ('pending', 'queued', 'running', 'succeeded', 'failed', 'errored', 'cancelled', 'skipped')", name="ck_job_runs_status_valid", diff --git a/src/mavedb/worker/jobs/registry.py b/src/mavedb/worker/jobs/registry.py index 94cd76e32..ba9754a05 100644 --- a/src/mavedb/worker/jobs/registry.py +++ b/src/mavedb/worker/jobs/registry.py @@ -71,8 +71,10 @@ cron( cleanup_stalled_jobs, name="cleanup_stalled_jobs_cron", - minute={15, 45}, # Run at :15 and :45 past each hour (every 30 minutes) - keep_result=timedelta(minutes=25).total_seconds(), + # Every 10 minutes. Paired with the 20-min progress-stall window this gives an effective + # stall-detection latency of ~20-30 min (vs. up to 50 min on the old 30-min cadence). + minute={5, 15, 25, 35, 45, 55}, + keep_result=timedelta(minutes=9).total_seconds(), ), ] diff --git a/src/mavedb/worker/jobs/system/cleanup.py b/src/mavedb/worker/jobs/system/cleanup.py index 89089828c..4a2dc975e 100644 --- a/src/mavedb/worker/jobs/system/cleanup.py +++ b/src/mavedb/worker/jobs/system/cleanup.py @@ -13,6 +13,7 @@ The cleanup job acts as a safety net to ensure jobs don't remain in limbo forever. """ +import asyncio import logging from datetime import datetime, timedelta, timezone @@ -37,14 +38,30 @@ logger = logging.getLogger(__name__) # Timeout thresholds for detecting stalled jobs (in minutes). -# RUNNING_TIMEOUT_MINUTES must stay below ArqWorkerSettings.job_timeout (currently 2 hours) -# to avoid marking legitimately running jobs as stalled. -RUNNING_TIMEOUT_MINUTES = 150 # RUNNING jobs should complete within 150 min (30 min buffer under ARQ timeout) +# +# RUNNING jobs are evaluated on two signals: +# 1. PROGRESS_STALL_MINUTES — the primary signal. A healthy job checkpoints progress frequently +# (JobRun.progress_updated_at is bumped on every committing progress update via onupdate=now()). +# If the heartbeat has not advanced in this window the job is considered wedged, regardless of +# how long it has legitimately been running. This is what lets a multi-hour VEP fan-out run to +# completion without being killed, while still catching a job that has genuinely hung. +# 2. RUNNING_TIMEOUT_MINUTES — a wall-clock backstop for jobs that never emit progress (or whose +# heartbeat column is somehow stuck). Sits ~30 min under ArqWorkerSettings.job_timeout (8h) so +# the DB-driven sweeper, not ARQ's hard kill, is what terminates a job — keeping DB state and +# actual work in sync. +PROGRESS_STALL_MINUTES = 20 # RUNNING jobs should checkpoint progress at least this often +RUNNING_TIMEOUT_MINUTES = 450 # Wall-clock backstop (7.5h), 30 min under the 8h ARQ ceiling PENDING_TIMEOUT_MINUTES = 5 # PENDING jobs which are actionable within pipelines should be enqueued within 5 minutes PIPELINE_STUCK_TIMEOUT_MINUTES = ( 5 # Pipelines in non-terminal states with no active jobs should resolve within 5 minutes ) +# How long the sweeper waits for an aborted RUNNING job to actually die before giving up. A job +# cancelled at an await point dies within a poll cycle (sub-second); a job wedged in a blocking sync +# call never honors the abort and will hit this timeout — in which case we do NOT re-drive it (see +# _handle_stalled_running_job) to avoid running it twice. +ABORT_CONFIRM_TIMEOUT_SECONDS = 15 + async def _handle_stalled_job_retry( job: JobRun, @@ -193,19 +210,90 @@ async def _handle_stalled_job_retry( return False +async def _running_job_confirmed_dead(arq_id: str, redis: ArqRedis) -> bool: + """Abort a RUNNING job's current ARQ attempt and report whether it is safe to re-drive. + + RUNNING recovery differs from QUEUED/PENDING recovery: a RUNNING job may still have a live + coroutine holding a worker slot. Re-enqueuing without cancelling it first may mean two + coroutines are running the same job simultaneously, which is unsafe. + + We therefore issue Job.abort() on the current attempt and interpret the outcome: + + - abort() returns True -> the coroutine was alive and has now been cancelled + recorded a + terminal result. Safe to re-drive. + - abort() returns False -> the job is absent from ARQ (crashed worker / never enqueued / already + finished). Nothing is running. Safe to re-drive — this is the common crash-recovery path. + - abort() raises TimeoutError -> the job is still present in ARQ and did not die within the + window: a coroutine wedged in a blocking sync call that cannot honor cooperative cancellation. + NOT safe to re-drive; leave it for the next sweep or ARQ's own ceiling. + - any other error -> treat as not-confirmably-alive and allow re-drive; a genuinely live job + would have either confirmed the abort or timed out, not errored here. + + Returns True when it is safe to re-drive the job, False when the job may still be running. + """ + try: + await ArqJob(arq_id, redis).abort(timeout=ABORT_CONFIRM_TIMEOUT_SECONDS) + return True + except asyncio.TimeoutError: + logger.warning( + f"Abort of RUNNING job attempt {arq_id} did not confirm within " + f"{ABORT_CONFIRM_TIMEOUT_SECONDS}s; job may be wedged in a blocking call — not re-driving" + ) + return False + except Exception as e: + logger.warning(f"Abort of RUNNING job attempt {arq_id} raised {e!r}; treating as not running and re-driving") + return True + + +async def _handle_stalled_running_job( + job: JobRun, + manager: JobManager, + redis: ArqRedis, + stall_reason: str, + db: Session, +) -> bool: + """Recover a stalled RUNNING job via abort-first ordering. + + Aborts the current attempt (urn#N) BEFORE any retry bookkeeping, and only proceeds to re-drive + (which increments retry_count and enqueues urn#N+1) once the abort confirms the original is dead + or absent. If the original refuses to die, we leave it alone rather than risk a double-run. + + Returns True if the sweeper acted on the job (re-drove or permanently failed it), False if it was + left in place because the abort could not be confirmed (so it should not be counted as cleaned). + """ + arq_id = arq_job_id(job) + + # Could not confirm the coroutine is dead. Leave the job RUNNING; the next sweep will retry + # the abort, and ARQ's job_timeout ceiling remains as a final backstop. + if not await _running_job_confirmed_dead(arq_id, redis): + logger.warning( + f"Leaving stalled RUNNING job {job.urn} in place — abort unconfirmed, re-driving would risk a double-run", + extra=manager.logging_context(), + ) + return False + + # Abort confirmed dead (or the job was absent from ARQ). Refresh our view — if the aborted + # coroutine's own lifecycle handler already marked the row terminal, we want to see that — then + # re-drive through the shared retry handler, which preserves pipeline dependency semantics. + db.refresh(job) + await _handle_stalled_job_retry(job, manager, redis, stall_reason, db) + return True + + @with_guaranteed_job_run_record("cron_job") @with_job_management async def cleanup_stalled_jobs(ctx: dict, job_id: int, job_manager: JobManager) -> JobExecutionOutcome: """Detect and handle jobs that have stalled in intermediate states. - This job runs periodically (every 15 minutes) to find jobs that have been - stuck in QUEUED, RUNNING, or PENDING states beyond reasonable timeouts - and handles them appropriately. + This job runs periodically to find jobs that have been stuck in QUEUED, RUNNING, or PENDING + states beyond reasonable timeouts and handles them appropriately. Stalled job detection criteria: - QUEUED: Present in DB as QUEUED but absent from ARQ's Redis queue (process crashed between prepare_queue and redis.enqueue_job) - - RUNNING: Started > 60 minutes ago but not finished (worker likely crashed) + - RUNNING: Progress heartbeat idle > PROGRESS_STALL_MINUTES, or wall-clock runtime past the + RUNNING_TIMEOUT_MINUTES backstop, and not finished. Recovered abort-first (see + _handle_stalled_running_job) so a wedged attempt is never re-driven alongside its own retry. - PENDING: Created > 5 minutes ago in a pipeline and currently runnable (coordination failure) - Pipeline stuck: Non-terminal pipeline with no active jobs older than 5 minutes @@ -230,7 +318,8 @@ async def cleanup_stalled_jobs(ctx: dict, job_id: int, job_manager: JobManager) Job stalled in RUNNING (worker crash): - Worker started job, marked it RUNNING, then crashed - - After 60 minutes (longer than ARQ timeout), janitor detects and retries + - Once the progress heartbeat goes stale (or the wall-clock backstop is hit), the janitor + aborts the dead attempt and re-drives it """ job_manager.save_to_context( { @@ -305,14 +394,19 @@ async def cleanup_stalled_jobs(ctx: dict, job_id: int, job_manager: JobManager) job_manager.save_to_context({"cleaned_queued_jobs": queued_jobs}) logger.debug("Completed cleaning stalled QUEUED jobs.", extra=job_manager.logging_context()) - # Find RUNNING jobs that have been running too long OR have missing started_at - # These likely indicate worker crashes (worker died mid-execution) or data inconsistencies + # Find RUNNING jobs that are wedged. A job is a candidate when EITHER its progress heartbeat has + # gone stale (the primary, runtime-agnostic signal) OR it has blown past the wall-clock backstop + # (or is missing started_at entirely — a data inconsistency). Healthy long-running jobs keep + # bumping progress_updated_at and so are never selected until the backstop. + progress_stall_threshold = now - timedelta(minutes=PROGRESS_STALL_MINUTES) running_threshold = now - timedelta(minutes=RUNNING_TIMEOUT_MINUTES) running_jobs = job_manager.db.scalars( select(JobRun).where( JobRun.status == JobStatus.RUNNING, - (JobRun.started_at < running_threshold) | (JobRun.started_at.is_(None)), JobRun.finished_at.is_(None), + (JobRun.progress_updated_at < progress_stall_threshold) + | (JobRun.started_at < running_threshold) + | (JobRun.started_at.is_(None)), ) ).all() @@ -333,18 +427,26 @@ async def cleanup_stalled_jobs(ctx: dict, job_id: int, job_manager: JobManager) continue elapsed_minutes = (now - job.started_at).total_seconds() / 60 + heartbeat_stale = job.progress_updated_at < progress_stall_threshold + if heartbeat_stale: + heartbeat_age = (now - job.progress_updated_at).total_seconds() / 60 + stall_reason = ( + f"Job stalled in RUNNING state: progress heartbeat idle for {heartbeat_age:.1f} minutes " + f"(running for {elapsed_minutes:.1f} minutes)" + ) + else: + stall_reason = ( + f"Job stalled in RUNNING state for {elapsed_minutes:.1f} minutes " + f"(exceeded {RUNNING_TIMEOUT_MINUTES} min backstop; likely worker crash)" + ) - logger.warning( - f"Detected stalled RUNNING job {job.urn} " - f"(started {job.started_at}, running for {elapsed_minutes:.1f} minutes)", - extra=manager.logging_context(), - ) + logger.warning(f"Detected stalled RUNNING job {job.urn}: {stall_reason}", extra=manager.logging_context()) - stall_reason = f"Job stalled in RUNNING state for {elapsed_minutes:.1f} minutes (likely worker crash)" - await _handle_stalled_job_retry(job, manager, job_manager.redis, stall_reason, job_manager.db) + acted = await _handle_stalled_running_job(job, manager, job_manager.redis, stall_reason, job_manager.db) manager.db.commit() - cleaned_jobs["running"].append(job.urn) + if acted: + cleaned_jobs["running"].append(job.urn) job_manager.save_to_context({"cleaned_running_jobs": running_jobs}) logger.debug("Completed cleaning stalled RUNNING jobs.", extra=job_manager.logging_context()) diff --git a/src/mavedb/worker/lib/decorators/job_management.py b/src/mavedb/worker/lib/decorators/job_management.py index 58aad427c..0dcb463ad 100644 --- a/src/mavedb/worker/lib/decorators/job_management.py +++ b/src/mavedb/worker/lib/decorators/job_management.py @@ -5,6 +5,7 @@ Includes JobManager injection for advanced operations and robust error handling. """ +import asyncio import functools import inspect import logging @@ -15,7 +16,7 @@ from mavedb.lib.slack import send_slack_error, send_slack_job_error, send_slack_job_failure from mavedb.lib.types.workflow import JobExecutionOutcome -from mavedb.models.enums.job_pipeline import JobStatus +from mavedb.models.enums.job_pipeline import FailureCategory, JobStatus from mavedb.worker.lib.decorators.utils import ensure_ctx, ensure_job_id, ensure_session_ctx, is_test_mode from mavedb.worker.lib.managers import JobManager from mavedb.worker.lib.managers.constants import TERMINAL_JOB_STATUSES @@ -143,6 +144,43 @@ async def _execute_managed_job(func: Callable[..., Awaitable[JobExecutionOutcome return result + # The coroutine is being cancelled — either ARQ hit job_timeout, or cleanup_stalled_jobs + # aborted this attempt (Job.abort()) as part of stall recovery. CancelledError is a + # BaseException, so the `except Exception` handler below never sees it; without this branch + # the JobRun row would be orphaned as RUNNING forever while the coroutine dies. + # + # We mark the row terminally (FAILED / TIMEOUT) so DB state matches reality, then RE-RAISE. + # Re-raising is essential: it lets ARQ record the cancellation as the job's result, which is + # what makes a concurrent Job.abort() confirm the job actually died. + except asyncio.CancelledError: + logger.warning(f"Job {job_id} cancelled (timeout or stall-recovery abort); marking FAILED before re-raising") + try: + db_session.rollback() + if job_manager is not None: + job_manager.fail_job( + result=JobExecutionOutcome.failed( + reason="Job cancelled due to timeout, stall-recovery, or internal ARQ abort", + data={"reason": "cancelled"}, + failure_category=FailureCategory.TIMEOUT, + ) + ) + db_session.commit() + job = job_manager.get_job() + send_slack_job_failure( + job_urn=job.urn, + job_function=job.job_function, + reason="Job cancelled due to timeout.", + failure_category=str(FailureCategory.TIMEOUT), + retry_count=job.retry_count, + max_retries=job.max_retries, + will_retry=False, + ) + + except Exception as inner_e: + logger.critical(f"Failed to mark cancelled job {job_id} as failed: {inner_e}") + send_slack_error(inner_e) + raise + except Exception as e: # Prioritize salvaging lifecycle state will_retry = False diff --git a/src/mavedb/worker/settings/worker.py b/src/mavedb/worker/settings/worker.py index 6a558e6b6..3b0ab4e89 100644 --- a/src/mavedb/worker/settings/worker.py +++ b/src/mavedb/worker/settings/worker.py @@ -25,7 +25,13 @@ # driver, enabling incremental migration of job functions without touching # the FastAPI layer. Once all jobs use async sessions, raise MAX_JOBS to 10+. MAX_JOBS = 2 -JOB_TIMEOUT_SECONDS = 3 * 60 * 60 # 3 hours — matches RUNNING_TIMEOUT_MINUTES (150 min) with buffer +# ARQ's hard coroutine-kill ceiling. Kept deliberately high (8h) so ARQ is the *last* resort: +# our own DB-driven sweeper (cleanup_stalled_jobs) should detect and recover stalls long before +# this fires, keeping the JobRun state machine in sync with reality. VEP fan-out over large allele +# sets can legitimately run for hours, so a low ceiling would kill healthy work. +# cleanup's RUNNING_TIMEOUT_MINUTES backstop sits ~30 min under this to try and sweep these jobs before +# ARQ's hard timeout fires. +JOB_TIMEOUT_SECONDS = 8 * 60 * 60 # 8 hours class ArqWorkerSettings: @@ -44,3 +50,7 @@ class ArqWorkerSettings: max_jobs = MAX_JOBS job_timeout = JOB_TIMEOUT_SECONDS + # Required for cleanup_stalled_jobs to abort a wedged RUNNING job's coroutine (via Job.abort()) + # before re-driving it. Without this, ARQ ignores abort requests and the sweeper cannot safely + # confirm a stalled job is dead before retrying it. + allow_abort_jobs = True diff --git a/tests/worker/jobs/system/test_cleanup.py b/tests/worker/jobs/system/test_cleanup.py index 522b6167d..104c3af6d 100644 --- a/tests/worker/jobs/system/test_cleanup.py +++ b/tests/worker/jobs/system/test_cleanup.py @@ -12,6 +12,7 @@ pytest.importorskip("arq") # Skip tests if arq is not installed +import asyncio from datetime import datetime, timedelta, timezone from unittest.mock import AsyncMock, patch @@ -29,6 +30,7 @@ from mavedb.worker.jobs.system.cleanup import ( PENDING_TIMEOUT_MINUTES, PIPELINE_STUCK_TIMEOUT_MINUTES, + PROGRESS_STALL_MINUTES, RUNNING_TIMEOUT_MINUTES, cleanup_stalled_jobs, ) @@ -40,17 +42,55 @@ @pytest.fixture def mock_arq_job_not_found(): - """Mock ArqJob.status() to return not_found, simulating a crashed-enqueue QUEUED job.""" + """Mock ArqJob for a crashed-enqueue QUEUED job: status() is not_found, and abort() confirms dead.""" with patch("mavedb.worker.jobs.system.cleanup.ArqJob") as mock_arq_job: mock_arq_job.return_value.status = AsyncMock(return_value=ArqJobStatus.not_found) + mock_arq_job.return_value.abort = AsyncMock(return_value=True) yield mock_arq_job @pytest.fixture def mock_arq_job_in_redis(): - """Mock ArqJob.status() to return queued, simulating a legitimately queued job in ARQ Redis.""" + """Mock ArqJob for a legitimately queued job: status() is queued, and abort() confirms dead.""" with patch("mavedb.worker.jobs.system.cleanup.ArqJob") as mock_arq_job: mock_arq_job.return_value.status = AsyncMock(return_value=ArqJobStatus.queued) + mock_arq_job.return_value.abort = AsyncMock(return_value=True) + yield mock_arq_job + + +@pytest.fixture +def mock_arq_abort_confirmed(): + """Mock ArqJob.abort() to return True — the stalled RUNNING attempt was alive and is now dead. + + This is the signal that makes it safe for the sweeper to re-drive a stalled RUNNING job. + """ + with patch("mavedb.worker.jobs.system.cleanup.ArqJob") as mock_arq_job: + mock_arq_job.return_value.status = AsyncMock(return_value=ArqJobStatus.not_found) + mock_arq_job.return_value.abort = AsyncMock(return_value=True) + yield mock_arq_job + + +@pytest.fixture +def mock_arq_abort_absent(): + """Mock ArqJob.abort() to return False — the job is absent from ARQ (crashed / never enqueued). + + Nothing is running, so it is still safe to re-drive. + """ + with patch("mavedb.worker.jobs.system.cleanup.ArqJob") as mock_arq_job: + mock_arq_job.return_value.status = AsyncMock(return_value=ArqJobStatus.not_found) + mock_arq_job.return_value.abort = AsyncMock(return_value=False) + yield mock_arq_job + + +@pytest.fixture +def mock_arq_abort_wedged(): + """Mock ArqJob.abort() to time out — the coroutine is wedged in a blocking call and won't die. + + The sweeper must NOT re-drive such a job (it could still be running), to avoid a double-run. + """ + with patch("mavedb.worker.jobs.system.cleanup.ArqJob") as mock_arq_job: + mock_arq_job.return_value.status = AsyncMock(return_value=ArqJobStatus.in_progress) + mock_arq_job.return_value.abort = AsyncMock(side_effect=asyncio.TimeoutError()) yield mock_arq_job @@ -148,7 +188,7 @@ async def test_cleanup_stalled_queued_job_max_retries_reached( assert "stalled" in stalled_job.error_message.lower() async def test_cleanup_stalled_running_job_with_retries( - self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job + self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job, mock_arq_abort_confirmed ): """Test cleanup of a stalled RUNNING job with retries remaining.""" # Create a stalled RUNNING job in the database @@ -184,7 +224,7 @@ async def test_cleanup_stalled_running_job_with_retries( assert stalled_job.finished_at is None async def test_cleanup_stalled_running_job_max_retries_reached( - self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job + self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job, mock_arq_abort_confirmed ): """Test cleanup of a stalled RUNNING job with max retries reached.""" # Create a stalled RUNNING job with max retries @@ -257,6 +297,115 @@ async def test_cleanup_stalled_running_job_missing_started_at( assert stalled_job.status == JobStatus.RUNNING assert stalled_job.retry_count == 0 + async def test_cleanup_detects_running_job_by_stale_heartbeat( + self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job, mock_arq_abort_confirmed + ): + """A RUNNING job with a recent started_at but a stale progress heartbeat is detected and re-driven. + + This is the primary staleness signal: staleness is judged by progress_updated_at, not total runtime. + """ + stalled_job = JobRun( + job_type="test_job", + job_function="test_function", + status=JobStatus.RUNNING, + created_at=datetime.now(timezone.utc) - timedelta(minutes=40), + started_at=datetime.now(timezone.utc) - timedelta(minutes=30), # well within the wall-clock backstop + progress_updated_at=datetime.now(timezone.utc) - timedelta(minutes=PROGRESS_STALL_MINUTES + 5), + finished_at=None, + max_retries=3, + retry_count=0, + job_params={}, + ) + session.add(stalled_job) + session.commit() + + result = await cleanup_stalled_jobs( + mock_worker_ctx, None, JobManager(session, mock_worker_ctx["redis"], sample_cleanup_job_run.id) + ) + + # Abort of the wedged attempt was confirmed, so the job was re-driven. + mock_arq_abort_confirmed.return_value.abort.assert_awaited_once() + mock_worker_ctx["redis"].enqueue_job.assert_called_once() + + assert result.data["total_cleaned"] == 1 + assert stalled_job.urn in result.data["running_jobs"] + + session.refresh(stalled_job) + assert stalled_job.status == JobStatus.QUEUED + assert stalled_job.retry_count == 1 + + async def test_cleanup_leaves_running_job_with_fresh_heartbeat( + self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job + ): + """A RUNNING job that has been running a long time but is still checkpointing progress is left alone. + + This is the VEP fan-out case: legitimate multi-hour runtime, well past the old 150-min limit, + but under the wall-clock backstop and with a fresh heartbeat — must NOT be treated as stalled. + """ + healthy_job = JobRun( + job_type="test_job", + job_function="test_function", + status=JobStatus.RUNNING, + created_at=datetime.now(timezone.utc) - timedelta(minutes=310), + started_at=datetime.now(timezone.utc) - timedelta(minutes=300), # 5h — past old limit, under backstop + progress_updated_at=datetime.now(timezone.utc) - timedelta(minutes=2), # fresh heartbeat + finished_at=None, + max_retries=3, + retry_count=0, + job_params={}, + ) + session.add(healthy_job) + session.commit() + + result = await cleanup_stalled_jobs( + mock_worker_ctx, None, JobManager(session, mock_worker_ctx["redis"], sample_cleanup_job_run.id) + ) + + assert result.data["total_cleaned"] == 0 + assert healthy_job.urn not in result.data["running_jobs"] + + session.refresh(healthy_job) + assert healthy_job.status == JobStatus.RUNNING + assert healthy_job.retry_count == 0 + + async def test_cleanup_does_not_redrive_wedged_running_job( + self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job, mock_arq_abort_wedged + ): + """A stalled RUNNING job whose abort does not confirm is left in place, not re-driven. + + Re-driving a job that may still be executing (wedged in a blocking sync call) would recreate + the historical double-run, so the sweeper takes no action and leaves it for a later sweep. + """ + wedged_job = JobRun( + job_type="test_job", + job_function="test_function", + status=JobStatus.RUNNING, + created_at=datetime.now(timezone.utc) - timedelta(minutes=40), + started_at=datetime.now(timezone.utc) - timedelta(minutes=30), + progress_updated_at=datetime.now(timezone.utc) - timedelta(minutes=PROGRESS_STALL_MINUTES + 5), + finished_at=None, + max_retries=3, + retry_count=0, + job_params={}, + ) + session.add(wedged_job) + session.commit() + + result = await cleanup_stalled_jobs( + mock_worker_ctx, None, JobManager(session, mock_worker_ctx["redis"], sample_cleanup_job_run.id) + ) + + # Abort was attempted but never confirmed, so no retry was enqueued. + mock_arq_abort_wedged.return_value.abort.assert_awaited_once() + mock_worker_ctx["redis"].enqueue_job.assert_not_called() + + assert result.data["total_cleaned"] == 0 + assert wedged_job.urn not in result.data["running_jobs"] + + session.refresh(wedged_job) + assert wedged_job.status == JobStatus.RUNNING + assert wedged_job.retry_count == 0 + async def test_cleanup_stalled_pending_job_with_retries( self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job ): @@ -477,7 +626,7 @@ async def test_cleanup_stalled_queued_standalone_job_enqueue_failure( assert "Failed to enqueue after stall recovery" in stalled_job.error_message async def test_cleanup_stalled_running_standalone_job_enqueue_failure( - self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job + self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job, mock_arq_abort_confirmed ): """Test that stalled standalone RUNNING job is marked FAILED if ARQ enqueue fails.""" @@ -559,7 +708,7 @@ async def test_cleanup_stalled_queued_pipeline_job_dependencies_satisfied( assert stalled_job.retry_count == 1 async def test_cleanup_stalled_running_pipeline_job_dependencies_satisfied( - self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job + self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job, mock_arq_abort_confirmed ): """Test that stalled pipeline RUNNING job with satisfied dependencies is enqueued.""" # Create a pipeline with all dependencies satisfied @@ -735,7 +884,7 @@ async def test_cleanup_stalled_queued_pipeline_job_dependencies_not_ready( assert stalled_job.retry_count == 1 async def test_cleanup_stalled_running_pipeline_job_dependencies_failed( - self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job + self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job, mock_arq_abort_confirmed ): """Test that stalled pipeline RUNNING job with failed dependencies is skipped.""" # Create a pipeline @@ -867,7 +1016,7 @@ async def test_cleanup_stalled_pending_pipeline_job_dependencies_failed( assert stalled_job.retry_count == 0 async def test_cleanup_stalled_running_pipeline_job_dependencies_not_ready( - self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job + self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job, mock_arq_abort_confirmed ): """Test that stalled pipeline RUNNING job with dependencies not ready is skipped.""" # Create a pipeline @@ -1249,7 +1398,7 @@ async def test_cleanup_sends_slack_when_max_retries_reached_queued_job( assert call_kwargs["max_retries"] == 3 async def test_cleanup_sends_slack_when_max_retries_reached_running_job( - self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job + self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job, mock_arq_abort_confirmed ): """Test that send_slack_job_failure is called when sweeper permanently fails a stalled RUNNING job.""" stalled_job = JobRun( diff --git a/tests/worker/lib/decorators/test_job_management.py b/tests/worker/lib/decorators/test_job_management.py index dd2e10ef4..fbfb370d8 100644 --- a/tests/worker/lib/decorators/test_job_management.py +++ b/tests/worker/lib/decorators/test_job_management.py @@ -16,7 +16,7 @@ from sqlalchemy import select from mavedb.lib.types.workflow import JobExecutionOutcome -from mavedb.models.enums.job_pipeline import JobStatus +from mavedb.models.enums.job_pipeline import FailureCategory, JobStatus from mavedb.models.job_run import JobRun from mavedb.worker.lib.decorators.job_management import with_job_management from mavedb.worker.lib.managers.exceptions import JobStateError @@ -133,6 +133,67 @@ async def sample_error(ctx: dict, job_id: int, job_manager: JobManager): mock_start_job.assert_called_once() mock_error_job.assert_called_once() + async def test_decorator_marks_failed_and_reraises_on_cancellation( + self, session, mock_worker_ctx, mock_job_manager + ): + """A CancelledError (ARQ timeout or stall-recovery abort) must mark the row FAILED then re-raise. + + Re-raising is what lets ARQ record the cancellation as the job result so a concurrent + Job.abort() can confirm the job actually died. Without this handler the row would be + orphaned as RUNNING (CancelledError is a BaseException, not caught by `except Exception`). + """ + + @with_job_management + async def sample_cancel(ctx: dict, job_id: int, job_manager: JobManager): + raise asyncio.CancelledError() + + with ( + patch("mavedb.worker.lib.decorators.job_management.JobManager") as mock_job_manager_class, + patch("mavedb.worker.lib.decorators.job_management.send_slack_job_failure") as mock_send_slack_job_failure, + patch.object(mock_job_manager, "start_job", return_value=None) as mock_start_job, + patch.object(mock_job_manager, "fail_job", return_value=None) as mock_fail_job, + ): + mock_job_manager_class.return_value = mock_job_manager + + with pytest.raises(asyncio.CancelledError): + await sample_cancel(mock_worker_ctx, 999) + + mock_start_job.assert_called_once() + mock_fail_job.assert_called_once() + # The row is failed with a TIMEOUT category. + outcome = mock_fail_job.call_args.kwargs["result"] + assert outcome.status == JobStatus.FAILED + assert outcome.failure_category == FailureCategory.TIMEOUT + # A permanent-failure alert is emitted for the cancellation. + mock_send_slack_job_failure.assert_called_once() + + async def test_decorator_reraises_cancellation_even_if_marking_failed_errors( + self, session, mock_worker_ctx, mock_job_manager + ): + """If marking the cancelled job FAILED itself raises, the decorator still re-raises CancelledError. + + The salvage path must never swallow the cancellation — ARQ needs it to record the result. + """ + + @with_job_management + async def sample_cancel(ctx: dict, job_id: int, job_manager: JobManager): + raise asyncio.CancelledError() + + with ( + patch("mavedb.worker.lib.decorators.job_management.JobManager") as mock_job_manager_class, + patch("mavedb.worker.lib.decorators.job_management.send_slack_error") as mock_send_slack_error, + patch("mavedb.worker.lib.decorators.job_management.send_slack_job_failure"), + patch.object(mock_job_manager, "start_job", return_value=None), + patch.object(mock_job_manager, "fail_job", side_effect=RuntimeError("db down")), + ): + mock_job_manager_class.return_value = mock_job_manager + + with pytest.raises(asyncio.CancelledError): + await sample_cancel(mock_worker_ctx, 999) + + # The inner failure is reported, and the original cancellation still propagates. + mock_send_slack_error.assert_called_once() + async def test_decorator_calls_start_job_and_skip_job_when_wrapped_function_returns_skipped_status( self, session, mock_worker_ctx, mock_job_manager ): diff --git a/tests/worker/lib/managers/test_job_manager.py b/tests/worker/lib/managers/test_job_manager.py index 980a30f02..84f709bf4 100644 --- a/tests/worker/lib/managers/test_job_manager.py +++ b/tests/worker/lib/managers/test_job_manager.py @@ -11,6 +11,7 @@ pytest.importorskip("arq") import re +from datetime import datetime, timedelta, timezone from unittest.mock import Mock, PropertyMock, patch from arq import ArqRedis @@ -1348,6 +1349,28 @@ def test_update_progress_success(self, session, arq_redis, with_populated_job_da assert job.progress_total == 100 assert job.progress_message == "Halfway done" + def test_update_progress_bumps_progress_updated_at_heartbeat( + self, session, arq_redis, with_populated_job_data, sample_job_run + ): + """Committing a progress update refreshes progress_updated_at via onupdate=now(). + + This is the liveness heartbeat the stalled-job sweeper reads: it must advance on every + committing progress checkpoint with no explicit code in the job. + """ + + manager = JobManager(session, arq_redis, sample_job_run.id) + + # Pin the heartbeat to a stale value so the bump is unambiguous. + stale = datetime.now(timezone.utc) - timedelta(hours=1) + job = session.execute(select(JobRun).where(JobRun.id == sample_job_run.id)).scalar_one() + job.progress_updated_at = stale + session.commit() + + manager.update_progress(10, 100, "tick", commit=True) # commit triggers the UPDATE + onupdate + + job = session.execute(select(JobRun).where(JobRun.id == sample_job_run.id)).scalar_one() + assert job.progress_updated_at > stale + def test_update_progress_success_does_not_overwrite_old_message_when_no_new_message_is_provided( self, session, arq_redis, with_populated_job_data, sample_job_run ): From 5751e9c2c455200ae68582d36a791d3f4042231c Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Sat, 18 Jul 2026 09:22:35 -0700 Subject: [PATCH 088/252] feat(cat-vrs): add co_encodes relation and convergent derivation Under a nucleotide assay the reverse-translation fan leaves synonymous "cousins" on the record: nt alleles in other projection groups that encode the same protein consequence as the measured change. They are distinct, unmeasured variants, not representations of the measured change, and were previously mislabelled as candidates/projections. - Cat-VRS: add the CO_ENCODES relation and carry cousins as members in the full-closure detail object (include_convergent=True); the VA-Spec subject (include_convergent=False) drops them, keeping only the measured change's coordinate partner and protein consequence. - variant detail: add the `convergent` derivation, held in lockstep with the co_encodes relation, so cousins are no longer labelled `candidate` (which now means only protein-assay reverse-translation ambiguity). - relation/derivation now key off projection_group to distinguish the measured change's coordinate partner (same group) from a cousin (a different group). --- src/mavedb/lib/cat_vrs.py | 107 ++++++++++++++++++++--- src/mavedb/lib/variant_detail.py | 66 +++++++++----- src/mavedb/view_models/variant_detail.py | 5 +- tests/lib/test_cat_vrs.py | 101 +++++++++++++++++++-- tests/lib/test_variant_detail.py | 22 ++++- 5 files changed, 254 insertions(+), 47 deletions(-) diff --git a/src/mavedb/lib/cat_vrs.py b/src/mavedb/lib/cat_vrs.py index 10f1920ec..b9ad6d39c 100644 --- a/src/mavedb/lib/cat_vrs.py +++ b/src/mavedb/lib/cat_vrs.py @@ -28,29 +28,32 @@ from sqlalchemy.orm import Session from mavedb.lib.alleles import get_live_record_allele_links -from mavedb.lib.annotation.util import vrs_object_from_mapped_variant from mavedb.lib.logging.context import logging_context +from mavedb.lib.vrs import vrs_object_from_mapped_variant from mavedb.models.allele import Allele from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.models.mapping_record_allele import MappingRecordAllele logger = logging.getLogger(__name__) -# MaveDB-namespaced relation codes, read as `member -> defining` (the star model, design D54). No -# off-the-shelf vocabulary fits, so these are namespaced now and raised with the GA4GH Cat-VRS group -# for standardization; `mappings` to a standard term can be added later without a shape change. +# MaveDB-namespaced relation codes, read as `member -> defining`. Namespaced for MaveDB now, +# TODO: Harmonize with GA4GH Cat-VRS group for standardization. _RELATION_SYSTEM = "https://mavedb.org/cat-vrs/relations" +# TODO: Verify Cat-VRS relation names with GA4GH Cat-VRS group; these are the MaveDB-namespaced codes we raised for standardization. class CatVrsRelation(str, Enum): """Relation of a member allele to the single defining (measured) allele.""" - # defining is nt; member is the same variant in other nt coordinates (genomic<->coding). Faithful. - COORDINATE_REPRESENTATION_OF = "coordinate_representation_of" # defining is protein; member is an nt allele (coding or genomic) that encodes it. Implied. ENCODES = "encodes" + # defining is nt; member is the same variant in other nt coordinates (genomic<->coding). Faithful. + COORDINATE_REPRESENTATION_OF = "coordinate_representation_of" # defining is nt; member is the protein consequence. Consequence, no independent score. TRANSLATION_OF = "translation_of" + # defining is nt; member is a *synonymous cousin* — an nt allele in a different projection group that + # encodes the same protein consequence as the measured change. + CO_ENCODES = "co_encodes" class CatVrsMode(str, Enum): @@ -73,25 +76,67 @@ class CategoricalVariantTransit: member_relations: dict[str, CatVrsRelation] -def _relation_for(defining_level: Optional[str], member_level: Optional[str]) -> Optional[CatVrsRelation]: +def is_convergent_cousin( + member_level: Optional[str], member_group: Optional[int], *, defining_group: Optional[int] +) -> bool: + """Whether a projection-mode member is a *synonymous cousin* of the measured change. + + Concretely: + - The defining (measured) allele is nt (genomic or cdna). + - The member is nt (genomic or cdna). + - The member is in a different projection group than the defining (measured) allele. + """ + return ( + member_level in _NUCLEOTIDE_LEVELS + and member_group is not None + and defining_group is not None + and member_group != defining_group + ) + + +def _relation_for( + defining_level: Optional[str], + member_level: Optional[str], + *, + defining_group: Optional[int] = None, + member_group: Optional[int] = None, +) -> Optional[CatVrsRelation]: """Derive the member -> defining relation from the two levels, or None for the defining itself. - Star model: every member relates to the single defining allele, so the relation depends only on - the (defining, member) level pair, never on a chain between members. + Star model: every member relates to the single defining allele, so the relation depends only on the + (defining, member) level pair — plus, in projection mode, on whether the nt member shares the measured + change's ``projection_group`` (its coordinate partner) or lives in another group (a synonymous cousin). """ - # Mode 2 — protein measured. Every nt member encodes it; the protein member is the defining. + # Protein measured. Every nt member encodes it, the protein member is the defining. Grouping is irrelevant. if defining_level == SequenceLevel.protein.value: return CatVrsRelation.ENCODES if member_level in _NUCLEOTIDE_LEVELS else None - # Mode 1 — nt measured (genomic or cdna). + # Nt measured (genomic or cdna). if member_level == SequenceLevel.protein.value: return CatVrsRelation.TRANSLATION_OF if member_level in _NUCLEOTIDE_LEVELS: + if is_convergent_cousin(member_level, member_group, defining_group=defining_group): + return CatVrsRelation.CO_ENCODES return CatVrsRelation.COORDINATE_REPRESENTATION_OF return None # pragma: no cover -- the levels are constrained by the DB and the mapping job +def _is_precise_projection_member( + member_level: Optional[str], member_group: Optional[int], *, defining_group: Optional[int] +) -> bool: + """In projection mode, whether a member is a *precise* representation of the measured change. + + True for the protein consequence (the apex, shared by the whole equivalence class) and the measured + change's own coordinate partner — the other member of its ``projection_group`` (a c↔g pair). False for + the synonymous cousins in other projection groups. + """ + if member_level == SequenceLevel.protein.value: + return True + + return not is_convergent_cousin(member_level, member_group, defining_group=defining_group) + + def _relation_concept(relation: CatVrsRelation) -> MappableConcept: """Wrap a MaveDB relation code as a Cat-VRS ``MappableConcept`` for the constraint.""" return MappableConcept( @@ -115,12 +160,30 @@ def _hydrate_vrs(allele: Allele) -> Optional[VrsAllele | CisPhasedBlock]: return variation -def build_categorical_variant(links: list[MappingRecordAllele], *, name: str) -> Optional[CategoricalVariantTransit]: +def build_categorical_variant( + links: list[MappingRecordAllele], *, name: str, include_convergent: bool = True +) -> Optional[CategoricalVariantTransit]: """Assemble a Cat-VRS ``CategoricalVariant`` from a variant's live allele links. ``links`` is the record-scoped live link set from ``get_live_record_allele_links`` — exactly one is authoritative (the measured/defining allele), the rest are derived members. Returns ``None`` when there is no authoritative, hydratable link to anchor on (e.g. an unmapped variant). + + **The categorical variant always anchors on the measured allele**. Member selection therefore differs by mode: + + - **Reverse translation** (protein measured): the defining allele is the protein change and every + nt allele in the record ``encodes`` it, so the **full equivalence class** is unfurled — that class + is exactly what the protein measurement's claim ranges over. ``include_convergent`` is inert here. + - **Projection** (nt measured): the record also carries the reverse-translation fan — the synonymous + *cousins* (other nt encoders of the same protein consequence, in different projection groups), which + are distinct, unmeasured variants rather than representations of the measured change. + ``include_convergent`` selects how they are handled: + + - ``True`` (default; the detail envelope): the cousins are kept as members wearing the + :attr:`CatVrsRelation.CO_ENCODES` relation, so the object is the full closure. The measured change's + coordinate partner and protein consequence stay ``coordinate_representation_of`` / ``translation_of``. + - ``False`` (the VA-Spec subject): the cousins are dropped (see :func:`_is_precise_projection_member`) + and only the measured change's precise coordinate partner and protein consequence remain. """ defining_link = next((link for link in links if link.is_authoritative), None) if defining_link is None: @@ -152,6 +215,19 @@ def build_categorical_variant(links: list[MappingRecordAllele], *, name: str) -> continue allele = link.allele + + # Projection mode, narrow object (include_convergent=False): drop any synonymous cousins the + # reverse-translation fan left on the record, keeping only the measured change's precise coordinate + # partner and its protein consequence. + if ( + mode is CatVrsMode.PROJECTION + and not include_convergent + and not _is_precise_projection_member( + allele.level, link.projection_group, defining_group=defining_link.projection_group + ) + ): + continue + member_vrs = _hydrate_vrs(allele) # A live link to an allele with no post_mapped is an unexpected data state. if member_vrs is None: @@ -165,7 +241,12 @@ def build_categorical_variant(links: list[MappingRecordAllele], *, name: str) -> continue members.append(member_vrs) - relation = _relation_for(defining_level, allele.level) + relation = _relation_for( + defining_level, + allele.level, + defining_group=defining_link.projection_group, + member_group=link.projection_group, + ) if relation is not None and allele.vrs_digest is not None: member_relations[allele.vrs_digest] = relation relations_present[relation] = None diff --git a/src/mavedb/lib/variant_detail.py b/src/mavedb/lib/variant_detail.py index 987e0e3ea..8f06c7cc5 100644 --- a/src/mavedb/lib/variant_detail.py +++ b/src/mavedb/lib/variant_detail.py @@ -26,7 +26,7 @@ from mavedb.lib.allele_annotations import AlleleAnnotations, get_allele_annotations from mavedb.lib.alleles import get_live_record_allele_links -from mavedb.lib.cat_vrs import categorical_variant_for_variant +from mavedb.lib.cat_vrs import categorical_variant_for_variant, is_convergent_cousin from mavedb.lib.score_calibrations import calibration_preference_key from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.models.mapping_record import MappingRecord @@ -65,12 +65,18 @@ class AlleleDerivation(str, Enum): # The assay's actual measurement (the authoritative link). Precise by definition. AUTHORITATIVE = "authoritative" - # Deterministic and precise. Every allele derived from a *nucleotide* measurement is a projection: - # nucleotide↔nucleotide and nucleotide→protein are both deterministic given (assembly, transcript). + # Deterministic and precise. Derived from the measured change itself: nucleotide↔nucleotide (its + # coordinate partner) and nucleotide→protein (its consequence) are both deterministic given + # (assembly, transcript). PROJECTION = "projection" # Reverse-translation output of a *protein* measurement — genuinely ambiguous (many synonymous # codons). One member of the fanned-out equivalence class, not a precise coordinate. CANDIDATE = "candidate" + # A distinct, precisely-known nucleotide change that *converges* on the measured protein consequence + # from a different codon (a synonymous cousin under a nucleotide assay). Not ambiguous like a + # candidate, and not a projection *of* the measured change — a separate, unmeasured variant that + # merely shares the consequence. Pairs with the ``co_encodes`` Cat-VRS relation. + CONVERGENT = "convergent" @dataclass(frozen=True) @@ -87,10 +93,11 @@ class AlleleIdentity: (``is_protein_of`` / ``coordinate_representation_of`` / …). ``None`` when it *is* the measured allele, or when the allele is not a Cat-VRS member. Sourced from ``cat_vrs.CategoricalVariantTransit.member_relations``. - - ``derivation`` (provenance): :class:`AlleleDerivation` — authoritative / projection / - candidate. Orthogonal to ``relation``; never conflate the two (a protein member of a nucleotide - assay has ``relation=translation_of`` but ``derivation=projection``, whereas the same-shaped - member of a protein assay is ``derivation=candidate``). + - ``derivation`` (provenance): :class:`AlleleDerivation` — authoritative / projection / candidate / + convergent. Orthogonal to ``relation``; never conflate the two (a protein member of a nucleotide + assay has ``relation=translation_of`` but ``derivation=projection``; a synonymous cousin has + ``relation=co_encodes`` and ``derivation=convergent``; the nucleotide fan-out of a protein assay is + ``derivation=candidate``). - ``projection_of`` (provenance): the VRS digest of this allele's projection sibling — the ≤1 *other* member of its ``projection_group`` (a c↔g pair). ``None`` for the protein apex (group ``NULL``), for pre-reverse-translation data, and where a level's projection failed. @@ -170,26 +177,34 @@ def _classifications_for_variant( ] -def _derivation_for(*, is_authoritative: bool, assay_level: Optional[SequenceLevel]) -> AlleleDerivation: +def _derivation_for( + *, is_authoritative: bool, assay_level: Optional[SequenceLevel], is_cousin: bool +) -> AlleleDerivation: """The provenance of a linked allele's representation, from ``is_authoritative`` + the assay level. - The measured allele is ``authoritative``. Every *other* allele's confidence is set by the *only* - ambiguous boundary in the stack — protein → codon: - - - **nucleotide assay** (``assay_level`` genomic/cdna) → ``projection`` for all derived alleles. - nucleotide↔nucleotide and nucleotide→protein are deterministic, so the whole equivalence class - is precise. - - **protein assay** (``assay_level`` protein) → ``candidate`` for the derived (nucleotide) fan-out. - Reverse translation is genuinely ambiguous. - - Only ``assay_level`` (not the allele's own level) distinguishes the two, because on a nucleotide - assay every derived level is deterministic and on a protein assay every derived allele is part of - the ambiguous nucleotide fan-out. + The measured allele is ``authoritative``. Every *other* allele's provenance turns on the protein → + codon boundary, which surfaces three ways: + + - **protein assay** (``assay_level`` protein) → ``candidate`` for the whole derived (nucleotide) + fan-out: reverse translation is genuinely ambiguous (which codon was measured is unknown). + - **nucleotide assay** (``assay_level`` genomic/cdna), synonymous *cousin* (``is_cousin``) → an nt + member in a different projection group: a distinct, precisely-known variant that merely + *converges* on the measured protein consequence. It is ``convergent`` — not ambiguous (so not a + ``candidate``) and not a projection *of* the measured change. + - **nucleotide assay**, otherwise → ``projection`` for the precise class derived from the measured + change itself (its coordinate partner and its protein consequence, both deterministic). + + ``is_cousin`` is :func:`cat_vrs.is_convergent_cousin` for this allele, keeping the ``derivation`` + (provenance) axis in lockstep with the Cat-VRS ``relation`` axis — ``co_encodes`` ↔ ``convergent``, + ``coordinate_representation_of`` / ``translation_of`` ↔ ``projection``, protein-assay ``encodes`` ↔ + ``candidate``. """ if is_authoritative: return AlleleDerivation.AUTHORITATIVE if assay_level == SequenceLevel.protein.value: return AlleleDerivation.CANDIDATE + if is_cousin: + return AlleleDerivation.CONVERGENT return AlleleDerivation.PROJECTION @@ -263,11 +278,13 @@ def get_variant_detail( if link.projection_group is not None and link.allele.vrs_digest is not None: group_digests.setdefault(link.projection_group, []).append(link.allele.vrs_digest) + # The measured change's projection group anchors the cousin test: an nt member in a *different* group + # is a synonymous cousin (co_encodes / convergent), not the measured change's coordinate partner. + defining_group = authoritative.projection_group if authoritative is not None else None + alleles: dict[str, AlleleIdentity] = {} for link in links: allele = link.allele - if allele.vrs_digest is None: - continue # The projection sibling: the other digest in this link's projection_group, if any. A group has # ≤2 members, so there is at most one sibling — this allele's projection partner. @@ -279,12 +296,15 @@ def get_variant_detail( ) relation = relations.get(allele.vrs_digest) + is_cousin = is_convergent_cousin(allele.level, link.projection_group, defining_group=defining_group) alleles[allele.vrs_digest] = AlleleIdentity( level=allele.level, hgvs=allele.hgvs_g or allele.hgvs_c or allele.hgvs_p, clingen_allele_id=allele.clingen_allele_id, relation=relation.value if relation is not None else None, - derivation=_derivation_for(is_authoritative=link.is_authoritative, assay_level=assay_level).value, + derivation=_derivation_for( + is_authoritative=link.is_authoritative, assay_level=assay_level, is_cousin=is_cousin + ).value, projection_of=projection_of, ) diff --git a/src/mavedb/view_models/variant_detail.py b/src/mavedb/view_models/variant_detail.py index 190f41d7a..31f5518aa 100644 --- a/src/mavedb/view_models/variant_detail.py +++ b/src/mavedb/view_models/variant_detail.py @@ -40,8 +40,9 @@ class AlleleIdentity(BaseModel): - ``relation`` (Cat-VRS, structural): member→defining relation; ``null`` when it *is* the measured allele, or when the allele is not a Cat-VRS member. - ``derivation`` (provenance): ``authoritative`` (measured) / ``projection`` (deterministic, - precise) / ``candidate`` (reverse-translation, ambiguous). Orthogonal to ``relation`` — never - conflate them. + precise, derived from the measured change) / ``candidate`` (protein-assay reverse-translation, + ambiguous) / ``convergent`` (a distinct, precise nucleotide change that converges on the measured + protein consequence). Orthogonal to ``relation``. Do not conflate them. - ``projectionOf`` (provenance): the VRS digest of this allele's projection sibling (the paired c↔g member of its projection pair group); ``null`` for the protein apex and pre-reverse-translation data. """ diff --git a/tests/lib/test_cat_vrs.py b/tests/lib/test_cat_vrs.py index ba6f54e47..47eb6ac64 100644 --- a/tests/lib/test_cat_vrs.py +++ b/tests/lib/test_cat_vrs.py @@ -57,16 +57,20 @@ def _post_mapped() -> dict: _DEFAULT_POST_MAPPED = object() -def _link(*, level: str, digest: str, is_authoritative: bool, post_mapped=_DEFAULT_POST_MAPPED) -> MappingRecordAllele: +def _link( + *, level: str, digest: str, is_authoritative: bool, post_mapped=_DEFAULT_POST_MAPPED, projection_group=None +) -> MappingRecordAllele: """A transient (record, allele) link with its allele attached — no session needed. ``digest`` is the ``Allele.vrs_digest`` *column* (the key the builder uses for member relations), deliberately distinct from the spec-valid VRS digest embedded in post_mapped. Pass - ``post_mapped=None`` to model an un-hydratable allele. + ``post_mapped=None`` to model an un-hydratable allele. ``projection_group`` pairs a c↔g projection + (the two links of one precise change share a value; the protein apex carries ``None``); the builder + uses it in projection mode to keep only the measured change's precise coordinate partner. """ pm = _post_mapped() if post_mapped is _DEFAULT_POST_MAPPED else post_mapped allele = Allele(level=level, vrs_digest=digest, post_mapped=pm) - return MappingRecordAllele(is_authoritative=is_authoritative, allele=allele) + return MappingRecordAllele(is_authoritative=is_authoritative, allele=allele, projection_group=projection_group) @pytest.mark.unit @@ -101,10 +105,10 @@ def test_mode_2_protein_measured_reverse_translation(): @pytest.mark.unit def test_mode_1_coding_measured_projection(): - """Coding measured: nt sibling is a coordinate representation; protein member is a translation.""" + """Coding measured: the projection_group partner is a coordinate representation; protein is a translation.""" links = [ - _link(level="cdna", digest="cdna", is_authoritative=True), - _link(level="genomic", digest="gen", is_authoritative=False), + _link(level="cdna", digest="cdna", is_authoritative=True, projection_group=0), + _link(level="genomic", digest="gen", is_authoritative=False, projection_group=0), _link(level="protein", digest="prot", is_authoritative=False), ] @@ -122,6 +126,91 @@ def test_mode_1_coding_measured_projection(): assert codes == {"coordinate_representation_of", "translation_of"} +@pytest.mark.unit +def test_mode_1_projection_includes_sibling_encoders(): + """Coding measured, full closure (default include_convergent=True): the reverse-translation fan on the + record (other encoders of the same protein consequence, in *different* projection groups) is kept as + members wearing the ``co_encodes`` relation, so the object is the full closure. The measured change's + own coordinate partner and protein consequence stay coordinate_representation_of / translation_of.""" + links = [ + _link(level="cdna", digest="cdna", is_authoritative=True, projection_group=0), + _link(level="genomic", digest="gen", is_authoritative=False, projection_group=0), + # A sibling encoder of the same protein change — a distinct variant, different projection group. + _link(level="cdna", digest="sibling_cdna", is_authoritative=False, projection_group=1), + _link(level="genomic", digest="sibling_gen", is_authoritative=False, projection_group=1), + _link(level="protein", digest="prot", is_authoritative=False), + ] + + transit = build_categorical_variant(links, name="urn:mavedb:test#2b") + assert transit is not None + + # The coordinate partner + protein consequence keep their faithful relations; the two cousins ride as + # co_encodes (distinct, unmeasured synonymous variants). + assert transit.member_relations == { + "gen": CatVrsRelation.COORDINATE_REPRESENTATION_OF, + "prot": CatVrsRelation.TRANSLATION_OF, + "sibling_cdna": CatVrsRelation.CO_ENCODES, + "sibling_gen": CatVrsRelation.CO_ENCODES, + } + # members = defining cdna + gen partner + protein apex + the two cousins (full closure). + assert len(transit.categorical_variant.members) == 5 + # The distinct relation kinds surface on the constraint, including co_encodes. + constraint = transit.categorical_variant.constraints[0].root + codes = {str(r.primaryCoding.code.root) for r in constraint.relations} + assert codes == {"coordinate_representation_of", "translation_of", "co_encodes"} + + +@pytest.mark.unit +def test_mode_1_projection_narrow_object_drops_sibling_encoders(): + """Coding measured, narrow object (include_convergent=False, the VA subject): the synonymous cousins + in other projection groups are dropped — only the measured change's coordinate partner and protein + consequence remain. This pins the shape the VA-Spec path builds.""" + links = [ + _link(level="cdna", digest="cdna", is_authoritative=True, projection_group=0), + _link(level="genomic", digest="gen", is_authoritative=False, projection_group=0), + _link(level="cdna", digest="sibling_cdna", is_authoritative=False, projection_group=1), + _link(level="genomic", digest="sibling_gen", is_authoritative=False, projection_group=1), + _link(level="protein", digest="prot", is_authoritative=False), + ] + + transit = build_categorical_variant(links, name="urn:mavedb:test#2b-narrow", include_convergent=False) + assert transit is not None + + # Only the measured change's coordinate partner + the protein consequence; the cousins are excluded. + assert transit.member_relations == { + "gen": CatVrsRelation.COORDINATE_REPRESENTATION_OF, + "prot": CatVrsRelation.TRANSLATION_OF, + } + # members = defining cdna + gen partner + protein apex (the two cousins are dropped). + assert len(transit.categorical_variant.members) == 3 + + +@pytest.mark.unit +def test_mode_2_reverse_translation_keeps_the_full_encoder_class(): + """Protein measured: every nt encoder stays, across projection groups — the full equivalence class + is what the protein claim ranges over (no sibling filtering in reverse-translation mode).""" + links = [ + _link(level="protein", digest="prot", is_authoritative=True), + _link(level="cdna", digest="cdna_a", is_authoritative=False, projection_group=0), + _link(level="genomic", digest="gen_a", is_authoritative=False, projection_group=0), + _link(level="cdna", digest="cdna_b", is_authoritative=False, projection_group=1), + _link(level="genomic", digest="gen_b", is_authoritative=False, projection_group=1), + ] + + transit = build_categorical_variant(links, name="urn:mavedb:test#2c") + assert transit is not None + + assert transit.mode == CatVrsMode.REVERSE_TRANSLATION + # All four nt encoders `encodes` the defining protein; none dropped. + assert transit.member_relations == { + "cdna_a": CatVrsRelation.ENCODES, + "gen_a": CatVrsRelation.ENCODES, + "cdna_b": CatVrsRelation.ENCODES, + "gen_b": CatVrsRelation.ENCODES, + } + assert len(transit.categorical_variant.members) == 5 + + @pytest.mark.unit def test_no_authoritative_link_returns_none(): """An unmapped variant (no authoritative link) has no defining allele to anchor on.""" diff --git a/tests/lib/test_variant_detail.py b/tests/lib/test_variant_detail.py index fa193b53d..4c8a3384c 100644 --- a/tests/lib/test_variant_detail.py +++ b/tests/lib/test_variant_detail.py @@ -150,11 +150,15 @@ def test_full_envelope_for_a_coding_assay(session, setup_lib_db_with_score_set): measured = _allele(session, "cdna-digest", level="cdna", clingen_allele_id="CA123", hgvs_c="NM_000546.6:c.1216G>A") genomic = _allele(session, "gen-digest", level="genomic", hgvs_g="NC_000017.11:g.7676154C>T") protein = _allele(session, "prot-digest", level="protein", hgvs_p="NP_000537.3:p.Ala406Thr") + # A synonymous cousin: an nt encoder of the same protein consequence in a *different* projection group + # — a distinct, unmeasured variant carried by the reverse-translation fan. + cousin = _allele(session, "cousin-digest", level="cdna", hgvs_c="NM_000546.6:c.1218C>T") # The measured cdna link and its genomic projection share a projection_group (the RT fold-in); the - # protein apex is in no pair (group None). + # protein apex is in no pair (group None); the cousin is in its own group. _link(session, record, measured, is_authoritative=True, projection_group=0) _link(session, record, genomic, projection_group=0) _link(session, record, protein) + _link(session, record, cousin, projection_group=1) _vep(session, measured, "missense_variant") detail = get_variant_detail(session, variant) @@ -169,10 +173,13 @@ def test_full_envelope_for_a_coding_assay(session, setup_lib_db_with_score_set): assert detail.mode == "projection" assert detail.molecular_representation is not None assert detail.molecular_representation["type"] == "CategoricalVariant" + # State A: the full-closure Cat-VRS member set agrees with the sidecar — every linked allele is a + # member, and vice versa (the cousin included). + assert len(detail.molecular_representation["members"]) == len(detail.alleles) # The alleles sidecar carries one identity per linked allele, keyed by digest: level + # reference-frame HGVS (coalesced from hgvs_g/c/p) + ClinGen id + member->defining relation + # derivation + projection_of. - assert set(detail.alleles) == {"cdna-digest", "gen-digest", "prot-digest"} + assert set(detail.alleles) == {"cdna-digest", "gen-digest", "prot-digest", "cousin-digest"} measured_identity = detail.alleles["cdna-digest"] assert measured_identity.level == "cdna" assert measured_identity.hgvs == "NM_000546.6:c.1216G>A" # coalesced from hgvs_c @@ -195,8 +202,17 @@ def test_full_envelope_for_a_coding_assay(session, setup_lib_db_with_score_set): # projection sibling (the apex is in no c/g pair). assert protein_identity.derivation == "projection" assert protein_identity.projection_of is None + # The synonymous cousin (different projection group) is a member wearing co_encodes (structural), and + # labelled `convergent` (provenance) — a distinct, precise change that converges on the measured + # consequence, not an ambiguous candidate and not a projection of the measured change. It pairs with + # nothing (its group has a single member). + cousin_identity = detail.alleles["cousin-digest"] + assert cousin_identity.level == "cdna" + assert cousin_identity.relation == "co_encodes" + assert cousin_identity.derivation == "convergent" + assert cousin_identity.projection_of is None # Annotations keyed by digest, covering all linked alleles; VEP rode in on the measured allele. - assert set(detail.annotations) == {"cdna-digest", "gen-digest", "prot-digest"} + assert set(detail.annotations) == {"cdna-digest", "gen-digest", "prot-digest", "cousin-digest"} assert detail.annotations["cdna-digest"].vep is not None assert detail.annotations["cdna-digest"].vep.consequence == "missense_variant" assert detail.is_current is True From 28f90ea6d292f69c3cb843109d9f5a834f78c883 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Sat, 18 Jul 2026 09:51:13 -0700 Subject: [PATCH 089/252] refactor(annotation): dissolve util.py into cohesive modules The annotation util.py had grown into an unrelated grab-bag. Split it so each helper lives with the concept it serves. - Extract VRS rehydration into lib/vrs.py, the annotation eligibility predicates into eligibility.py, calibration selection into calibration.py, and the VariantAnnotationContext builder into context.py. - Co-locate sequence_feature_for_variant in proposition.py, its sole consumer. - Repoint the importers (annotate, classification, document, evidence_line, proposition, statement, study_result) and split the tests to match the new module homes, with patch targets now pointing at the consuming module rather than the old util module. --- src/mavedb/lib/annotation/annotate.py | 105 +-- src/mavedb/lib/annotation/calibration.py | 177 ++++ src/mavedb/lib/annotation/classification.py | 24 +- src/mavedb/lib/annotation/context.py | 80 ++ src/mavedb/lib/annotation/document.py | 31 +- src/mavedb/lib/annotation/eligibility.py | 128 +++ src/mavedb/lib/annotation/evidence_line.py | 18 +- src/mavedb/lib/annotation/proposition.py | 81 +- src/mavedb/lib/annotation/statement.py | 26 +- src/mavedb/lib/annotation/study_result.py | 33 +- src/mavedb/lib/annotation/util.py | 504 ------------ src/mavedb/lib/vrs.py | 134 +++ tests/lib/annotation/conftest.py | 52 ++ tests/lib/annotation/test_annotate.py | 151 ++-- tests/lib/annotation/test_calibration.py | 344 ++++++++ tests/lib/annotation/test_classification.py | 36 +- tests/lib/annotation/test_context.py | 108 +++ tests/lib/annotation/test_document.py | 44 +- tests/lib/annotation/test_eligibility.py | 375 +++++++++ tests/lib/annotation/test_evidence_line.py | 83 +- tests/lib/annotation/test_proposition.py | 133 ++- tests/lib/annotation/test_statement.py | 54 +- tests/lib/annotation/test_study_result.py | 34 +- tests/lib/annotation/test_util.py | 868 -------------------- tests/lib/test_vrs.py | 96 +++ 25 files changed, 1993 insertions(+), 1726 deletions(-) create mode 100644 src/mavedb/lib/annotation/calibration.py create mode 100644 src/mavedb/lib/annotation/context.py create mode 100644 src/mavedb/lib/annotation/eligibility.py delete mode 100644 src/mavedb/lib/annotation/util.py create mode 100644 src/mavedb/lib/vrs.py create mode 100644 tests/lib/annotation/test_calibration.py create mode 100644 tests/lib/annotation/test_context.py create mode 100644 tests/lib/annotation/test_eligibility.py delete mode 100644 tests/lib/annotation/test_util.py create mode 100644 tests/lib/test_vrs.py diff --git a/src/mavedb/lib/annotation/annotate.py b/src/mavedb/lib/annotation/annotate.py index e5e289a39..e6676f18b 100644 --- a/src/mavedb/lib/annotation/annotate.py +++ b/src/mavedb/lib/annotation/annotate.py @@ -1,11 +1,15 @@ """ -This module supports the construction of three main VA-Spec data structures based on the MaveDB MappedVariant object: +This module supports the construction of three main VA-Spec data structures from a variant's +:class:`VariantAnnotationContext` (the new mapping-record substrate — see ``context.py``): - StudyResult See: https://va-ga4gh.readthedocs.io/en/latest/modeling-foundations/data-structures.html#study-result-structure - Statement See: https://va-ga4gh.readthedocs.io/en/latest/modeling-foundations/data-structures.html#statement-structure - VariantPathogenicityStatement See: https://va-spec.ga4gh.org/en/latest/va-standard-profiles/community-profiles/acmg-2015-profiles.html#variant-pathogenicity-statement-acmg-2015 + +Callers build the context at the edge (router/script) via ``variant_annotation_context`` and pass it in; +these builders never touch the database. ``as_of`` selection and supersession are baked into the context. """ from typing import Optional, Union @@ -13,46 +17,45 @@ from ga4gh.va_spec.acmg_2015 import VariantPathogenicityStatement from ga4gh.va_spec.base.core import ExperimentalVariantFunctionalImpactStudyResult, Statement -from mavedb.lib.annotation.classification import functional_classification_of_variant -from mavedb.lib.annotation.exceptions import MappingDataDoesntExistException -from mavedb.lib.annotation.evidence_line import acmg_evidence_line, functional_evidence_line -from mavedb.lib.annotation.proposition import ( - mapped_variant_to_experimental_variant_clinical_impact_proposition, - mapped_variant_to_experimental_variant_functional_impact_proposition, -) -from mavedb.lib.annotation.statement import ( - mapped_variant_to_functional_statement, - mapped_variant_to_pathogenicity_statement, -) -from mavedb.lib.annotation.study_result import mapped_variant_to_experimental_variant_impact_study_result -from mavedb.lib.annotation.util import ( - can_annotate_variant_for_functional_statement, - can_annotate_variant_for_pathogenicity_evidence, +from mavedb.lib.annotation.calibration import ( score_calibration_may_be_used_for_annotation, select_strongest_functional_calibration, select_strongest_pathogenicity_calibration, ) -from mavedb.models.mapped_variant import MappedVariant +from mavedb.lib.annotation.classification import functional_classification_of_variant +from mavedb.lib.annotation.context import VariantAnnotationContext +from mavedb.lib.annotation.eligibility import ( + can_annotate_variant_for_functional_statement, + can_annotate_variant_for_pathogenicity_evidence, +) +from mavedb.lib.annotation.evidence_line import acmg_evidence_line, functional_evidence_line +from mavedb.lib.annotation.exceptions import MappingDataDoesntExistException +from mavedb.lib.annotation.proposition import ( + variant_functional_impact_proposition, + variant_pathogenicity_proposition, +) +from mavedb.lib.annotation.statement import functional_statement, pathogenicity_statement +from mavedb.lib.annotation.study_result import variant_impact_study_result -def variant_study_result(mapped_variant: MappedVariant) -> ExperimentalVariantFunctionalImpactStudyResult: - return mapped_variant_to_experimental_variant_impact_study_result(mapped_variant) +def variant_study_result(context: VariantAnnotationContext) -> ExperimentalVariantFunctionalImpactStudyResult: + return variant_impact_study_result(context) def variant_functional_impact_statement( - mapped_variant: MappedVariant, allow_research_use_only_calibrations: bool = False + context: VariantAnnotationContext, allow_research_use_only_calibrations: bool = False ) -> Optional[Statement]: if not can_annotate_variant_for_functional_statement( - mapped_variant, allow_research_use_only_calibrations=allow_research_use_only_calibrations + context.variant, allow_research_use_only_calibrations=allow_research_use_only_calibrations ): return None - study_result = mapped_variant_to_experimental_variant_impact_study_result(mapped_variant) - functional_proposition = mapped_variant_to_experimental_variant_functional_impact_proposition(mapped_variant) + study_result = variant_impact_study_result(context) + functional_proposition = variant_functional_impact_proposition(context) # Collect eligible calibrations eligible_calibrations = [] - for score_calibration in mapped_variant.variant.score_set.score_calibrations: + for score_calibration in context.variant.score_set.score_calibrations: if score_calibration_may_be_used_for_annotation( score_calibration, annotation_type="functional", @@ -62,7 +65,7 @@ def variant_functional_impact_statement( # Select the calibration with the strongest evidence strongest_calibration, strongest_range = select_strongest_functional_calibration( - mapped_variant, eligible_calibrations + context.variant, eligible_calibrations ) if not strongest_calibration: @@ -70,33 +73,33 @@ def variant_functional_impact_statement( # Get the classification from the strongest range # If strongest_range is None, the variant is not in any range, so classification will be INDETERMINATE - _, classification = functional_classification_of_variant(mapped_variant, strongest_calibration) + _, classification = functional_classification_of_variant(context.variant, strongest_calibration) # Build evidence lines for all eligible calibrations functional_evidence = [] for score_calibration in eligible_calibrations: - functional_evidence.append(functional_evidence_line(mapped_variant, score_calibration, [study_result])) + functional_evidence.append(functional_evidence_line(context, score_calibration, [study_result])) - return mapped_variant_to_functional_statement( - mapped_variant, functional_proposition, functional_evidence, strongest_calibration, classification + return functional_statement( + context, functional_proposition, functional_evidence, strongest_calibration, classification ) def variant_pathogenicity_statement( - mapped_variant: MappedVariant, allow_research_use_only_calibrations: bool = False + context: VariantAnnotationContext, allow_research_use_only_calibrations: bool = False ) -> Optional[VariantPathogenicityStatement]: if not can_annotate_variant_for_pathogenicity_evidence( - mapped_variant, allow_research_use_only_calibrations=allow_research_use_only_calibrations + context.variant, allow_research_use_only_calibrations=allow_research_use_only_calibrations ): return None - study_result = mapped_variant_to_experimental_variant_impact_study_result(mapped_variant) - functional_proposition = mapped_variant_to_experimental_variant_functional_impact_proposition(mapped_variant) - clinical_proposition = mapped_variant_to_experimental_variant_clinical_impact_proposition(mapped_variant) + study_result = variant_impact_study_result(context) + functional_proposition = variant_functional_impact_proposition(context) + clinical_proposition = variant_pathogenicity_proposition(context) # Collect eligible calibrations eligible_calibrations = [] - for score_calibration in mapped_variant.variant.score_set.score_calibrations: + for score_calibration in context.variant.score_set.score_calibrations: if score_calibration_may_be_used_for_annotation( score_calibration, annotation_type="pathogenicity", @@ -106,7 +109,7 @@ def variant_pathogenicity_statement( # Select the calibration with the strongest evidence strongest_calibration, strongest_range = select_strongest_pathogenicity_calibration( - mapped_variant, eligible_calibrations + context.variant, eligible_calibrations ) if not strongest_calibration: @@ -114,7 +117,7 @@ def variant_pathogenicity_statement( # Get the classification from the strongest range (used for the functional statement within clinical evidence) # If strongest_range is None, the variant is not in any range, so classification will be INDETERMINATE - _, classification = functional_classification_of_variant(mapped_variant, strongest_calibration) + _, classification = functional_classification_of_variant(context.variant, strongest_calibration) # Note: strongest_range is used in the pathogenicity statement for ACMG classification # If None, the statement will use UNCERTAIN_SIGNIFICANCE @@ -122,33 +125,33 @@ def variant_pathogenicity_statement( # Build evidence lines for all eligible calibrations clinical_evidence = [] for score_calibration in eligible_calibrations: - functional_evidence = functional_evidence_line(mapped_variant, score_calibration, [study_result]) - functional_statement = mapped_variant_to_functional_statement( - mapped_variant, functional_proposition, [functional_evidence], score_calibration, classification + functional_evidence = functional_evidence_line(context, score_calibration, [study_result]) + functional_impact_statement = functional_statement( + context, functional_proposition, [functional_evidence], score_calibration, classification ) clinical_evidence.append( - acmg_evidence_line(mapped_variant, score_calibration, clinical_proposition, [functional_statement]) + acmg_evidence_line(context, score_calibration, clinical_proposition, [functional_impact_statement]) ) - return mapped_variant_to_pathogenicity_statement( - mapped_variant, clinical_proposition, clinical_evidence, strongest_calibration, strongest_range + return pathogenicity_statement( + context, clinical_proposition, clinical_evidence, strongest_calibration, strongest_range ) def variant_highest_level_annotation( - mapped_variant: MappedVariant, + context: VariantAnnotationContext, ) -> Optional[Union[ExperimentalVariantFunctionalImpactStudyResult, Statement, VariantPathogenicityStatement]]: """ - Build the single highest-materialized VA-Spec layer for a mapped variant. + Build the single highest-materialized VA-Spec layer for a variant's annotation context. Layer ladder (highest to lowest): pathogenicity statement -> functional impact statement -> study result. - Returns None when the variant has no post-mapped allele and therefore cannot be annotated. + Returns None when the variant lacks the target/mapping data required to build any layer. """ try: - if can_annotate_variant_for_pathogenicity_evidence(mapped_variant): - return variant_pathogenicity_statement(mapped_variant) - if can_annotate_variant_for_functional_statement(mapped_variant): - return variant_functional_impact_statement(mapped_variant) - return variant_study_result(mapped_variant) + if can_annotate_variant_for_pathogenicity_evidence(context.variant): + return variant_pathogenicity_statement(context) + if can_annotate_variant_for_functional_statement(context.variant): + return variant_functional_impact_statement(context) + return variant_study_result(context) except MappingDataDoesntExistException: return None diff --git a/src/mavedb/lib/annotation/calibration.py b/src/mavedb/lib/annotation/calibration.py new file mode 100644 index 000000000..ec6f67f50 --- /dev/null +++ b/src/mavedb/lib/annotation/calibration.py @@ -0,0 +1,177 @@ +"""Score-calibration eligibility and selection for VA-Spec annotation. + +Which of a score set's calibrations may back an annotation (``score_calibration_may_be_used_for_annotation``) +and, among the eligible ones, which carries the strongest evidence for a given variant +(``select_strongest_*``). The selection resolves ties/conflicts to the conservative call (normal / uncertain +significance). Consumed by the annotation builders in ``annotate.py``. +""" + +from typing import Literal, Optional + +from ga4gh.va_spec.base.enums import StrengthOfEvidenceProvided as VaSpecStrengthOfEvidenceProvided + +from mavedb.lib.annotation.classification import ( + ExperimentalVariantFunctionalImpactClassification, + functional_classification_of_variant, + pathogenicity_classification_of_variant, +) +from mavedb.models.score_calibration import ScoreCalibration +from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification +from mavedb.models.variant import Variant + + +def score_calibration_may_be_used_for_annotation( + score_calibration: ScoreCalibration, + annotation_type: Literal["pathogenicity", "functional"], + allow_research_use_only_calibrations: bool = False, +) -> bool: + """ + Check if a score calibration may be used for annotation based on its properties. + + This function evaluates whether a given score calibration is suitable for use in + annotation by checking its research use only status and the presence of required + classifications based on the annotation type. + + Args: + score_calibration (ScoreCalibration): The score calibration to evaluate. + annotation_type (Literal["pathogenicity", "functional"]): The type of annotation + being considered, which determines the required classifications for validity. + allow_research_use_only_calibrations (bool, optional): Whether to allow calibrations + marked as research use only for annotation. Defaults to False. + + Returns: + bool: True if the score calibration can be used for annotation, False otherwise. + """ + if score_calibration.research_use_only and not allow_research_use_only_calibrations: + return False + + if score_calibration.functional_classifications is None or len(score_calibration.functional_classifications) == 0: + return False + + if annotation_type == "pathogenicity" and all( + fr.acmg_classification is None for fr in score_calibration.functional_classifications + ): + return False + + return True + + +def select_strongest_functional_calibration( + variant: Variant, + calibrations: list[ScoreCalibration], +) -> tuple[Optional[ScoreCalibration], Optional[ScoreCalibrationFunctionalClassification]]: + """ + Select the calibration with the strongest evidence for functional classification. + + In case of ties or conflicting classifications, defaults to normal classification. + Returns the calibration and its functional classification range that contains the variant. + + If the variant is not contained in any range, returns (first_calibration, None) to indicate + the variant should be classified as INDETERMINATE but still receive annotations. + """ + if not calibrations: + return None, None + + # Collect all calibrations and their classifications + candidates: list[ + tuple[ + ScoreCalibration, + ScoreCalibrationFunctionalClassification, + ExperimentalVariantFunctionalImpactClassification, + ] + ] = [] + + for calibration in calibrations: + functional_range, classification = functional_classification_of_variant(variant, calibration) + if functional_range is not None: + candidates.append((calibration, functional_range, classification)) + + # If variant is not in any range, return first calibration with None to indicate INDETERMINATE + if not candidates: + return calibrations[0], None + + # If only one candidate, return it + if len(candidates) == 1: + return candidates[0][0], candidates[0][1] + + # Check if all classifications agree + classifications = [c[2] for c in candidates] + if all(cls == classifications[0] for cls in classifications): + # All agree, return the first one + return candidates[0][0], candidates[0][1] + + # Conflict exists: default to normal classification + normal_candidates = [c for c in candidates if c[2] == ExperimentalVariantFunctionalImpactClassification.NORMAL] + if normal_candidates: + return normal_candidates[0][0], normal_candidates[0][1] + + # If no normal classification, return the first candidate + return candidates[0][0], candidates[0][1] + + +def select_strongest_pathogenicity_calibration( + variant: Variant, + calibrations: list[ScoreCalibration], +) -> tuple[Optional[ScoreCalibration], Optional[ScoreCalibrationFunctionalClassification]]: + """ + Select the calibration with the strongest evidence for pathogenicity classification. + + Uses ACMG evidence strength to determine the strongest evidence. + In case of ties with conflicting evidence (both benign and pathogenic), defaults to uncertain + significance by returning None for the functional range. + Returns the calibration and its functional classification range that contains the variant. + + If the variant is not contained in any range, returns (first_calibration, None) to indicate + the variant should receive annotations even though it's not classified in any range. + """ + if not calibrations: + return None, None + + # Define evidence strength ordering (higher index = stronger evidence) + # Note: VA-Spec StrengthOfEvidenceProvided doesn't have MODERATE_PLUS, only our MaveDB enum does. + # The classification.py module maps MODERATE_PLUS to MODERATE when returning VA-Spec enum values. + strength_order = { + None: 0, + VaSpecStrengthOfEvidenceProvided.SUPPORTING: 1, + VaSpecStrengthOfEvidenceProvided.MODERATE: 2, + VaSpecStrengthOfEvidenceProvided.STRONG: 3, + VaSpecStrengthOfEvidenceProvided.VERY_STRONG: 4, + } + + # Collect all calibrations with their evidence strength and criterion + candidates: list[tuple[ScoreCalibration, ScoreCalibrationFunctionalClassification, int, bool]] = [] + + for calibration in calibrations: + functional_range, criterion, evidence_strength = pathogenicity_classification_of_variant(variant, calibration) + if functional_range is not None: + strength_value = strength_order.get(evidence_strength, 0) + is_benign = criterion.name.startswith("B") if criterion else False + candidates.append((calibration, functional_range, strength_value, is_benign)) + + # If variant is not in any range, return first calibration with None + if not candidates: + return calibrations[0], None + + # If only one candidate, return it + if len(candidates) == 1: + return candidates[0][0], candidates[0][1] + + # Find the maximum strength + max_strength = max(c[2] for c in candidates) + strongest_candidates = [c for c in candidates if c[2] == max_strength] + + # If only one with max strength, return it + if len(strongest_candidates) == 1: + return strongest_candidates[0][0], strongest_candidates[0][1] + + # Tie: check for conflicting evidence (both benign and pathogenic) + has_benign = any(c[3] for c in strongest_candidates) + has_pathogenic = any(not c[3] for c in strongest_candidates) + + # If there's a conflict between benign and pathogenic evidence of equal strength, + # return None for the functional range to indicate uncertain significance + if has_benign and has_pathogenic: + return strongest_candidates[0][0], None + + # Otherwise return the first of the strongest + return strongest_candidates[0][0], strongest_candidates[0][1] diff --git a/src/mavedb/lib/annotation/classification.py b/src/mavedb/lib/annotation/classification.py index 08fc3b208..9e6e774a6 100644 --- a/src/mavedb/lib/annotation/classification.py +++ b/src/mavedb/lib/annotation/classification.py @@ -6,7 +6,7 @@ from ga4gh.va_spec.base.enums import StrengthOfEvidenceProvided from mavedb.models.enums.functional_classification import FunctionalClassification as FunctionalClassificationOptions -from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.variant import Variant from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification @@ -22,22 +22,22 @@ class ExperimentalVariantFunctionalImpactClassification(StrEnum): def functional_classification_of_variant( - mapped_variant: MappedVariant, score_calibration: ScoreCalibration + variant: Variant, score_calibration: ScoreCalibration ) -> tuple[Optional[ScoreCalibrationFunctionalClassification], ExperimentalVariantFunctionalImpactClassification]: """Classify a variant's functional impact as normal, abnormal, or indeterminate. Uses the primary score calibration and its functional ranges. Raises ValueError if required calibration or score is missing. """ - if not mapped_variant.variant.score_set.score_calibrations: + if not variant.score_set.score_calibrations: raise ValueError( - f"Variant {mapped_variant.variant.urn} does not have a score set with score calibrations." + f"Variant {variant.urn} does not have a score set with score calibrations." " Unable to classify functional impact." ) if not score_calibration.functional_classifications: raise ValueError( - f"Variant {mapped_variant.variant.urn} does not have ranges defined in its primary score calibration." + f"Variant {variant.urn} does not have ranges defined in its primary score calibration." " Unable to classify functional impact." ) @@ -45,7 +45,7 @@ def functional_classification_of_variant( # DB-agnostic function. Resolve class-based matches in an upstream DB-aware layer using the association table, # pass matched functional classification IDs into this function, and use O(1) ID membership checks here. for functional_range in score_calibration.functional_classifications: - if mapped_variant.variant in functional_range.variants: + if variant in functional_range.variants: if functional_range.functional_classification is FunctionalClassificationOptions.normal: return functional_range, ExperimentalVariantFunctionalImpactClassification.NORMAL elif functional_range.functional_classification is FunctionalClassificationOptions.abnormal: @@ -56,7 +56,7 @@ def functional_classification_of_variant( def pathogenicity_classification_of_variant( - mapped_variant: MappedVariant, + variant: Variant, score_calibration: ScoreCalibration, ) -> tuple[ Optional[ScoreCalibrationFunctionalClassification], @@ -75,15 +75,15 @@ def pathogenicity_classification_of_variant( Raises ValueError if required calibration, score, or evidence strength is missing. """ - if not mapped_variant.variant.score_set.score_calibrations: + if not variant.score_set.score_calibrations: raise ValueError( - f"Variant {mapped_variant.variant.urn} does not have a score set with score calibrations." + f"Variant {variant.urn} does not have a score set with score calibrations." " Unable to classify clinical impact." ) if not score_calibration.functional_classifications: raise ValueError( - f"Variant {mapped_variant.variant.urn} does not have ranges defined in its primary score calibration." + f"Variant {variant.urn} does not have ranges defined in its primary score calibration." " Unable to classify clinical impact." ) @@ -91,7 +91,7 @@ def pathogenicity_classification_of_variant( # DB-agnostic function. Resolve class-based matches in an upstream DB-aware layer using the association table, # pass matched functional classification IDs into this function, and use O(1) ID membership checks here. for pathogenicity_range in score_calibration.functional_classifications: - if mapped_variant.variant in pathogenicity_range.variants: + if variant in pathogenicity_range.variants: if pathogenicity_range.acmg_classification is None: return (pathogenicity_range, VariantPathogenicityEvidenceLine.Criterion.PS3, None) @@ -130,7 +130,7 @@ def pathogenicity_classification_of_variant( not in VariantPathogenicityEvidenceLine.Criterion._member_names_ ): # pragma: no cover - enforced by model validators in FunctionalClassification view model raise ValueError( - f"Variant {mapped_variant.variant.urn} is contained in a clinical calibration range with an invalid criterion." + f"Variant {variant.urn} is contained in a clinical calibration range with an invalid criterion." " Unable to classify clinical impact." ) diff --git a/src/mavedb/lib/annotation/context.py b/src/mavedb/lib/annotation/context.py new file mode 100644 index 000000000..3e67f6d39 --- /dev/null +++ b/src/mavedb/lib/annotation/context.py @@ -0,0 +1,80 @@ +"""Allele-graph variant context for annotation builders. + +Serves ``MappingRecord`` / ``Allele`` / ``MappingRecordAllele`` substrate to VA-Spec builders, +and provides a single point of truth for the live (or as-of) mapping record, authoritative allele, +and pre-built VA proposition subject. +""" + +from dataclasses import dataclass +from datetime import datetime +from typing import Optional + +from ga4gh.cat_vrs.models import CategoricalVariant +from ga4gh.vrs.models import MolecularVariation +from sqlalchemy import select +from sqlalchemy.orm import Session + +from mavedb.lib.alleles import get_live_record_allele_links +from mavedb.lib.cat_vrs import build_categorical_variant +from mavedb.lib.vrs import vrs_object_from_mapped_variant +from mavedb.models.allele import Allele +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.variant import Variant + + +@dataclass +class VariantAnnotationContext: + """A variant's annotation inputs, sourced from it's allele-graph. + + ``record`` is the live (or as-of) ``MappingRecord`` — its ``mapping_api_version`` / ``mapped_date`` + supply VA provenance, and its ``ValidTime`` is what ``as_of`` and supersession are evaluated against. + ``measured_allele`` is the authoritative allele (the assayed representation); its ``post_mapped`` VRS is + the concrete study-result focus. ``subject_variant`` is the VA *proposition* subject. When projections + exist, this is a Cat-VRS ``CategoricalVariant`` object. Otherwise, it is the measured ``MolecularVariation``. + """ + + variant: Variant + record: MappingRecord + measured_allele: Allele + subject_variant: MolecularVariation | CategoricalVariant + as_of: Optional[datetime] + + +def variant_annotation_context( + db: Session, variant: Variant, *, as_of: Optional[datetime] = None +) -> Optional[VariantAnnotationContext]: + """Assemble the annotation context for ``variant`` from the live (or as-of) mapping substrate. + + Returns ``None`` when the variant is unmapped at ``as_of`` — no live mapping record, or an authoritative + allele that carries no ``post_mapped`` VRS (nothing to annotate). One record fetch + one allele-link + fetch; the Cat-VRS transit is built from the same links, so the subject costs no redundant query. + """ + record = db.scalar( + select(MappingRecord).where(MappingRecord.variant_id == variant.id).where(MappingRecord.live_at(as_of)) + ) + if record is None: + return None + + links = get_live_record_allele_links(db, variant.id, as_of=as_of) + measured_allele = next((link.allele for link in links if link.is_authoritative), None) + if measured_allele is None or measured_allele.post_mapped is None: + return None + + # The proposition subject follows the measured as anchor rule: the categorical variant when it carries a + # projection member, else the concrete measured variation. The VA subject is deliberately *narrow* + # (include_convergent=False): the synonymous cousins are dropped, because VA-Spec carries no per-member + # provenance to mark them as unmeasured, and StudyResult.focusVariant already pins the concrete measured + # allele. + transit = build_categorical_variant(links, name=variant.urn or "", include_convergent=False) + if transit is not None and len(transit.categorical_variant.members) > 1: + subject_variant: MolecularVariation | CategoricalVariant = transit.categorical_variant + else: + subject_variant = vrs_object_from_mapped_variant(measured_allele.post_mapped) + + return VariantAnnotationContext( + variant=variant, + record=record, + measured_allele=measured_allele, + subject_variant=subject_variant, + as_of=as_of, + ) diff --git a/src/mavedb/lib/annotation/document.py b/src/mavedb/lib/annotation/document.py index 039142030..d9e58cfc2 100644 --- a/src/mavedb/lib/annotation/document.py +++ b/src/mavedb/lib/annotation/document.py @@ -7,8 +7,8 @@ from ga4gh.va_spec.base.core import Document from mavedb.constants import MAVEDB_FRONTEND_URL +from mavedb.models.allele import Allele from mavedb.models.experiment import Experiment -from mavedb.models.mapped_variant import MappedVariant from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.score_set import ScoreSet from mavedb.models.variant import Variant @@ -128,31 +128,34 @@ def score_calibration_as_document(score_calibration: ScoreCalibration) -> Docume ) -def mapped_variant_as_iri(mapped_variant: MappedVariant) -> Optional[IRI]: +def measured_allele_as_iri(allele: Allele) -> Optional[IRI]: """ - Create an IRI as described in for the provided MaveDB mapped variant. Within - the context of VA-Spec, these can be used interchangeably with an equivalent document object for brevity. + Create an IRI as described in for the + measured (authoritative) allele, keyed by its ClinGen allele id. Within the context of VA-Spec, these + can be used interchangeably with an equivalent document object for brevity. ``None`` when the allele + has no ClinGen allele id. """ - if not mapped_variant.clingen_allele_id: + if not allele.clingen_allele_id: return None - return IRI(f"https://mavedb.org/variant/{urllib.parse.quote_plus(mapped_variant.clingen_allele_id)}") + return IRI(f"https://mavedb.org/variant/{urllib.parse.quote_plus(allele.clingen_allele_id)}") -def mapped_variant_to_document(mapped_variant: MappedVariant) -> Optional[Document]: +def measured_allele_to_document(allele: Allele) -> Optional[Document]: """ Create a [VA Document](https://va-ga4gh.readthedocs.io/en/latest/core-information-model/entities/information-entities/document.html#document) - object from the provided MaveDB mapped variant. + object from the provided MaveDB measured (authoritative) allele, keyed by its ClinGen allele id. ``None`` + when the allele has no ClinGen allele id. """ - if not mapped_variant.clingen_allele_id: + if not allele.clingen_allele_id: return None return Document( - id=mapped_variant.variant.urn, - name="MaveDB Mapped Variant", - documentType="mapped genomic variant description", - # We only reach this point if a IRI is guaranteed to exist - urls=[mapped_variant_as_iri(mapped_variant).root], # type: ignore + id=allele.clingen_allele_id, + name="MaveDB Measured Allele", + documentType="measured allele", + # The measured allele document is keyed by its ClinGen allele id, so the IRI is guaranteed to exist. + urls=[measured_allele_as_iri(allele).root], # type: ignore ) diff --git a/src/mavedb/lib/annotation/eligibility.py b/src/mavedb/lib/annotation/eligibility.py new file mode 100644 index 000000000..8a6e05941 --- /dev/null +++ b/src/mavedb/lib/annotation/eligibility.py @@ -0,0 +1,128 @@ +"""Whether a variant is eligible for VA-Spec annotation. + +Gate predicates the annotation builders consult before constructing a statement: a variant needs a real +score (the base assumption) and its score set needs at least one calibration usable for the target +annotation type (:func:`calibration.score_calibration_may_be_used_for_annotation`). Split by annotation +type into ``can_annotate_variant_for_pathogenicity_evidence`` / ``can_annotate_variant_for_functional_statement``. +""" + +from typing import Literal + +from mavedb.lib.annotation.calibration import score_calibration_may_be_used_for_annotation +from mavedb.lib.variants import variant_score +from mavedb.models.variant import Variant + + +def _can_annotate_variant_base_assumptions(variant: Variant) -> bool: + """ + Check if a variant meets the basic requirements for annotation. + + This function validates that a variant has the necessary data to proceed with + annotation by checking for a valid score value. + + Args: + variant (Variant): The variant to check for annotation eligibility. + + Returns: + bool: True if the variant can be annotated (has a non-None numeric score), False otherwise. + """ + # A variant is annotatable only if it carries a real (non-null, numeric) score. + if variant_score(variant) is None: + return False + + return True + + +def _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( + variant: Variant, + annotation_type: Literal["pathogenicity", "functional"], + allow_research_use_only_calibrations: bool = False, +) -> bool: + """ + Check if a variant's score set contains any of the required calibrations for annotation. + + Args: + variant (Variant): The variant whose score set is checked. + annotation_type (Literal["pathogenicity", "functional"]): The type of annotation to check for. + Must be either "pathogenicity" or "functional". + allow_research_use_only_calibrations (bool, optional): Whether to consider calibrations marked as + research use only as valid for annotation. Defaults to False. + + Returns: + bool: True if the variant's score set contains at least one valid calibration with the required + classifications for the specified annotation type. False otherwise. + """ + if variant.score_set.score_calibrations is None: + return False + + return any( + score_calibration_may_be_used_for_annotation( + score_calibration, + annotation_type, + allow_research_use_only_calibrations=allow_research_use_only_calibrations, + ) + for score_calibration in variant.score_set.score_calibrations + ) + + +def can_annotate_variant_for_pathogenicity_evidence( + variant: Variant, allow_research_use_only_calibrations=False +) -> bool: + """ + Determine if a variant can be annotated for pathogenicity evidence. + + This function checks if a variant meets all the necessary conditions to receive + pathogenicity evidence annotations by validating base assumptions and ensuring the variant's + score calibrations contain the required kinds for pathogenicity evidence annotation. + + Args: + variant (Variant): The variant to evaluate for pathogenicity evidence annotation eligibility. + + Returns: + bool: True if the variant can be annotated for pathogenicity evidence, False otherwise. + + Notes: + The function performs two main validation checks: + 1. Basic annotation assumptions via _can_annotate_variant_base_assumptions + 2. Verifies score calibrations have an appropriate calibration for pathogenicity evidence annotation. + + Both checks must pass for the variant to be considered eligible for + pathogenicity evidence annotation. + """ + if not _can_annotate_variant_base_assumptions(variant): + return False + if not _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( + variant, "pathogenicity", allow_research_use_only_calibrations=allow_research_use_only_calibrations + ): + return False + + return True + + +def can_annotate_variant_for_functional_statement(variant: Variant, allow_research_use_only_calibrations=False) -> bool: + """ + Determine if a variant can be annotated for functional statements. + + This function checks if a variant meets all the necessary conditions to receive + functional annotations by validating base assumptions and ensuring the variant's + score calibrations contain the required kinds for functional annotation. + + Args: + variant (Variant): The variant to check for annotation eligibility. + + Returns: + bool: True if the variant can be annotated for functional statements, False otherwise. + + Notes: + The function performs two main checks: + 1. Validates base assumptions using _can_annotate_variant_base_assumptions + 2. Verifies score calibrations have an appropriate calibration for functional annotation. + """ + if not _can_annotate_variant_base_assumptions(variant): + return False + if not _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( + variant, "functional", allow_research_use_only_calibrations=allow_research_use_only_calibrations + ): + return False + + return True diff --git a/src/mavedb/lib/annotation/evidence_line.py b/src/mavedb/lib/annotation/evidence_line.py index 8ebf7f163..5ec2d5dcb 100644 --- a/src/mavedb/lib/annotation/evidence_line.py +++ b/src/mavedb/lib/annotation/evidence_line.py @@ -16,6 +16,7 @@ functional_classification_of_variant, pathogenicity_classification_of_variant, ) +from mavedb.lib.annotation.context import VariantAnnotationContext from mavedb.lib.annotation.contribution import ( mavedb_api_contribution, mavedb_score_calibration_contribution, @@ -30,18 +31,17 @@ functional_score_calibration_as_method, pathogenicity_score_calibration_as_method, ) -from mavedb.models.mapped_variant import MappedVariant from mavedb.models.score_calibration import ScoreCalibration def acmg_evidence_line( - mapped_variant: MappedVariant, + context: VariantAnnotationContext, score_calibration: ScoreCalibration, proposition: VariantPathogenicityProposition, evidence: list[Union[StudyResult, EvidenceLineType, StatementType, iriReference]], ) -> VariantPathogenicityEvidenceLine: containing_evidence_range, evidence_outcome, evidence_strength = pathogenicity_classification_of_variant( - mapped_variant, score_calibration + context.variant, score_calibration ) if not evidence_strength: @@ -63,7 +63,7 @@ def acmg_evidence_line( direction_of_evidence = direction_of_support_for_pathogenicity_classification(evidence_outcome) return VariantPathogenicityEvidenceLine( - description=f"Pathogenicity evidence line for {mapped_variant.variant.urn}.", + description=f"Pathogenicity evidence line for {context.variant.urn}.", hasEvidenceItems=list(evidence), specifiedBy=pathogenicity_score_calibration_as_method(score_calibration, evidence_outcome), evidenceOutcome={ @@ -77,7 +77,7 @@ def acmg_evidence_line( directionOfEvidenceProvided=direction_of_evidence, contributions=[ mavedb_api_contribution(), - mavedb_vrs_contribution(mapped_variant), + mavedb_vrs_contribution(context), mavedb_score_calibration_contribution(score_calibration), ], targetProposition=proposition, @@ -93,14 +93,14 @@ def acmg_evidence_line( def functional_evidence_line( - mapped_variant: MappedVariant, + context: VariantAnnotationContext, score_calibration: ScoreCalibration, evidence: list[Union[StudyResult, EvidenceLineType, StatementType, iriReference]], ) -> EvidenceLine: - containing_evidence_range, classification = functional_classification_of_variant(mapped_variant, score_calibration) + containing_evidence_range, classification = functional_classification_of_variant(context.variant, score_calibration) return EvidenceLine( - description=f"Functional evidence line for {mapped_variant.variant.urn}", + description=f"Functional evidence line for {context.variant.urn}", hasEvidenceItems=[StudyResult(root=item) for item in evidence], directionOfEvidenceProvided=direction_of_support_for_functional_classification(classification), evidenceOutcome=MappableConcept( @@ -112,7 +112,7 @@ def functional_evidence_line( specifiedBy=functional_score_calibration_as_method(score_calibration), contributions=[ mavedb_api_contribution(), - mavedb_vrs_contribution(mapped_variant), + mavedb_vrs_contribution(context), mavedb_score_calibration_contribution(score_calibration), ], reportedIn=[score_calibration_as_document(score_calibration)], diff --git a/src/mavedb/lib/annotation/proposition.py b/src/mavedb/lib/annotation/proposition.py index 963327c73..143d5d608 100644 --- a/src/mavedb/lib/annotation/proposition.py +++ b/src/mavedb/lib/annotation/proposition.py @@ -2,22 +2,77 @@ from ga4gh.va_spec.base.core import ExperimentalVariantFunctionalImpactProposition, VariantPathogenicityProposition from mavedb.lib.annotation.condition import generic_disease_condition +from mavedb.lib.annotation.context import VariantAnnotationContext from mavedb.lib.annotation.document import experiment_to_document -from mavedb.lib.annotation.util import sequence_feature_for_mapped_variant, variation_from_mapped_variant -from mavedb.models.mapped_variant import MappedVariant +from mavedb.lib.annotation.exceptions import MappingDataDoesntExistException +from mavedb.lib.mapping import extract_ids_from_post_mapped_metadata +from mavedb.lib.types.annotation import SequenceFeature +from mavedb.lib.variants import target_for_variant +from mavedb.models.variant import Variant -def mapped_variant_to_experimental_variant_clinical_impact_proposition( - mapped_variant: MappedVariant, +def sequence_feature_for_variant(variant: Variant) -> SequenceFeature: + """ + Extract the sequence feature (e.g., gene or transcript) associated with a variant. + + This function retrieves the sequence feature from the variant's target data, which is + necessary for generating annotations that reference specific genomic features. Co-located with the + propositions that consume it — its only caller. + + Args: + variant (Variant): The variant whose target gene supplies the sequence feature. + + Returns: + SequenceFeature: Named tuple with: + - `identifier`: sequence feature identifier (gene/transcript ID or name) + - `system`: source/system URL for the identifier + + """ + target = target_for_variant(variant) + if target is None: + raise MappingDataDoesntExistException( + f"Variant {variant.urn} does not have an identifiable target gene." + " Unable to extract sequence feature for annotation." + ) + + # Prefer the mapped HGNC name if it's available, as this is more likely to be stable and recognizable than accessions or other identifiers. + # If the mapped HGNC name is not available, fall back to extracting an identifier from the post-mapped metadata, which may be a gene or + # transcript identifier of varying formats. If neither of those options are available, fall back to the target gene's name as listed in MaveDB. + if target.mapped_hgnc_name: + return SequenceFeature(target.mapped_hgnc_name, "https://www.genenames.org/") + + post_mapped_ids = extract_ids_from_post_mapped_metadata( + target.post_mapped_metadata if target.post_mapped_metadata else {} # type: ignore + ) + if post_mapped_ids: + post_mapped_id = post_mapped_ids[0] + if post_mapped_id.startswith("ENSG") or post_mapped_id.startswith("ENST") or post_mapped_id.startswith("ENSP"): + return SequenceFeature(post_mapped_id, "https://www.ensembl.org/index.html") + elif post_mapped_id.startswith("NM_") or post_mapped_id.startswith("NR_") or post_mapped_id.startswith("NP_"): + return SequenceFeature(post_mapped_id, "https://www.ncbi.nlm.nih.gov/refseq/") + + return SequenceFeature(post_mapped_id, "transcript or gene identifier of unknown source") + + if target.name: + return SequenceFeature(target.name, "https://www.mavedb.org/") + + raise MappingDataDoesntExistException( + f"Variant {variant.urn} does not have an identifiable sequence feature in its target gene data." + " Unable to extract sequence feature for annotation." + ) + + +def variant_pathogenicity_proposition( + context: VariantAnnotationContext, ) -> VariantPathogenicityProposition: - coding, system = sequence_feature_for_mapped_variant(mapped_variant) + coding, system = sequence_feature_for_variant(context.variant) sequence_feature = MappableConcept( primaryCoding=Coding(code=coding, system=system), ) return VariantPathogenicityProposition( - description=f"Variant pathogenicity proposition for {mapped_variant.variant.urn}.", - subjectVariant=variation_from_mapped_variant(mapped_variant), + description=f"Variant pathogenicity proposition for {context.variant.urn}.", + subjectVariant=context.subject_variant, predicate="isCausalFor", objectCondition=generic_disease_condition(), geneContextQualifier=sequence_feature @@ -26,18 +81,18 @@ def mapped_variant_to_experimental_variant_clinical_impact_proposition( ) -def mapped_variant_to_experimental_variant_functional_impact_proposition( - mapped_variant: MappedVariant, +def variant_functional_impact_proposition( + context: VariantAnnotationContext, ) -> ExperimentalVariantFunctionalImpactProposition: - coding, system = sequence_feature_for_mapped_variant(mapped_variant) + coding, system = sequence_feature_for_variant(context.variant) sequence_feature = MappableConcept( primaryCoding=Coding(code=coding, system=system), ) return ExperimentalVariantFunctionalImpactProposition( - description=f"Variant functional impact proposition for {mapped_variant.variant.urn}.", - subjectVariant=variation_from_mapped_variant(mapped_variant), + description=f"Variant functional impact proposition for {context.variant.urn}.", + subjectVariant=context.subject_variant, predicate="impactsFunctionOf", objectSequenceFeature=sequence_feature, - experimentalContextQualifier=experiment_to_document(mapped_variant.variant.score_set.experiment), + experimentalContextQualifier=experiment_to_document(context.variant.score_set.experiment), ) diff --git a/src/mavedb/lib/annotation/statement.py b/src/mavedb/lib/annotation/statement.py index c77d30f1a..7db3b05d8 100644 --- a/src/mavedb/lib/annotation/statement.py +++ b/src/mavedb/lib/annotation/statement.py @@ -10,6 +10,7 @@ ) from mavedb.lib.annotation.classification import ExperimentalVariantFunctionalImpactClassification +from mavedb.lib.annotation.context import VariantAnnotationContext from mavedb.lib.annotation.contribution import ( mavedb_api_contribution, mavedb_score_calibration_contribution, @@ -20,23 +21,22 @@ variant_interpretation_functional_guideline_method, variant_interpretation_pathogenicity_guideline_method, ) -from mavedb.models.mapped_variant import MappedVariant from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification -def mapped_variant_to_functional_statement( - mapped_variant: MappedVariant, +def functional_statement( + context: VariantAnnotationContext, proposition: ExperimentalVariantFunctionalImpactProposition, evidence: list[EvidenceLine], score_calibration: ScoreCalibration, functional_classification: ExperimentalVariantFunctionalImpactClassification, ) -> Statement: """ - Create a functional impact statement for a mapped variant. + Create a functional impact statement for a variant. Args: - mapped_variant: The variant being classified + context: The variant's annotation context (variant + mapping-record provenance) proposition: The functional impact proposition evidence: List of evidence lines supporting the statement score_calibration: The score calibration with the strongest evidence @@ -48,11 +48,11 @@ def mapped_variant_to_functional_statement( direction = aggregate_direction_of_evidence(evidence) return Statement( - description=f"Variant functional impact statement for {mapped_variant.variant.urn}.", + description=f"Variant functional impact statement for {context.variant.urn}.", specifiedBy=variant_interpretation_functional_guideline_method(), contributions=[ mavedb_api_contribution(), - mavedb_vrs_contribution(mapped_variant), + mavedb_vrs_contribution(context), mavedb_score_calibration_contribution(score_calibration), ], proposition=proposition, @@ -67,18 +67,18 @@ def mapped_variant_to_functional_statement( ) -def mapped_variant_to_pathogenicity_statement( - mapped_variant: MappedVariant, +def pathogenicity_statement( + context: VariantAnnotationContext, proposition: VariantPathogenicityProposition, evidence: list[VariantPathogenicityEvidenceLine], score_calibration: ScoreCalibration, functional_range: Optional[ScoreCalibrationFunctionalClassification], ) -> VariantPathogenicityStatement: """ - Create a pathogenicity statement for a mapped variant. + Create a pathogenicity statement for a variant. Args: - mapped_variant: The variant being classified + context: The variant's annotation context (variant + mapping-record provenance) proposition: The pathogenicity proposition evidence: List of evidence lines supporting the statement score_calibration: The score calibration with the strongest evidence @@ -105,11 +105,11 @@ def mapped_variant_to_pathogenicity_statement( acmg_classification = AcmgClassification.UNCERTAIN_SIGNIFICANCE return VariantPathogenicityStatement( - description=f"Variant pathogenicity statement for {mapped_variant.variant.urn}.", + description=f"Variant pathogenicity statement for {context.variant.urn}.", specifiedBy=variant_interpretation_pathogenicity_guideline_method(), contributions=[ mavedb_api_contribution(), - mavedb_vrs_contribution(mapped_variant), + mavedb_vrs_contribution(context), mavedb_score_calibration_contribution(score_calibration), ], proposition=proposition, diff --git a/src/mavedb/lib/annotation/study_result.py b/src/mavedb/lib/annotation/study_result.py index b93ea1fee..c45457468 100644 --- a/src/mavedb/lib/annotation/study_result.py +++ b/src/mavedb/lib/annotation/study_result.py @@ -1,5 +1,6 @@ from ga4gh.va_spec.base.core import ExperimentalVariantFunctionalImpactStudyResult +from mavedb.lib.annotation.context import VariantAnnotationContext from mavedb.lib.annotation.contribution import ( mavedb_api_contribution, mavedb_creator_contribution, @@ -7,31 +8,31 @@ mavedb_vrs_contribution, ) from mavedb.lib.annotation.dataset import score_set_to_data_set -from mavedb.lib.annotation.document import mapped_variant_as_iri, variant_as_iri +from mavedb.lib.annotation.document import measured_allele_as_iri, variant_as_iri from mavedb.lib.annotation.method import ( publication_identifiers_to_method, ) -from mavedb.lib.annotation.util import variation_from_mapped_variant +from mavedb.lib.vrs import vrs_object_from_mapped_variant from mavedb.lib.variants import variant_score -from mavedb.models.mapped_variant import MappedVariant -def mapped_variant_to_experimental_variant_impact_study_result( - mapped_variant: MappedVariant, +def variant_impact_study_result( + context: VariantAnnotationContext, ) -> ExperimentalVariantFunctionalImpactStudyResult: + # The study result's focus is the concrete measured allele — never a CategoricalVariant (VA-Spec + # narrows ``focusVariant`` to MolecularVariation). The context guarantees a hydratable post_mapped. return ExperimentalVariantFunctionalImpactStudyResult( - description=f"Variant effect study result for {mapped_variant.variant.urn}.", - focusVariant=variation_from_mapped_variant(mapped_variant), - functionalImpactScore=variant_score(mapped_variant.variant), - specifiedBy=publication_identifiers_to_method( - mapped_variant.variant.score_set.publication_identifier_associations - ), - sourceDataSet=score_set_to_data_set(mapped_variant.variant.score_set), + description=f"Variant effect study result for {context.variant.urn}.", + # post_mapped is guaranteed non-null by variant_annotation_context (it returns None otherwise). + focusVariant=vrs_object_from_mapped_variant(context.measured_allele.post_mapped), # type: ignore[arg-type] + functionalImpactScore=variant_score(context.variant), + specifiedBy=publication_identifiers_to_method(context.variant.score_set.publication_identifier_associations), + sourceDataSet=score_set_to_data_set(context.variant.score_set), contributions=[ mavedb_api_contribution(), - mavedb_vrs_contribution(mapped_variant), - mavedb_creator_contribution(mapped_variant.variant, mapped_variant.variant.score_set.created_by), - mavedb_modifier_contribution(mapped_variant.variant, mapped_variant.variant.score_set.modified_by), + mavedb_vrs_contribution(context), + mavedb_creator_contribution(context.variant, context.variant.score_set.created_by), + mavedb_modifier_contribution(context.variant, context.variant.score_set.modified_by), ], - reportedIn=filter(None, [variant_as_iri(mapped_variant.variant), mapped_variant_as_iri(mapped_variant)]), + reportedIn=filter(None, [variant_as_iri(context.variant), measured_allele_as_iri(context.measured_allele)]), ) diff --git a/src/mavedb/lib/annotation/util.py b/src/mavedb/lib/annotation/util.py deleted file mode 100644 index ed90c5d03..000000000 --- a/src/mavedb/lib/annotation/util.py +++ /dev/null @@ -1,504 +0,0 @@ -from typing import Literal, Optional - -from ga4gh.core.models import Extension -from ga4gh.va_spec.base.enums import StrengthOfEvidenceProvided as VaSpecStrengthOfEvidenceProvided -from ga4gh.vrs.models import ( - Allele, - CisPhasedBlock, - Expression, - LengthExpression, - LiteralSequenceExpression, - MolecularVariation, - ReferenceLengthExpression, - SequenceLocation, - SequenceReference, -) - -from mavedb.lib.annotation.classification import ( - ExperimentalVariantFunctionalImpactClassification, - functional_classification_of_variant, - pathogenicity_classification_of_variant, -) -from mavedb.lib.annotation.exceptions import MappingDataDoesntExistException -from mavedb.lib.mapping import extract_ids_from_post_mapped_metadata -from mavedb.lib.types.annotation import SequenceFeature -from mavedb.lib.variants import target_for_variant, variant_score -from mavedb.models.mapped_variant import MappedVariant -from mavedb.models.score_calibration import ScoreCalibration -from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification - - -def allele_from_mapped_variant_dictionary_result(allelic_mapping_results: dict) -> Allele: - """ - Converts a dictionary containing allelic mapping results into an Allele object. - - This function handles the possibility of an extra nesting level in early VRS 1.3 objects, - where Allele objects are contained within a `variation` property. If the `variation` key - is not present, the function assumes the dictionary itself represents the variation. - - Args: - allelic_mapping_results (dict): A dictionary containing allelic mapping results. - It may include a `variation` key or directly represent the variation. - - Returns: - Allele: An Allele object constructed from the provided mapping results. - - Raises: - KeyError: If required keys are missing from the input dictionary. - """ - - # NOTE: Early VRS 1.3 objects may contain an extra nesting level, where Allele objects - # are contained in a `variation` property. Although it's unlikely variants of this form - # will ever be exported in this format, we handle the possibility. - try: - variation = allelic_mapping_results["variation"] - except KeyError: - variation = allelic_mapping_results - - state_dict = variation["state"] - state: ReferenceLengthExpression | LengthExpression | LiteralSequenceExpression - if state_dict.get("type") == "ReferenceLengthExpression": - state = ReferenceLengthExpression(**state_dict) - elif state_dict.get("type") == "LengthExpression": - state = LengthExpression(**state_dict) - elif state_dict.get("type") == "LiteralSequenceExpression": - state = LiteralSequenceExpression(**state_dict) - else: - raise ValueError( - f"Unsupported VRS Allele state type {state_dict.get('type')!r}. " - "Update allele_from_mapped_variant_dictionary_result to handle this type." - ) - - # Explicit field extraction for alleles is intentional: stored dicts may contain extra fields (e.g. "type" on - # Extension, "label" on SequenceReference) that the strict VRS Pydantic models forbid, so using model_validate() - # directly is not possible against stored mapping results. - return Allele( - id=variation.get("id"), - state=state, - digest=variation.get("digest"), - location=SequenceLocation( - start=variation.get("location", {}).get("start"), - end=variation.get("location", {}).get("end"), - digest=variation.get("location", {}).get("digest"), - id=variation.get("location", {}).get("id"), - sequenceReference=SequenceReference( - name=variation.get("location", {}).get("sequenceReference", {}).get("name"), - refgetAccession=variation.get("location", {}).get("sequenceReference", {}).get("refgetAccession"), - ), - ), - extensions=[ - Extension( - id=extension.get("id"), - name=extension["name"], - description=extension.get("description"), - value=extension.get("value"), - ) - for extension in variation.get("extensions", []) - ], - expressions=[ - Expression( - id=expression.get("id"), - syntax=expression["syntax"], - syntax_version=expression.get("syntax_version"), - value=expression["value"], - ) - for expression in variation.get("expressions", []) - ], - ) - - -def vrs_object_from_mapped_variant(mapping_results: dict) -> MolecularVariation: - """ - Extracts a VRS (Variation Representation Specification) object from a mapped variant. - - This function processes a dictionary of mapping results and returns a VRS object, - which can either be an `Allele` or a `CisPhasedBlock`. The type of VRS object - returned depends on the "type" field in the input dictionary. - - Args: - mapping_results (dict): A dictionary containing the mapping results of a variant. - It must include a "type" key indicating the type of VRS object - ("CisPhasedBlock" or "Haplotype" for a CisPhasedBlock, or "Allele"). - If the type is "CisPhasedBlock" or "Haplotype", the dictionary must also - include a "members" key containing a list of member alleles. - - Returns: - MolecularVariation: A VRS object representing the mapped variant. This will be - either a `CisPhasedBlock` containing its member variants or an `Allele` - derived from the mapping results. - - Raises: - KeyError: If required keys are missing from the `mapping_results` dictionary. - """ - if mapping_results.get("type") == "CisPhasedBlock" or mapping_results.get("type") == "Haplotype": - return MolecularVariation( - # It's unclear why MyPy complains about the missing id field, so just add it as None (it is None by default anyway) - CisPhasedBlock( - id=None, - members=[allele_from_mapped_variant_dictionary_result(member) for member in mapping_results["members"]], - ) - ) - - return MolecularVariation(allele_from_mapped_variant_dictionary_result(mapping_results)) - - -def variation_from_mapped_variant(mapped_variant: MappedVariant) -> MolecularVariation: - """ - Converts mapping results from a mapped variant into a MolecularVariation object. - - This function takes a `MappedVariant` object and extracts its post-mapped - variant to generate a corresponding `MolecularVariation` object. If the - `post_mapped` attribute of the `MappedVariant` is `None`, an exception is raised. - - Args: - mapped_variant (MappedVariant): The mapped variant object containing - the variant data. - - Returns: - MolecularVariation: The molecular variation derived from the post-mapped - variant. - - Raises: - MappingDataDoesntExistException: If the `post_mapped` attribute of the - `mapped_variant` is `None`, indicating that the post-mapped variant - data is unavailable. - """ - if mapped_variant.post_mapped is None: - raise MappingDataDoesntExistException( - f"Variant {mapped_variant.variant.urn} does not have a post mapped variant." - " Unable to extract variation data." - ) - - return vrs_object_from_mapped_variant(mapped_variant.post_mapped) - - -def _can_annotate_variant_base_assumptions(mapped_variant: MappedVariant) -> bool: - """ - Check if a mapped variant meets the basic requirements for annotation. - - This function validates that a mapped variant has the necessary data - to proceed with annotation by checking for a valid score value. - - Args: - mapped_variant (MappedVariant): The mapped variant to check for - annotation eligibility. - - Returns: - bool: True if the variant can be annotated (has score ranges and - a non-None score), False otherwise. - """ - # A variant is annotatable only if it carries a real (non-null, numeric) score. - if variant_score(mapped_variant.variant) is None: - return False - - return True - - -def score_calibration_may_be_used_for_annotation( - score_calibration: ScoreCalibration, - annotation_type: Literal["pathogenicity", "functional"], - allow_research_use_only_calibrations: bool = False, -) -> bool: - """ - Check if a score calibration may be used for annotation based on its properties. - - This function evaluates whether a given score calibration is suitable for use in - annotation by checking its research use only status and the presence of required - classifications based on the annotation type. - - Args: - score_calibration (ScoreCalibration): The score calibration to evaluate. - annotation_type (Literal["pathogenicity", "functional"]): The type of annotation - being considered, which determines the required classifications for validity. - allow_research_use_only_calibrations (bool, optional): Whether to allow calibrations - marked as research use only for annotation. Defaults to False. - - Returns: - bool: True if the score calibration can be used for annotation, False otherwise. - """ - if score_calibration.research_use_only and not allow_research_use_only_calibrations: - return False - - if score_calibration.functional_classifications is None or len(score_calibration.functional_classifications) == 0: - return False - - if annotation_type == "pathogenicity" and all( - fr.acmg_classification is None for fr in score_calibration.functional_classifications - ): - return False - - return True - - -def _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( - mapped_variant: MappedVariant, - annotation_type: Literal["pathogenicity", "functional"], - allow_research_use_only_calibrations: bool = False, -) -> bool: - """ - Check if a mapped variant's score set contains any of the required calibrations for annotation. - - Args: - mapped_variant (MappedVariant): The mapped variant object containing the variant with score set data. - annotation_type (Literal["pathogenicity", "functional"]): The type of annotation to check for. - Must be either "pathogenicity" or "functional". - allow_research_use_only_calibrations (bool, optional): Whether to consider calibrations marked as - research use only as valid for annotation. Defaults to False. - - Returns: - bool: True if the variant's score set contains at least one valid calibration with the required - classifications for the specified annotation type. False otherwise. - """ - if mapped_variant.variant.score_set.score_calibrations is None: - return False - - return any( - score_calibration_may_be_used_for_annotation( - score_calibration, - annotation_type, - allow_research_use_only_calibrations=allow_research_use_only_calibrations, - ) - for score_calibration in mapped_variant.variant.score_set.score_calibrations - ) - - -def can_annotate_variant_for_pathogenicity_evidence( - mapped_variant: MappedVariant, allow_research_use_only_calibrations=False -) -> bool: - """ - Determine if a mapped variant can be annotated for pathogenicity evidence. - - This function checks if a variant meets all the necessary conditions to receive - pathogenicity evidence annotations by validating base assumptions and ensuring the variant's - score calibrations contain the required kinds for pathogenicity evidence annotation. - - Args: - mapped_variant (MappedVariant): The mapped variant object to evaluate - for pathogenicity evidence annotation eligibility. - - Returns: - bool: True if the variant can be annotated for pathogenicity evidence, - False otherwise. - - Notes: - The function performs two main validation checks: - 1. Basic annotation assumptions via _can_annotate_variant_base_assumptions - 2. Verifies score calibrations have an appropriate calibration for pathogenicity evidence annotation. - - Both checks must pass for the variant to be considered eligible for - pathogenicity evidence annotation. - """ - if not _can_annotate_variant_base_assumptions(mapped_variant): - return False - if not _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( - mapped_variant, "pathogenicity", allow_research_use_only_calibrations=allow_research_use_only_calibrations - ): - return False - - return True - - -def can_annotate_variant_for_functional_statement( - mapped_variant: MappedVariant, allow_research_use_only_calibrations=False -) -> bool: - """ - Determine if a mapped variant can be annotated for functional statements. - - This function checks if a variant meets all the necessary conditions to receive - functional annotations by validating base assumptions and ensuring the variant's - score calibrations contain the required kinds for functional annotation. - - Args: - mapped_variant (MappedVariant): The variant object to check for annotation - eligibility, containing mapping information and score data. - - Returns: - bool: True if the variant can be annotated for functional statements, - False otherwise. - - Notes: - The function performs two main checks: - 1. Validates base assumptions using _can_annotate_variant_base_assumptions - 2. Verifies score calibrations have an appropriate calibration for functional annotation. - """ - if not _can_annotate_variant_base_assumptions(mapped_variant): - return False - if not _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( - mapped_variant, "functional", allow_research_use_only_calibrations=allow_research_use_only_calibrations - ): - return False - - return True - - -def sequence_feature_for_mapped_variant(mapped_variant: MappedVariant) -> SequenceFeature: - """ - Extract the sequence feature (e.g., gene or transcript) associated with a mapped variant. - - This function retrieves the sequence feature from the mapped variant's data, which is - necessary for generating annotations that reference specific genomic features. - - Args: - mapped_variant (MappedVariant): The mapped variant object containing the variant data. - - Returns: - SequenceFeature: Named tuple with: - - `identifier`: sequence feature identifier (gene/transcript ID or name) - - `system`: source/system URL for the identifier - - """ - target = target_for_variant(mapped_variant.variant) - if target is None: - raise MappingDataDoesntExistException( - f"Variant {mapped_variant.variant.urn} does not have an identifiable target gene." - " Unable to extract sequence feature for annotation." - ) - - # Prefer the mapped HGNC name if it's available, as this is more likely to be stable and recognizable than accessions or other identifiers. - # If the mapped HGNC name is not available, fall back to extracting an identifier from the post-mapped metadata, which may be a gene or - # transcript identifier of varying formats. If neither of those options are available, fall back to the target gene's name as listed in MaveDB. - if target.mapped_hgnc_name: - return SequenceFeature(target.mapped_hgnc_name, "https://www.genenames.org/") - - post_mapped_ids = extract_ids_from_post_mapped_metadata( - target.post_mapped_metadata if target.post_mapped_metadata else {} # type: ignore - ) - if post_mapped_ids: - post_mapped_id = post_mapped_ids[0] - if post_mapped_id.startswith("ENSG") or post_mapped_id.startswith("ENST") or post_mapped_id.startswith("ENSP"): - return SequenceFeature(post_mapped_id, "https://www.ensembl.org/index.html") - elif post_mapped_id.startswith("NM_") or post_mapped_id.startswith("NR_") or post_mapped_id.startswith("NP_"): - return SequenceFeature(post_mapped_id, "https://www.ncbi.nlm.nih.gov/refseq/") - - return SequenceFeature(post_mapped_id, "transcript or gene identifier of unknown source") - - if target.name: - return SequenceFeature(target.name, "https://www.mavedb.org/") - - raise MappingDataDoesntExistException( - f"Variant {mapped_variant.variant.urn} does not have an identifiable sequence feature in its target gene data." - " Unable to extract sequence feature for annotation." - ) - - -def select_strongest_functional_calibration( - mapped_variant: MappedVariant, - calibrations: list[ScoreCalibration], -) -> tuple[Optional[ScoreCalibration], Optional[ScoreCalibrationFunctionalClassification]]: - """ - Select the calibration with the strongest evidence for functional classification. - - In case of ties or conflicting classifications, defaults to normal classification. - Returns the calibration and its functional classification range that contains the variant. - - If the variant is not contained in any range, returns (first_calibration, None) to indicate - the variant should be classified as INDETERMINATE but still receive annotations. - """ - if not calibrations: - return None, None - - # Collect all calibrations and their classifications - candidates: list[ - tuple[ - ScoreCalibration, - ScoreCalibrationFunctionalClassification, - ExperimentalVariantFunctionalImpactClassification, - ] - ] = [] - - for calibration in calibrations: - functional_range, classification = functional_classification_of_variant(mapped_variant, calibration) - if functional_range is not None: - candidates.append((calibration, functional_range, classification)) - - # If variant is not in any range, return first calibration with None to indicate INDETERMINATE - if not candidates: - return calibrations[0], None - - # If only one candidate, return it - if len(candidates) == 1: - return candidates[0][0], candidates[0][1] - - # Check if all classifications agree - classifications = [c[2] for c in candidates] - if all(cls == classifications[0] for cls in classifications): - # All agree, return the first one - return candidates[0][0], candidates[0][1] - - # Conflict exists: default to normal classification - normal_candidates = [c for c in candidates if c[2] == ExperimentalVariantFunctionalImpactClassification.NORMAL] - if normal_candidates: - return normal_candidates[0][0], normal_candidates[0][1] - - # If no normal classification, return the first candidate - return candidates[0][0], candidates[0][1] - - -def select_strongest_pathogenicity_calibration( - mapped_variant: MappedVariant, - calibrations: list[ScoreCalibration], -) -> tuple[Optional[ScoreCalibration], Optional[ScoreCalibrationFunctionalClassification]]: - """ - Select the calibration with the strongest evidence for pathogenicity classification. - - Uses ACMG evidence strength to determine the strongest evidence. - In case of ties with conflicting evidence (both benign and pathogenic), defaults to uncertain - significance by returning None for the functional range. - Returns the calibration and its functional classification range that contains the variant. - - If the variant is not contained in any range, returns (first_calibration, None) to indicate - the variant should receive annotations even though it's not classified in any range. - """ - if not calibrations: - return None, None - - # Define evidence strength ordering (higher index = stronger evidence) - # Note: VA-Spec StrengthOfEvidenceProvided doesn't have MODERATE_PLUS, only our MaveDB enum does. - # The classification.py module maps MODERATE_PLUS to MODERATE when returning VA-Spec enum values. - strength_order = { - None: 0, - VaSpecStrengthOfEvidenceProvided.SUPPORTING: 1, - VaSpecStrengthOfEvidenceProvided.MODERATE: 2, - VaSpecStrengthOfEvidenceProvided.STRONG: 3, - VaSpecStrengthOfEvidenceProvided.VERY_STRONG: 4, - } - - # Collect all calibrations with their evidence strength and criterion - candidates: list[tuple[ScoreCalibration, ScoreCalibrationFunctionalClassification, int, bool]] = [] - - for calibration in calibrations: - functional_range, criterion, evidence_strength = pathogenicity_classification_of_variant( - mapped_variant, calibration - ) - if functional_range is not None: - strength_value = strength_order.get(evidence_strength, 0) - is_benign = criterion.name.startswith("B") if criterion else False - candidates.append((calibration, functional_range, strength_value, is_benign)) - - # If variant is not in any range, return first calibration with None - if not candidates: - return calibrations[0], None - - # If only one candidate, return it - if len(candidates) == 1: - return candidates[0][0], candidates[0][1] - - # Find the maximum strength - max_strength = max(c[2] for c in candidates) - strongest_candidates = [c for c in candidates if c[2] == max_strength] - - # If only one with max strength, return it - if len(strongest_candidates) == 1: - return strongest_candidates[0][0], strongest_candidates[0][1] - - # Tie: check for conflicting evidence (both benign and pathogenic) - has_benign = any(c[3] for c in strongest_candidates) - has_pathogenic = any(not c[3] for c in strongest_candidates) - - # If there's a conflict between benign and pathogenic evidence of equal strength, - # return None for the functional range to indicate uncertain significance - if has_benign and has_pathogenic: - return strongest_candidates[0][0], None - - # Otherwise return the first of the strongest - return strongest_candidates[0][0], strongest_candidates[0][1] diff --git a/src/mavedb/lib/vrs.py b/src/mavedb/lib/vrs.py new file mode 100644 index 000000000..c5ee00ee1 --- /dev/null +++ b/src/mavedb/lib/vrs.py @@ -0,0 +1,134 @@ +"""Deserialization of stored ``post_mapped`` JSONB into GA4GH VRS objects. + +The mapping pipeline writes each allele's mapped representation as a VRS-shaped dict in +``Allele.post_mapped``. These helpers rehydrate that dict into strict ``ga4gh.vrs`` Pydantic models for the +Cat-VRS transit, the variant-detail envelope, and the VA-Spec annotation subjects. +""" + +from ga4gh.core.models import Extension +from ga4gh.vrs.models import ( + Allele, + CisPhasedBlock, + Expression, + LengthExpression, + LiteralSequenceExpression, + MolecularVariation, + ReferenceLengthExpression, + SequenceLocation, + SequenceReference, +) + + +def allele_from_mapped_variant_dictionary_result(allelic_mapping_results: dict) -> Allele: + """ + Converts a dictionary containing allelic mapping results into an Allele object. + + This function handles the possibility of an extra nesting level in early VRS 1.3 objects, + where Allele objects are contained within a `variation` property. If the `variation` key + is not present, the function assumes the dictionary itself represents the variation. + + Args: + allelic_mapping_results (dict): A dictionary containing allelic mapping results. + It may include a `variation` key or directly represent the variation. + + Returns: + Allele: An Allele object constructed from the provided mapping results. + + Raises: + KeyError: If required keys are missing from the input dictionary. + """ + + # NOTE: Early VRS 1.3 objects may contain an extra nesting level, where Allele objects + # are contained in a `variation` property. Although it's unlikely variants of this form + # will ever be exported in this format, we handle the possibility. + try: + variation = allelic_mapping_results["variation"] + except KeyError: + variation = allelic_mapping_results + + state_dict = variation["state"] + state: ReferenceLengthExpression | LengthExpression | LiteralSequenceExpression + if state_dict.get("type") == "ReferenceLengthExpression": + state = ReferenceLengthExpression(**state_dict) + elif state_dict.get("type") == "LengthExpression": + state = LengthExpression(**state_dict) + elif state_dict.get("type") == "LiteralSequenceExpression": + state = LiteralSequenceExpression(**state_dict) + else: + raise ValueError( + f"Unsupported VRS Allele state type {state_dict.get('type')!r}. " + "Update allele_from_mapped_variant_dictionary_result to handle this type." + ) + + # Mapping results were not guaranteed to be generated on this version of VRS. + # Explicit field extraction for alleles is intentional: stored dicts may contain extra fields (e.g. "type" on + # Extension, "label" on SequenceReference) that the strict VRS Pydantic models forbid. In such cases, + # using model_validate() directly is not possible. + return Allele( + id=variation.get("id"), + state=state, + digest=variation.get("digest"), + location=SequenceLocation( + start=variation.get("location", {}).get("start"), + end=variation.get("location", {}).get("end"), + digest=variation.get("location", {}).get("digest"), + id=variation.get("location", {}).get("id"), + sequenceReference=SequenceReference( + name=variation.get("location", {}).get("sequenceReference", {}).get("name"), + refgetAccession=variation.get("location", {}).get("sequenceReference", {}).get("refgetAccession"), + ), + ), + extensions=[ + Extension( + id=extension.get("id"), + name=extension["name"], + description=extension.get("description"), + value=extension.get("value"), + ) + for extension in variation.get("extensions", []) + ], + expressions=[ + Expression( + id=expression.get("id"), + syntax=expression["syntax"], + syntax_version=expression.get("syntax_version"), + value=expression["value"], + ) + for expression in variation.get("expressions", []) + ], + ) + + +def vrs_object_from_mapped_variant(mapping_results: dict) -> MolecularVariation: + """ + Extracts a VRS (Variation Representation Specification) object from a stored ``post_mapped`` dict. + + This function processes a dictionary of mapping results and returns a VRS object, + which can either be an `Allele` or a `CisPhasedBlock`. The type of VRS object + returned depends on the "type" field in the input dictionary. + + Args: + mapping_results (dict): A dictionary containing the mapping results of a variant. + It must include a "type" key indicating the type of VRS object + ("CisPhasedBlock" or "Haplotype" for a CisPhasedBlock, or "Allele"). + If the type is "CisPhasedBlock" or "Haplotype", the dictionary must also + include a "members" key containing a list of member alleles. + + Returns: + MolecularVariation: A VRS object representing the mapped variant. This will be + either a `CisPhasedBlock` containing its member variants or an `Allele` + derived from the mapping results. + + Raises: + KeyError: If required keys are missing from the `mapping_results` dictionary. + """ + if mapping_results.get("type") == "CisPhasedBlock" or mapping_results.get("type") == "Haplotype": + return MolecularVariation( + # It's unclear why MyPy complains about the missing id field, so just add it as None (it is None by default anyway) + CisPhasedBlock( + id=None, + members=[allele_from_mapped_variant_dictionary_result(member) for member in mapping_results["members"]], + ) + ) + + return MolecularVariation(allele_from_mapped_variant_dictionary_result(mapping_results)) diff --git a/tests/lib/annotation/conftest.py b/tests/lib/annotation/conftest.py index 851f6fcf0..f9a9480fa 100644 --- a/tests/lib/annotation/conftest.py +++ b/tests/lib/annotation/conftest.py @@ -5,8 +5,12 @@ including mock objects with proper calibrations and configurations. """ +from types import SimpleNamespace + import pytest +from mavedb.lib.annotation.context import VariantAnnotationContext +from mavedb.lib.vrs import vrs_object_from_mapped_variant from tests.helpers.mocks.factories import ( create_mock_mapped_variant, create_mock_mapped_variant_with_functional_calibration_score_set, @@ -14,6 +18,32 @@ ) +def annotation_context_for(mapped_variant, subject_variant=None) -> VariantAnnotationContext: + """Build a DB-free ``VariantAnnotationContext`` around a mock mapped variant. + + The annotation builders read a context's ``variant`` (scores/calibrations), ``record`` (VRS-mapper + provenance), ``measured_allele`` (study-result focus + ClinGen IRI), and ``subject_variant`` (the VA + proposition subject). We synthesize the record/allele from the mock's mapping fields; the subject + defaults to the concrete measured variation, matching a non-projected variant. Mutating + ``context.variant`` mutates the underlying mock variant, since it is the same object. + """ + record = SimpleNamespace( + mapping_api_version=mapped_variant.mapping_api_version, + mapped_date=mapped_variant.mapped_date, + ) + measured_allele = SimpleNamespace( + post_mapped=mapped_variant.post_mapped, + clingen_allele_id=mapped_variant.clingen_allele_id, + ) + return VariantAnnotationContext( + variant=mapped_variant.variant, + record=record, + measured_allele=measured_allele, + subject_variant=subject_variant or vrs_object_from_mapped_variant(mapped_variant.post_mapped), + as_of=None, + ) + + @pytest.fixture def mock_mapped_variant(): """Override main fixture with properly configured mock for annotation tests.""" @@ -30,3 +60,25 @@ def mock_mapped_variant_with_functional_calibration_score_set(): def mock_mapped_variant_with_pathogenicity_calibration_score_set(): """Fixture for mock mapped variant with pathogenicity calibration score set.""" return create_mock_mapped_variant_with_pathogenicity_calibration_score_set(clingen_allele_id="CA123456") + + +@pytest.fixture +def mock_annotation_context(mock_mapped_variant) -> VariantAnnotationContext: + """A DB-free annotation context for an uncalibrated variant.""" + return annotation_context_for(mock_mapped_variant) + + +@pytest.fixture +def mock_annotation_context_with_functional_calibration_score_set( + mock_mapped_variant_with_functional_calibration_score_set, +) -> VariantAnnotationContext: + """A DB-free annotation context whose variant carries a functional calibration.""" + return annotation_context_for(mock_mapped_variant_with_functional_calibration_score_set) + + +@pytest.fixture +def mock_annotation_context_with_pathogenicity_calibration_score_set( + mock_mapped_variant_with_pathogenicity_calibration_score_set, +) -> VariantAnnotationContext: + """A DB-free annotation context whose variant carries a pathogenicity calibration.""" + return annotation_context_for(mock_mapped_variant_with_pathogenicity_calibration_score_set) diff --git a/tests/lib/annotation/test_annotate.py b/tests/lib/annotation/test_annotate.py index b05c2c18b..eca223df3 100644 --- a/tests/lib/annotation/test_annotate.py +++ b/tests/lib/annotation/test_annotate.py @@ -2,7 +2,9 @@ Tests for mavedb.lib.annotation.annotate module. This module tests the main annotation functions that create statements and study results -for variants, focusing on object structure and validation. +for variants, focusing on object structure and validation. The builders take a DB-free +``VariantAnnotationContext`` (assembled by the ``mock_annotation_context*`` fixtures); the real +substrate-backed context build is covered by the integration tests in ``test_util.py``. """ # ruff: noqa: E402 @@ -25,9 +27,9 @@ class TestVariantStudyResult: """Unit tests for variant study result creation.""" - def test_variant_study_result_creates_valid_result(self, mock_mapped_variant): + def test_variant_study_result_creates_valid_result(self, mock_annotation_context): """Test that variant study result creates a valid result object.""" - result = variant_study_result(mock_mapped_variant) + result = variant_study_result(mock_annotation_context) assert result is not None assert result.type == "ExperimentalVariantFunctionalImpactStudyResult" @@ -37,35 +39,35 @@ def test_variant_study_result_creates_valid_result(self, mock_mapped_variant): class TestVariantFunctionalImpactStatement: """Unit tests for variant functional impact statement creation.""" - def test_no_calibrations_returns_none(self, mock_mapped_variant): + def test_no_calibrations_returns_none(self, mock_annotation_context): """Test that statement returns None when no calibrations exist.""" - result = variant_functional_impact_statement(mock_mapped_variant) + result = variant_functional_impact_statement(mock_annotation_context) assert result is None def test_only_research_use_only_calibrations_returns_none( - self, mock_mapped_variant_with_functional_calibration_score_set + self, mock_annotation_context_with_functional_calibration_score_set ): """Test that statement returns None when only research use only primary calibrations exist.""" + context = mock_annotation_context_with_functional_calibration_score_set # Set all calibrations to research use only - for ( - calibration - ) in mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations: + for calibration in context.variant.score_set.score_calibrations: calibration.research_use_only = True - result = variant_functional_impact_statement(mock_mapped_variant_with_functional_calibration_score_set) + result = variant_functional_impact_statement(context) assert result is None - def test_no_score_returns_none(self, mock_mapped_variant_with_functional_calibration_score_set): + def test_no_score_returns_none(self, mock_annotation_context_with_functional_calibration_score_set): """Test that statement returns None when variant has no score.""" - mock_mapped_variant_with_functional_calibration_score_set.variant.data = {"score_data": {"score": None}} - result = variant_functional_impact_statement(mock_mapped_variant_with_functional_calibration_score_set) + context = mock_annotation_context_with_functional_calibration_score_set + context.variant.data = {"score_data": {"score": None}} + result = variant_functional_impact_statement(context) assert result is None - def test_valid_statement_creation(self, mock_mapped_variant_with_functional_calibration_score_set): + def test_valid_statement_creation(self, mock_annotation_context_with_functional_calibration_score_set): """Test creating valid functional impact statement with proper structure.""" - result = variant_functional_impact_statement(mock_mapped_variant_with_functional_calibration_score_set) + result = variant_functional_impact_statement(mock_annotation_context_with_functional_calibration_score_set) assert result is not None assert result.type == "Statement" @@ -77,38 +79,37 @@ def test_valid_statement_creation(self, mock_mapped_variant_with_functional_cali ) def test_skips_research_use_only_calibrations_when_mixed( - self, mock_mapped_variant_with_functional_calibration_score_set + self, mock_annotation_context_with_functional_calibration_score_set ): """Test that research-use-only calibrations are skipped when mixed with regular calibrations.""" - calibrations = mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations + context = mock_annotation_context_with_functional_calibration_score_set + calibrations = context.variant.score_set.score_calibrations mixed_calibrations = [deepcopy(calibrations[0]), deepcopy(calibrations[0])] mixed_calibrations[0].research_use_only = True mixed_calibrations[1].research_use_only = False - mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations = ( - mixed_calibrations - ) + context.variant.score_set.score_calibrations = mixed_calibrations - result = variant_functional_impact_statement(mock_mapped_variant_with_functional_calibration_score_set) + result = variant_functional_impact_statement(context) assert result is not None assert len(result.hasEvidenceLines) == 1 def test_variant_not_in_any_range_returns_indeterminate( - self, mock_mapped_variant_with_functional_calibration_score_set + self, mock_annotation_context_with_functional_calibration_score_set ): """Test that variant not in any functional range gets INDETERMINATE classification.""" from unittest.mock import patch from mavedb.lib.annotation.classification import ExperimentalVariantFunctionalImpactClassification - mapped_variant = mock_mapped_variant_with_functional_calibration_score_set + context = mock_annotation_context_with_functional_calibration_score_set # Mock functional_classification_of_variant to return None range (variant not in any range) with patch( "mavedb.lib.annotation.annotate.functional_classification_of_variant", return_value=(None, ExperimentalVariantFunctionalImpactClassification.INDETERMINATE), ): - result = variant_functional_impact_statement(mapped_variant) + result = variant_functional_impact_statement(context) assert result is not None assert result.type == "Statement" @@ -120,49 +121,52 @@ def test_variant_not_in_any_range_returns_indeterminate( class TestVariantPathogenicityStatement: """Unit tests for variant pathogenicity statement creation.""" - def test_no_calibrations_returns_none(self, mock_mapped_variant): + def test_no_calibrations_returns_none(self, mock_annotation_context): """Test that statement returns None when no calibrations exist.""" - result = variant_pathogenicity_statement(mock_mapped_variant) + result = variant_pathogenicity_statement(mock_annotation_context) assert result is None - def test_no_score_returns_none(self, mock_mapped_variant_with_pathogenicity_calibration_score_set): + def test_no_score_returns_none(self, mock_annotation_context_with_pathogenicity_calibration_score_set): """Test that statement returns None when variant has no score.""" - mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.data = {"score_data": {"score": None}} - result = variant_pathogenicity_statement(mock_mapped_variant_with_pathogenicity_calibration_score_set) + context = mock_annotation_context_with_pathogenicity_calibration_score_set + context.variant.data = {"score_data": {"score": None}} + result = variant_pathogenicity_statement(context) assert result is None def test_only_research_use_only_calibration_returns_none( - self, mock_mapped_variant_with_pathogenicity_calibration_score_set + self, mock_annotation_context_with_pathogenicity_calibration_score_set ): """Test that statement returns None when only research use only primary calibrations exist.""" + context = mock_annotation_context_with_pathogenicity_calibration_score_set # Set all calibrations to research use only - for ( - calibration - ) in mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations: + for calibration in context.variant.score_set.score_calibrations: calibration.research_use_only = True - result = variant_pathogenicity_statement(mock_mapped_variant_with_pathogenicity_calibration_score_set) + result = variant_pathogenicity_statement(context) assert result is None - def test_no_acmg_classifications_returns_none(self, mock_mapped_variant_with_pathogenicity_calibration_score_set): + def test_no_acmg_classifications_returns_none( + self, mock_annotation_context_with_pathogenicity_calibration_score_set + ): """Test that statement returns None when no ACMG classifications exist.""" + context = mock_annotation_context_with_pathogenicity_calibration_score_set # Remove ACMG classifications from all calibrations - for ( - calibration - ) in mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations: + for calibration in context.variant.score_set.score_calibrations: acmg_removed = [deepcopy(r) for r in calibration.functional_classifications] for functional_classification in acmg_removed: functional_classification["acmgClassification"] = None calibration.functional_classifications = acmg_removed - result = variant_pathogenicity_statement(mock_mapped_variant_with_pathogenicity_calibration_score_set) + result = variant_pathogenicity_statement(context) assert result is None - def test_valid_pathogenicity_statement_creation(self, mock_mapped_variant_with_pathogenicity_calibration_score_set): + def test_valid_pathogenicity_statement_creation( + self, mock_annotation_context_with_pathogenicity_calibration_score_set + ): """Test creating valid pathogenicity statement with proper structure.""" - result = variant_pathogenicity_statement(mock_mapped_variant_with_pathogenicity_calibration_score_set) + result = variant_pathogenicity_statement(mock_annotation_context_with_pathogenicity_calibration_score_set) assert result is not None assert result.proposition.type == "VariantPathogenicityProposition" @@ -186,27 +190,27 @@ def test_valid_pathogenicity_statement_creation(self, mock_mapped_variant_with_p ) def test_skips_research_use_only_calibrations_when_mixed( - self, mock_mapped_variant_with_pathogenicity_calibration_score_set + self, mock_annotation_context_with_pathogenicity_calibration_score_set ): """Test that research-use-only pathogenicity calibrations are skipped when mixed with regular calibrations.""" - calibrations = mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations + context = mock_annotation_context_with_pathogenicity_calibration_score_set + calibrations = context.variant.score_set.score_calibrations mixed_calibrations = [deepcopy(calibrations[0]), deepcopy(calibrations[0])] mixed_calibrations[0].research_use_only = True mixed_calibrations[1].research_use_only = False - mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations = ( - mixed_calibrations - ) + context.variant.score_set.score_calibrations = mixed_calibrations - result = variant_pathogenicity_statement(mock_mapped_variant_with_pathogenicity_calibration_score_set) + result = variant_pathogenicity_statement(context) assert result is not None assert len(result.hasEvidenceLines) == 1 def test_skips_invalid_calibrations_when_functional_annotation( - self, mock_mapped_variant_with_functional_calibration_score_set + self, mock_annotation_context_with_functional_calibration_score_set ): """Test that functional annotation skips calibrations invalid under score_calibration_may_be_used_for_annotation.""" - calibrations = mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations + context = mock_annotation_context_with_functional_calibration_score_set + calibrations = context.variant.score_set.score_calibrations mixed_calibrations = [deepcopy(calibrations[0]), deepcopy(calibrations[0])] # Invalid: no functional classifications @@ -214,20 +218,19 @@ def test_skips_invalid_calibrations_when_functional_annotation( # Valid: retain default functional classifications mixed_calibrations[1].functional_classifications = deepcopy(calibrations[0].functional_classifications) - mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations = ( - mixed_calibrations - ) + context.variant.score_set.score_calibrations = mixed_calibrations - result = variant_functional_impact_statement(mock_mapped_variant_with_functional_calibration_score_set) + result = variant_functional_impact_statement(context) assert result is not None assert len(result.hasEvidenceLines) == 1 def test_skips_invalid_calibrations_when_pathogenicity_annotation( - self, mock_mapped_variant_with_pathogenicity_calibration_score_set + self, mock_annotation_context_with_pathogenicity_calibration_score_set ): """Test that pathogenicity annotation skips calibrations invalid under score_calibration_may_be_used_for_annotation.""" - calibrations = mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations + context = mock_annotation_context_with_pathogenicity_calibration_score_set + calibrations = context.variant.score_set.score_calibrations mixed_calibrations = [deepcopy(calibrations[0]), deepcopy(calibrations[0])] # Invalid calibration: no functional classifications @@ -236,17 +239,15 @@ def test_skips_invalid_calibrations_when_pathogenicity_annotation( # Valid calibration retained mixed_calibrations[1].functional_classifications = deepcopy(calibrations[0].functional_classifications) - mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations = ( - mixed_calibrations - ) + context.variant.score_set.score_calibrations = mixed_calibrations - result = variant_pathogenicity_statement(mock_mapped_variant_with_pathogenicity_calibration_score_set) + result = variant_pathogenicity_statement(context) assert result is not None assert len(result.hasEvidenceLines) == 1 def test_variant_not_in_any_range_returns_uncertain_significance( - self, mock_mapped_variant_with_pathogenicity_calibration_score_set + self, mock_annotation_context_with_pathogenicity_calibration_score_set ): """Test that variant not in any range gets UNCERTAIN_SIGNIFICANCE classification.""" from unittest.mock import patch @@ -255,7 +256,7 @@ def test_variant_not_in_any_range_returns_uncertain_significance( from mavedb.lib.annotation.classification import ExperimentalVariantFunctionalImpactClassification - mapped_variant = mock_mapped_variant_with_pathogenicity_calibration_score_set + context = mock_annotation_context_with_pathogenicity_calibration_score_set # Mock both classification functions to return None range (variant not in any range) with ( @@ -264,11 +265,11 @@ def test_variant_not_in_any_range_returns_uncertain_significance( return_value=(None, ExperimentalVariantFunctionalImpactClassification.INDETERMINATE), ), patch( - "mavedb.lib.annotation.util.pathogenicity_classification_of_variant", + "mavedb.lib.annotation.calibration.pathogenicity_classification_of_variant", return_value=(None, VariantPathogenicityEvidenceLine.Criterion.PS3, None), ), ): - result = variant_pathogenicity_statement(mapped_variant) + result = variant_pathogenicity_statement(context) assert result is not None assert result.type == "Statement" @@ -276,7 +277,7 @@ def test_variant_not_in_any_range_returns_uncertain_significance( assert result.classification.primaryCoding.code.root == "uncertain significance" def test_pathogenicity_evidence_line_has_evidence_items_are_statement_instances( - self, mock_mapped_variant_with_pathogenicity_calibration_score_set + self, mock_annotation_context_with_pathogenicity_calibration_score_set ): """Regression test: hasEvidenceItems on VariantPathogenicityEvidenceLine must be model instances. @@ -284,7 +285,7 @@ def test_pathogenicity_evidence_line_has_evidence_items_are_statement_instances( VariantPathogenicityEvidenceLine validation to fail when reconstructing nested VRS objects (e.g. Allele with production genomic coordinates). Model instances must be stored directly. """ - result = variant_pathogenicity_statement(mock_mapped_variant_with_pathogenicity_calibration_score_set) + result = variant_pathogenicity_statement(mock_annotation_context_with_pathogenicity_calibration_score_set) assert result is not None for evidence_line in result.hasEvidenceLines: @@ -301,32 +302,28 @@ def test_pathogenicity_evidence_line_has_evidence_items_are_statement_instances( class TestVariantHighestLevelAnnotation: """Unit tests for the highest-materialized-layer resolver used by the public data dump.""" - def test_study_result_when_uncalibrated(self, mock_mapped_variant): - result = variant_highest_level_annotation(mock_mapped_variant) + def test_study_result_when_uncalibrated(self, mock_annotation_context): + result = variant_highest_level_annotation(mock_annotation_context) assert result is not None assert result.type == "ExperimentalVariantFunctionalImpactStudyResult" - def test_functional_statement_when_functional_only(self, mock_mapped_variant_with_functional_calibration_score_set): + def test_functional_statement_when_functional_only( + self, mock_annotation_context_with_functional_calibration_score_set + ): # The functional calibration fixture has no ACMG classifications, so the variant qualifies for the # functional layer but not pathogenicity. - result = variant_highest_level_annotation(mock_mapped_variant_with_functional_calibration_score_set) + result = variant_highest_level_annotation(mock_annotation_context_with_functional_calibration_score_set) assert result is not None assert result.type == "Statement" assert result.proposition.type == "ExperimentalVariantFunctionalImpactProposition" def test_pathogenicity_statement_when_calibrated( - self, mock_mapped_variant_with_pathogenicity_calibration_score_set + self, mock_annotation_context_with_pathogenicity_calibration_score_set ): - result = variant_highest_level_annotation(mock_mapped_variant_with_pathogenicity_calibration_score_set) + result = variant_highest_level_annotation(mock_annotation_context_with_pathogenicity_calibration_score_set) assert result is not None assert result.type == "Statement" assert result.proposition.type == "VariantPathogenicityProposition" - - def test_none_when_unmapped(self, mock_mapped_variant): - mock_mapped_variant.post_mapped = None - - result = variant_highest_level_annotation(mock_mapped_variant) - assert result is None diff --git a/tests/lib/annotation/test_calibration.py b/tests/lib/annotation/test_calibration.py new file mode 100644 index 000000000..1614ce819 --- /dev/null +++ b/tests/lib/annotation/test_calibration.py @@ -0,0 +1,344 @@ +# ruff: noqa: E402 + +"""Tests for mavedb.lib.annotation.calibration — calibration eligibility + strongest-evidence selection.""" + +from copy import deepcopy + +import pytest + +pytest.importorskip("psycopg2") + +from mavedb.lib.annotation.calibration import ( + score_calibration_may_be_used_for_annotation, + select_strongest_functional_calibration, + select_strongest_pathogenicity_calibration, +) + + +@pytest.mark.unit +class TestScoreCalibrationMayBeUsedForAnnotation: + def test_returns_false_for_research_use_only_when_not_allowed( + self, mock_mapped_variant_with_functional_calibration_score_set + ): + calibration = mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations[0] + calibration.research_use_only = True + + assert ( + score_calibration_may_be_used_for_annotation( + calibration, + annotation_type="functional", + allow_research_use_only_calibrations=False, + ) + is False + ) + + def test_returns_true_for_research_use_only_when_allowed( + self, mock_mapped_variant_with_functional_calibration_score_set + ): + calibration = mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations[0] + calibration.research_use_only = True + + assert ( + score_calibration_may_be_used_for_annotation( + calibration, + annotation_type="functional", + allow_research_use_only_calibrations=True, + ) + is True + ) + + def test_returns_false_when_functional_classifications_missing( + self, mock_mapped_variant_with_functional_calibration_score_set + ): + calibration = mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations[0] + calibration.functional_classifications = [] + + assert score_calibration_may_be_used_for_annotation(calibration, annotation_type="functional") is False + + def test_returns_false_for_pathogenicity_without_acmg_classifications( + self, mock_mapped_variant_with_pathogenicity_calibration_score_set + ): + calibration = mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations[ + 0 + ] + acmg_removed = [deepcopy(fc) for fc in calibration.functional_classifications] + for functional_classification in acmg_removed: + functional_classification["acmgClassification"] = None + calibration.functional_classifications = acmg_removed + + assert score_calibration_may_be_used_for_annotation(calibration, annotation_type="pathogenicity") is False + + def test_returns_true_for_pathogenicity_with_any_acmg_classification( + self, mock_mapped_variant_with_pathogenicity_calibration_score_set + ): + calibration = mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations[ + 0 + ] + + assert score_calibration_may_be_used_for_annotation(calibration, annotation_type="pathogenicity") is True + + +@pytest.mark.unit +class TestSelectStrongestFunctionalCalibrationUnit: + """Unit tests for select_strongest_functional_calibration function.""" + + def test_returns_none_for_empty_calibrations(self, mock_mapped_variant): + """Test that empty calibration list returns None.""" + calibration, functional_range = select_strongest_functional_calibration(mock_mapped_variant.variant, []) + assert calibration is None + assert functional_range is None + + def test_returns_single_calibration(self, mock_mapped_variant_with_functional_calibration_score_set): + """Test that single calibration is returned.""" + variant = mock_mapped_variant_with_functional_calibration_score_set.variant + calibrations = variant.score_set.score_calibrations + + calibration, functional_range = select_strongest_functional_calibration(variant, calibrations) + + assert calibration is not None + assert calibration == calibrations[0] + assert functional_range is not None + + def test_returns_first_when_all_agree(self, mock_mapped_variant_with_functional_calibration_score_set): + """Test that first calibration is returned when all have same classification.""" + variant = mock_mapped_variant_with_functional_calibration_score_set.variant + # Create multiple calibrations with same classification + calibration1 = variant.score_set.score_calibrations[0] + calibration2 = deepcopy(calibration1) + calibration2.id = 999 + calibrations = [calibration1, calibration2] + + calibration, functional_range = select_strongest_functional_calibration(variant, calibrations) + + assert calibration is not None + assert calibration == calibrations[0] # Should return the first one + + def test_defaults_to_normal_on_conflict(self, mock_mapped_variant_with_functional_calibration_score_set): + """Test that normal classification is preferred when there are conflicts.""" + from unittest.mock import MagicMock, patch + + from mavedb.lib.annotation.classification import ExperimentalVariantFunctionalImpactClassification + + variant = mock_mapped_variant_with_functional_calibration_score_set.variant + calibration1 = variant.score_set.score_calibrations[0] + calibration2 = deepcopy(calibration1) + calibration2.id = 999 + calibrations = [calibration1, calibration2] + + # Mock to return different classifications + with patch( + "mavedb.lib.annotation.calibration.functional_classification_of_variant", + side_effect=[ + (MagicMock(label="Abnormal"), ExperimentalVariantFunctionalImpactClassification.ABNORMAL), + (MagicMock(label="Normal"), ExperimentalVariantFunctionalImpactClassification.NORMAL), + ], + ): + calibration, functional_range = select_strongest_functional_calibration(variant, calibrations) + + # Should return the normal classification (second one) + assert calibration == calibration2 + assert functional_range.label == "Normal" + + def test_returns_first_calibration_when_no_variants_in_ranges( + self, mock_mapped_variant_with_functional_calibration_score_set + ): + """Test that first calibration with None range is returned when variant is not in any functional range.""" + from unittest.mock import patch + + from mavedb.lib.annotation.classification import ExperimentalVariantFunctionalImpactClassification + + variant = mock_mapped_variant_with_functional_calibration_score_set.variant + calibrations = variant.score_set.score_calibrations + + # Mock to return None range but INDETERMINATE classification (variant not in any range) + with patch( + "mavedb.lib.annotation.calibration.functional_classification_of_variant", + return_value=(None, ExperimentalVariantFunctionalImpactClassification.INDETERMINATE), + ): + calibration, functional_range = select_strongest_functional_calibration(variant, calibrations) + + # Should return first calibration with None range (indicating variant not in any range) + assert calibration == calibrations[0] + assert functional_range is None + + +@pytest.mark.unit +class TestSelectStrongestPathogenicityCalibrationUnit: + """Unit tests for select_strongest_pathogenicity_calibration function.""" + + def test_returns_none_for_empty_calibrations(self, mock_mapped_variant): + """Test that empty calibration list returns None.""" + calibration, functional_range = select_strongest_pathogenicity_calibration(mock_mapped_variant.variant, []) + assert calibration is None + assert functional_range is None + + def test_returns_single_calibration(self, mock_mapped_variant_with_pathogenicity_calibration_score_set): + """Test that single calibration is returned.""" + variant = mock_mapped_variant_with_pathogenicity_calibration_score_set.variant + calibrations = variant.score_set.score_calibrations + + calibration, functional_range = select_strongest_pathogenicity_calibration(variant, calibrations) + + assert calibration is not None + assert calibration == calibrations[0] + assert functional_range is not None + + def test_selects_strongest_evidence_strength(self, mock_mapped_variant_with_pathogenicity_calibration_score_set): + """Test that calibration with strongest evidence is selected.""" + from unittest.mock import MagicMock, patch + + from ga4gh.va_spec.acmg_2015 import VariantPathogenicityEvidenceLine + from ga4gh.va_spec.base.enums import StrengthOfEvidenceProvided + + variant = mock_mapped_variant_with_pathogenicity_calibration_score_set.variant + calibration1 = variant.score_set.score_calibrations[0] + calibration2 = deepcopy(calibration1) + calibration2.id = 999 + calibrations = [calibration1, calibration2] + + # Mock to return different evidence strengths + with patch( + "mavedb.lib.annotation.calibration.pathogenicity_classification_of_variant", + side_effect=[ + ( + MagicMock(label="Moderate"), + VariantPathogenicityEvidenceLine.Criterion.PS3, + StrengthOfEvidenceProvided.MODERATE, + ), + ( + MagicMock(label="Strong"), + VariantPathogenicityEvidenceLine.Criterion.PS3, + StrengthOfEvidenceProvided.VERY_STRONG, + ), + ], + ): + calibration, functional_range = select_strongest_pathogenicity_calibration(variant, calibrations) + + # Should return the one with VERY_STRONG evidence + assert calibration == calibration2 + assert functional_range.label == "Strong" + + def test_defaults_to_uncertain_on_tie(self, mock_mapped_variant_with_pathogenicity_calibration_score_set): + """Test that uncertain significance is returned when benign and pathogenic evidence tie.""" + from unittest.mock import MagicMock, patch + + from ga4gh.va_spec.acmg_2015 import VariantPathogenicityEvidenceLine + from ga4gh.va_spec.base.enums import StrengthOfEvidenceProvided + + variant = mock_mapped_variant_with_pathogenicity_calibration_score_set.variant + calibration1 = variant.score_set.score_calibrations[0] + calibration2 = deepcopy(calibration1) + calibration2.id = 999 + calibrations = [calibration1, calibration2] + + # Mock to return same evidence strength but different criteria (pathogenic vs benign) + with patch( + "mavedb.lib.annotation.calibration.pathogenicity_classification_of_variant", + side_effect=[ + ( + MagicMock(label="Pathogenic"), + VariantPathogenicityEvidenceLine.Criterion.PS3, + StrengthOfEvidenceProvided.STRONG, + ), + ( + MagicMock(label="Benign"), + VariantPathogenicityEvidenceLine.Criterion.BS3, + StrengthOfEvidenceProvided.STRONG, + ), + ], + ): + calibration, functional_range = select_strongest_pathogenicity_calibration(variant, calibrations) + + # Should return first calibration but None for range to indicate uncertain significance + assert calibration == calibration1 + assert functional_range is None + + def test_returns_classification_when_all_tied_are_same_type( + self, mock_mapped_variant_with_pathogenicity_calibration_score_set + ): + """Test that classification is returned normally when all tied candidates are the same type.""" + from unittest.mock import MagicMock, patch + + from ga4gh.va_spec.acmg_2015 import VariantPathogenicityEvidenceLine + from ga4gh.va_spec.base.enums import StrengthOfEvidenceProvided + + variant = mock_mapped_variant_with_pathogenicity_calibration_score_set.variant + calibration1 = variant.score_set.score_calibrations[0] + calibration2 = deepcopy(calibration1) + calibration2.id = 999 + calibrations = [calibration1, calibration2] + + # Mock to return same evidence strength and same type of criteria (both benign) + with patch( + "mavedb.lib.annotation.calibration.pathogenicity_classification_of_variant", + side_effect=[ + ( + MagicMock(label="Benign1"), + VariantPathogenicityEvidenceLine.Criterion.BS3, + StrengthOfEvidenceProvided.STRONG, + ), + ( + MagicMock(label="Benign2"), + VariantPathogenicityEvidenceLine.Criterion.BP1, + StrengthOfEvidenceProvided.STRONG, + ), + ], + ): + calibration, functional_range = select_strongest_pathogenicity_calibration(variant, calibrations) + + # Should return first calibration with its range (no conflict, so normal classification) + assert calibration == calibration1 + assert functional_range.label == "Benign1" + + def test_handles_none_evidence_strength(self, mock_mapped_variant_with_pathogenicity_calibration_score_set): + """Test that None evidence strength is handled correctly.""" + from unittest.mock import MagicMock, patch + + from ga4gh.va_spec.acmg_2015 import VariantPathogenicityEvidenceLine + from ga4gh.va_spec.base.enums import StrengthOfEvidenceProvided + + variant = mock_mapped_variant_with_pathogenicity_calibration_score_set.variant + calibration1 = variant.score_set.score_calibrations[0] + calibration2 = deepcopy(calibration1) + calibration2.id = 999 + calibrations = [calibration1, calibration2] + + # Mock with None and actual strength + with patch( + "mavedb.lib.annotation.calibration.pathogenicity_classification_of_variant", + side_effect=[ + (MagicMock(label="No Strength"), VariantPathogenicityEvidenceLine.Criterion.PS3, None), + ( + MagicMock(label="Moderate"), + VariantPathogenicityEvidenceLine.Criterion.PS3, + StrengthOfEvidenceProvided.MODERATE, + ), + ], + ): + calibration, functional_range = select_strongest_pathogenicity_calibration(variant, calibrations) + + # Should return the one with actual strength (second) + assert calibration == calibration2 + assert functional_range.label == "Moderate" + + def test_returns_first_calibration_when_no_variants_in_ranges( + self, mock_mapped_variant_with_pathogenicity_calibration_score_set + ): + """Test that first calibration with None range is returned when variant is not in any functional range.""" + from unittest.mock import patch + + from ga4gh.va_spec.acmg_2015 import VariantPathogenicityEvidenceLine + + variant = mock_mapped_variant_with_pathogenicity_calibration_score_set.variant + calibrations = variant.score_set.score_calibrations + + # Mock to return None range for all calibrations (variant not in any range) + with patch( + "mavedb.lib.annotation.calibration.pathogenicity_classification_of_variant", + return_value=(None, VariantPathogenicityEvidenceLine.Criterion.PS3, None), + ): + calibration, functional_range = select_strongest_pathogenicity_calibration(variant, calibrations) + + # Should return first calibration with None range (indicating variant not in any range) + assert calibration == calibrations[0] + assert functional_range is None diff --git a/tests/lib/annotation/test_classification.py b/tests/lib/annotation/test_classification.py index 82cd90684..fdbaf23cb 100644 --- a/tests/lib/annotation/test_classification.py +++ b/tests/lib/annotation/test_classification.py @@ -63,7 +63,7 @@ def test_normal_classification(self): functional_range = create_mock_functional_classification(FunctionalClassificationOptions.normal) score_calibration = create_mock_score_calibration(functional_classifications=[functional_range]) - result = functional_classification_of_variant(mapped_variant, score_calibration) + result = functional_classification_of_variant(mapped_variant.variant, score_calibration) assert result[0] == functional_range assert result[1] == ExperimentalVariantFunctionalImpactClassification.NORMAL @@ -76,7 +76,7 @@ def test_abnormal_classification(self): functional_range = create_mock_functional_classification(FunctionalClassificationOptions.abnormal) score_calibration = create_mock_score_calibration(functional_classifications=[functional_range]) - result = functional_classification_of_variant(mapped_variant, score_calibration) + result = functional_classification_of_variant(mapped_variant.variant, score_calibration) assert result[0] == functional_range assert result[1] == ExperimentalVariantFunctionalImpactClassification.ABNORMAL @@ -89,7 +89,7 @@ def test_indeterminate_classification(self): functional_range = create_mock_functional_classification(FunctionalClassificationOptions.not_specified) score_calibration = create_mock_score_calibration(functional_classifications=[functional_range]) - result = functional_classification_of_variant(mapped_variant, score_calibration) + result = functional_classification_of_variant(mapped_variant.variant, score_calibration) assert result[0] == functional_range assert result[1] == ExperimentalVariantFunctionalImpactClassification.INDETERMINATE @@ -104,7 +104,7 @@ def test_variant_not_in_any_range(self): ) score_calibration = create_mock_score_calibration(functional_classifications=[functional_range]) - result = functional_classification_of_variant(mapped_variant, score_calibration) + result = functional_classification_of_variant(mapped_variant.variant, score_calibration) assert result[0] is None assert result[1] == ExperimentalVariantFunctionalImpactClassification.INDETERMINATE @@ -122,7 +122,7 @@ def test_multiple_ranges_returns_first_match(self): second_range = create_mock_functional_classification(FunctionalClassificationOptions.abnormal) score_calibration = create_mock_score_calibration(functional_classifications=[first_range, second_range]) - result = functional_classification_of_variant(mapped_variant, score_calibration) + result = functional_classification_of_variant(mapped_variant.variant, score_calibration) assert result[0] == first_range assert result[1] == ExperimentalVariantFunctionalImpactClassification.NORMAL @@ -134,7 +134,7 @@ def test_missing_score_calibrations_raises_error(self): score_calibration = create_mock_score_calibration() with pytest.raises(ValueError, match="does not have a score set with score calibrations"): - functional_classification_of_variant(mapped_variant, score_calibration) + functional_classification_of_variant(mapped_variant.variant, score_calibration) def test_empty_score_calibrations_raises_error(self): """Test that empty score calibrations list raises ValueError.""" @@ -143,7 +143,7 @@ def test_empty_score_calibrations_raises_error(self): score_calibration = create_mock_score_calibration() with pytest.raises(ValueError, match="does not have a score set with score calibrations"): - functional_classification_of_variant(mapped_variant, score_calibration) + functional_classification_of_variant(mapped_variant.variant, score_calibration) def test_missing_functional_classifications_raises_error(self): """Test that missing functional classifications raises ValueError.""" @@ -152,7 +152,7 @@ def test_missing_functional_classifications_raises_error(self): score_calibration = create_mock_score_calibration(functional_classifications=[]) with pytest.raises(ValueError, match="does not have ranges defined in its primary score calibration"): - functional_classification_of_variant(mapped_variant, score_calibration) + functional_classification_of_variant(mapped_variant.variant, score_calibration) @pytest.mark.unit @@ -168,7 +168,7 @@ def test_valid_pathogenicity_classification(self): pathogenicity_range = create_mock_pathogenicity_range(acmg_classification) score_calibration = create_mock_score_calibration(functional_classifications=[pathogenicity_range]) - result = pathogenicity_classification_of_variant(mapped_variant, score_calibration) + result = pathogenicity_classification_of_variant(mapped_variant.variant, score_calibration) assert result[0] == pathogenicity_range assert result[1] == VariantPathogenicityEvidenceLine.Criterion.PS3 @@ -186,7 +186,7 @@ def test_moderate_plus_maps_to_moderate(self): pathogenicity_range = create_mock_pathogenicity_range(acmg_classification) score_calibration = create_mock_score_calibration(functional_classifications=[pathogenicity_range]) - result = pathogenicity_classification_of_variant(mapped_variant, score_calibration) + result = pathogenicity_classification_of_variant(mapped_variant.variant, score_calibration) # Verify the mapped evidence strength is MODERATE, not MODERATE_PLUS assert result[0] == pathogenicity_range @@ -201,7 +201,7 @@ def test_none_acmg_classification(self): pathogenicity_range = create_mock_pathogenicity_range(acmg_classification=None) score_calibration = create_mock_score_calibration(functional_classifications=[pathogenicity_range]) - result = pathogenicity_classification_of_variant(mapped_variant, score_calibration) + result = pathogenicity_classification_of_variant(mapped_variant.variant, score_calibration) assert result[0] == pathogenicity_range assert result[1] == VariantPathogenicityEvidenceLine.Criterion.PS3 @@ -216,7 +216,7 @@ def test_variant_not_in_any_range(self): pathogenicity_range = create_mock_pathogenicity_range(acmg_classification, variant_in_range=False) score_calibration = create_mock_score_calibration(functional_classifications=[pathogenicity_range]) - result = pathogenicity_classification_of_variant(mapped_variant, score_calibration) + result = pathogenicity_classification_of_variant(mapped_variant.variant, score_calibration) assert result[0] is None assert result[1] == VariantPathogenicityEvidenceLine.Criterion.PS3 @@ -229,7 +229,7 @@ def test_missing_score_calibrations_raises_error(self): score_calibration = create_mock_score_calibration() with pytest.raises(ValueError, match="does not have a score set with score calibrations"): - pathogenicity_classification_of_variant(mapped_variant, score_calibration) + pathogenicity_classification_of_variant(mapped_variant.variant, score_calibration) def test_empty_score_calibrations_raises_error(self): """Test that empty score calibrations list raises ValueError.""" @@ -238,7 +238,7 @@ def test_empty_score_calibrations_raises_error(self): score_calibration = create_mock_score_calibration() with pytest.raises(ValueError, match="does not have a score set with score calibrations"): - pathogenicity_classification_of_variant(mapped_variant, score_calibration) + pathogenicity_classification_of_variant(mapped_variant.variant, score_calibration) def test_missing_functional_classifications_raises_error(self): """Test that missing functional classifications raises ValueError.""" @@ -247,7 +247,7 @@ def test_missing_functional_classifications_raises_error(self): score_calibration = create_mock_score_calibration(functional_classifications=[]) with pytest.raises(ValueError, match="does not have ranges defined in its primary score calibration"): - pathogenicity_classification_of_variant(mapped_variant, score_calibration) + pathogenicity_classification_of_variant(mapped_variant.variant, score_calibration) def test_moderate_plus_evidence_strength_maps_to_moderate(self): """Test that MODERATE_PLUS evidence strength maps to Moderate.""" @@ -258,7 +258,7 @@ def test_moderate_plus_evidence_strength_maps_to_moderate(self): pathogenicity_range = create_mock_pathogenicity_range(acmg_classification) score_calibration = create_mock_score_calibration(functional_classifications=[pathogenicity_range]) - result = pathogenicity_classification_of_variant(mapped_variant, score_calibration) + result = pathogenicity_classification_of_variant(mapped_variant.variant, score_calibration) assert result[0] == pathogenicity_range assert result[1] == VariantPathogenicityEvidenceLine.Criterion.PS3 @@ -282,7 +282,7 @@ def test_various_valid_acmg_combinations(self, criterion, strength): pathogenicity_range = create_mock_pathogenicity_range(acmg_classification) score_calibration = create_mock_score_calibration(functional_classifications=[pathogenicity_range]) - result = pathogenicity_classification_of_variant(mapped_variant, score_calibration) + result = pathogenicity_classification_of_variant(mapped_variant.variant, score_calibration) assert result[0] == pathogenicity_range assert result[1] == VariantPathogenicityEvidenceLine.Criterion[criterion] @@ -301,7 +301,7 @@ def test_multiple_ranges_returns_first_match(self): score_calibration = create_mock_score_calibration(functional_classifications=[first_range, second_range]) - result = pathogenicity_classification_of_variant(mapped_variant, score_calibration) + result = pathogenicity_classification_of_variant(mapped_variant.variant, score_calibration) assert result[0] == first_range assert result[1] == VariantPathogenicityEvidenceLine.Criterion.PS3 diff --git a/tests/lib/annotation/test_context.py b/tests/lib/annotation/test_context.py new file mode 100644 index 000000000..e75256512 --- /dev/null +++ b/tests/lib/annotation/test_context.py @@ -0,0 +1,108 @@ +# ruff: noqa: E402 + +"""Tests for mavedb.lib.annotation.context — the VA proposition-subject grain (Slice 5.1).""" + +import pytest + +pytest.importorskip("psycopg2") + +from ga4gh.cat_vrs.models import CategoricalVariant +from ga4gh.vrs.models import MolecularVariation + +from mavedb.lib.annotation.context import variant_annotation_context +from tests.helpers.constants import TEST_VALID_POST_MAPPED_VRS_ALLELE +from tests.helpers.util.annotation import AlleleSpec, seed_mapping_record + + +@pytest.mark.integration +class TestVariantAnnotationContextSubject: + """variant_annotation_context — the VA *proposition* subject grain (Slice 5.1). + + The subject always anchors on the **measured** allele. A protein assay unfurls the full equivalence + class of encoders; a nucleotide assay keeps its precise coordinate partner + protein consequence and + excludes the sibling encoders (distinct variants) that the reverse-translation fan left on the record. + Either way the subject is a Cat-VRS ``CategoricalVariant`` when a projection member exists; it falls + back to the concrete measured ``MolecularVariation`` when the categorical can't be assembled. + """ + + def test_protein_assay_subject_is_a_categorical_variant(self, session, setup_lib_db_with_mapped_variant): + mapped_variant = setup_lib_db_with_mapped_variant + seed_mapping_record( + session, + mapped_variant.variant, + assay_level="protein", + alleles=[ + AlleleSpec( + digest="prot", level="protein", is_authoritative=True, post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE + ), + AlleleSpec(digest="cdna", level="cdna", post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE), + ], + ) + + context = variant_annotation_context(session, mapped_variant.variant) + + assert context is not None + assert isinstance(context.subject_variant, CategoricalVariant) + + def test_nucleotide_assay_subject_is_a_categorical_variant_over_the_measured_change( + self, session, setup_lib_db_with_mapped_variant + ): + """Anchored on the measured nt change: keep its projection_group coordinate partner + the protein + consequence, exclude the sibling encoder (a distinct variant, different projection group).""" + mapped_variant = setup_lib_db_with_mapped_variant + seed_mapping_record( + session, + mapped_variant.variant, + assay_level="cdna", + alleles=[ + AlleleSpec( + digest="cdna", + level="cdna", + is_authoritative=True, + projection_group=0, + post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE, + ), + # The measured change's precise coordinate partner (same projection group). + AlleleSpec( + digest="gen", level="genomic", projection_group=0, post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE + ), + # A sibling encoder of the same protein consequence — distinct variant, other group: excluded. + AlleleSpec( + digest="sibling", level="cdna", projection_group=1, post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE + ), + AlleleSpec(digest="prot", level="protein", post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE), + ], + ) + + context = variant_annotation_context(session, mapped_variant.variant) + + assert context is not None + assert isinstance(context.subject_variant, CategoricalVariant) + # defining cdna + gen partner + protein apex; the sibling encoder is dropped. + assert len(context.subject_variant.members) == 3 + + def test_single_allele_subject_falls_back_to_molecular_variation(self, session, setup_lib_db_with_mapped_variant): + """A lone measured allele (no projection member) → the categorical build yields the concrete allele.""" + mapped_variant = setup_lib_db_with_mapped_variant + seed_mapping_record( + session, + mapped_variant.variant, + assay_level="cdna", + alleles=[ + AlleleSpec( + digest="cdna", level="cdna", is_authoritative=True, post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE + ), + ], + ) + + context = variant_annotation_context(session, mapped_variant.variant) + + assert context is not None + assert isinstance(context.subject_variant, MolecularVariation) + assert not isinstance(context.subject_variant, CategoricalVariant) + + def test_unmapped_variant_has_no_context(self, session, setup_lib_db_with_mapped_variant): + """No live mapping record on the new substrate → no annotation context.""" + context = variant_annotation_context(session, setup_lib_db_with_mapped_variant.variant) + + assert context is None diff --git a/tests/lib/annotation/test_document.py b/tests/lib/annotation/test_document.py index 0a9b0ac86..d87e9b227 100644 --- a/tests/lib/annotation/test_document.py +++ b/tests/lib/annotation/test_document.py @@ -8,6 +8,7 @@ """ import urllib.parse +from types import SimpleNamespace import pytest @@ -19,8 +20,7 @@ from mavedb.lib.annotation.document import ( experiment_as_iri, experiment_to_document, - mapped_variant_as_iri, - mapped_variant_to_document, + measured_allele_as_iri, score_calibration_as_document, score_set_as_iri, score_set_to_document, @@ -81,42 +81,24 @@ def test_score_set_to_document(self, mock_score_set): @pytest.mark.unit -class TestMappedVariantDocumentFunctions: - """Unit tests for mapped variant document creation functions.""" +class TestMeasuredAlleleDocumentFunctions: + """Unit tests for measured-allele IRI generation.""" - def test_mapped_variant_as_iri(self, mock_mapped_variant): - """Test IRI generation for mapped variants with ClinGen allele ID.""" - expected_iri_root = ( - f"https://mavedb.org/variant/{urllib.parse.quote_plus(mock_mapped_variant.clingen_allele_id)}" - ) - result = mapped_variant_as_iri(mock_mapped_variant) + def test_measured_allele_as_iri(self): + """Test IRI generation for a measured allele with a ClinGen allele ID.""" + allele = SimpleNamespace(clingen_allele_id="CA123456") + expected_iri_root = f"https://mavedb.org/variant/{urllib.parse.quote_plus(allele.clingen_allele_id)}" + result = measured_allele_as_iri(allele) assert result.root == expected_iri_root - def test_mapped_variant_as_iri_no_caid(self, mock_mapped_variant): - """Test IRI generation for mapped variants without ClinGen allele ID returns None.""" - mock_mapped_variant.clingen_allele_id = None - result = mapped_variant_as_iri(mock_mapped_variant) + def test_measured_allele_as_iri_no_caid(self): + """Test IRI generation for a measured allele without a ClinGen allele ID returns None.""" + allele = SimpleNamespace(clingen_allele_id=None) + result = measured_allele_as_iri(allele) assert result is None - def test_mapped_variant_to_document(self, mock_mapped_variant): - """Test document creation for mapped variants with ClinGen allele ID.""" - document = mapped_variant_to_document(mock_mapped_variant) - - assert document.id == mock_mapped_variant.variant.urn - assert document.name == "MaveDB Mapped Variant" - assert document.documentType == "mapped genomic variant description" - assert len(document.urls) > 0 - assert mapped_variant_as_iri(mock_mapped_variant).root in document.urls - - def test_mapped_variant_to_document_no_caid(self, mock_mapped_variant): - """Test document creation for mapped variants without ClinGen allele ID returns None.""" - mock_mapped_variant.clingen_allele_id = None - document = mapped_variant_to_document(mock_mapped_variant) - - assert document is None - @pytest.mark.unit class TestVariantDocumentFunctions: diff --git a/tests/lib/annotation/test_eligibility.py b/tests/lib/annotation/test_eligibility.py new file mode 100644 index 000000000..239ed5af3 --- /dev/null +++ b/tests/lib/annotation/test_eligibility.py @@ -0,0 +1,375 @@ +# ruff: noqa: E402 + +"""Tests for mavedb.lib.annotation.eligibility — variant annotatability predicates.""" + +from copy import deepcopy +from unittest.mock import patch + +import pytest + +pytest.importorskip("psycopg2") + +from mavedb.lib.annotation.eligibility import ( + _can_annotate_variant_base_assumptions, + _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation, + can_annotate_variant_for_functional_statement, + can_annotate_variant_for_pathogenicity_evidence, +) + + +@pytest.mark.unit +class TestBaseAnnotationAssumptionsUnit: + def test_base_assumption_check_returns_false_when_score_is_none(self, mock_mapped_variant): + mock_mapped_variant.variant.data = {"score_data": {"score": None}} + + assert _can_annotate_variant_base_assumptions(mock_mapped_variant.variant) is False + + def test_base_assumption_check_returns_true_when_all_conditions_met(self, mock_mapped_variant): + assert _can_annotate_variant_base_assumptions(mock_mapped_variant.variant) is True + + +@pytest.mark.unit +class TestVariantScoreCalibrationsHaveRequiredCalibrationsAndRangesForAnnotation: + """ + Unit tests for the _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation function. + This function is used by both functional and pathogenicity annotation checks, so we test it separately here to avoid duplication in the tests for those checks. + """ + + @pytest.mark.parametrize("kind", ["functional", "pathogenicity"], ids=["functional", "pathogenicity"]) + def test_score_range_check_returns_false_when_calibrations_are_none(self, mock_mapped_variant, kind): + mock_mapped_variant.variant.score_set.score_calibrations = None + assert ( + _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( + mock_mapped_variant.variant, kind + ) + is False + ) + + @pytest.mark.parametrize("kind", ["functional", "pathogenicity"], ids=["functional", "pathogenicity"]) + def test_score_range_check_returns_false_when_no_calibrations_present(self, mock_mapped_variant, kind): + mock_mapped_variant.variant.score_set.score_calibrations = [] + assert ( + _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( + mock_mapped_variant.variant, kind + ) + is False + ) + + @pytest.mark.parametrize("annotation_type", ["functional", "pathogenicity"], ids=["functional", "pathogenicity"]) + def test_score_range_check_returns_false_when_all_calibrations_are_research_use_only_and_not_allowed( + self, mock_mapped_variant_with_functional_calibration_score_set, annotation_type + ): + """Test that research use only calibrations are excluded by default.""" + # Make all calibrations research use only + for ( + calibration + ) in mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations: + calibration.research_use_only = True + + assert ( + _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( + mock_mapped_variant_with_functional_calibration_score_set.variant, annotation_type + ) + is False + ) + + @pytest.mark.parametrize( + "kind,variant_fixture", + [ + ("functional", "mock_mapped_variant_with_functional_calibration_score_set"), + ("pathogenicity", "mock_mapped_variant_with_pathogenicity_calibration_score_set"), + ], + ids=["functional_fixture", "pathogenicity_fixture"], + ) + def test_score_range_check_returns_true_when_research_use_only_calibrations_are_allowed( + self, kind, variant_fixture, request + ): + """Test that research use only calibrations are included when explicitly allowed.""" + mock_mapped_variant = request.getfixturevalue(variant_fixture) + # Make all calibrations research use only + for calibration in mock_mapped_variant.variant.score_set.score_calibrations: + calibration.primary = False + calibration.research_use_only = True + + assert ( + _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( + mock_mapped_variant.variant, kind, allow_research_use_only_calibrations=True + ) + is True + ) + + @pytest.mark.parametrize( + "kind,variant_fixture", + [ + ("functional", "mock_mapped_variant_with_functional_calibration_score_set"), + ("pathogenicity", "mock_mapped_variant_with_pathogenicity_calibration_score_set"), + ], + ids=["functional_fixture", "pathogenicity_fixture"], + ) + def test_score_range_check_returns_false_when_calibrations_present_with_empty_ranges( + self, kind, variant_fixture, request + ): + mock_mapped_variant = request.getfixturevalue(variant_fixture) + + for calibration in mock_mapped_variant.variant.score_set.score_calibrations: + calibration.functional_classifications = None + + assert ( + _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( + mock_mapped_variant.variant, kind + ) + is False + ) + + def test_pathogenicity_range_check_returns_false_when_no_acmg_calibration( + self, + mock_mapped_variant_with_pathogenicity_calibration_score_set, + ): + for ( + calibration + ) in mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations: + acmg_classification_removed = [deepcopy(r) for r in calibration.functional_classifications] + for fr in acmg_classification_removed: + fr["acmgClassification"] = None + + calibration.functional_classifications = acmg_classification_removed + + assert ( + _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( + mock_mapped_variant_with_pathogenicity_calibration_score_set.variant, "pathogenicity" + ) + is False + ) + + def test_pathogenicity_range_check_returns_true_when_some_acmg_calibration( + self, + mock_mapped_variant_with_pathogenicity_calibration_score_set, + ): + for ( + calibration + ) in mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations: + acmg_classification_removed = [deepcopy(r) for r in calibration.functional_classifications] + acmg_classification_removed[0]["acmgClassification"] = None + + calibration.functional_classifications = acmg_classification_removed + + assert ( + _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( + mock_mapped_variant_with_pathogenicity_calibration_score_set.variant, "pathogenicity" + ) + is True + ) + + @pytest.mark.parametrize( + "kind,variant_fixture", + [ + ("functional", "mock_mapped_variant_with_functional_calibration_score_set"), + ("pathogenicity", "mock_mapped_variant_with_pathogenicity_calibration_score_set"), + ], + ids=["functional_fixture", "pathogenicity_fixture"], + ) + def test_score_range_check_returns_true_when_calibration_kind_exists_with_ranges( + self, kind, variant_fixture, request + ): + mock_mapped_variant = request.getfixturevalue(variant_fixture) + + assert ( + _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( + mock_mapped_variant.variant, kind + ) + is True + ) + + def test_score_range_check_returns_true_when_mixed_research_use_calibrations_exist_functional( + self, mock_mapped_variant_with_functional_calibration_score_set + ): + """Test behavior with mixed research use only and regular calibrations for functional annotation.""" + calibrations = mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations + + # If there's only one calibration, add another for testing + if len(calibrations) == 1: + # Create a copy of the existing calibration + new_calibration = deepcopy(calibrations[0]) + calibrations.append(new_calibration) + + # Make the first one research use only, leave the second as regular + calibrations[0].research_use_only = True + calibrations[1].research_use_only = False + + # Should return True because at least one non-research-only calibration has valid classifications + assert ( + _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( + mock_mapped_variant_with_functional_calibration_score_set.variant, "functional" + ) + is True + ) + + def test_score_range_check_returns_true_when_mixed_research_use_calibrations_exist_pathogenicity( + self, mock_mapped_variant_with_pathogenicity_calibration_score_set + ): + """Test behavior with mixed research use only and regular calibrations for pathogenicity annotation.""" + calibrations = mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations + + # If there's only one calibration, add another for testing + if len(calibrations) == 1: + # Create a copy of the existing calibration + new_calibration = deepcopy(calibrations[0]) + calibrations.append(new_calibration) + + # Make the first one research use only, leave the second as regular + calibrations[0].research_use_only = True + calibrations[1].research_use_only = False + + # Should return True because at least one non-research-only calibration has valid classifications + assert ( + _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( + mock_mapped_variant_with_pathogenicity_calibration_score_set.variant, "pathogenicity" + ) + is True + ) + + def test_score_range_check_handles_mixed_functional_classifications( + self, + mock_mapped_variant_with_functional_calibration_score_set, + ): + """Test behavior when some calibrations have functional classifications and some don't.""" + calibrations = mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations + + # If there's only one calibration, add another for testing + if len(calibrations) == 1: + new_calibration = deepcopy(calibrations[0]) + calibrations.append(new_calibration) + + # First calibration has functional classifications (should already exist) + # Second calibration has no functional classifications + calibrations[1].functional_classifications = None + + # Should return True because at least one calibration has valid functional classifications + assert ( + _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( + mock_mapped_variant_with_functional_calibration_score_set.variant, "functional" + ) + is True + ) + + def test_pathogenicity_annotation_with_functional_classifications_but_no_acmg( + self, + mock_mapped_variant_with_functional_calibration_score_set, + ): + """Test that pathogenicity annotation fails when functional classifications exist but have no ACMG classifications.""" + calibrations = mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations + + # Remove ACMG classifications from all functional classifications + for calibration in calibrations: + if hasattr(calibration, "functional_classifications") and calibration.functional_classifications: + acmg_classification_removed = [deepcopy(fc) for fc in calibration.functional_classifications] + for fc in acmg_classification_removed: + if "acmgClassification" in fc: + fc["acmgClassification"] = None + calibration.functional_classifications = acmg_classification_removed + + # Should return False because no ACMG classifications exist + assert ( + _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( + mock_mapped_variant_with_functional_calibration_score_set.variant, "pathogenicity" + ) + is False + ) + + def test_functional_annotation_with_empty_functional_classifications_list( + self, + mock_mapped_variant_with_functional_calibration_score_set, + ): + """Test that functional annotation fails when functional classifications list is empty.""" + calibrations = mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations + + # Set functional classifications to empty list + for calibration in calibrations: + calibration.functional_classifications = [] + + assert ( + _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( + mock_mapped_variant_with_functional_calibration_score_set.variant, "functional" + ) + is False + ) + + +@pytest.mark.unit +class TestPathogenicityAnnotationEligibilityUnit: + def test_pathogenicity_range_check_returns_false_when_base_assumptions_fail(self, mock_mapped_variant): + with patch("mavedb.lib.annotation.eligibility._can_annotate_variant_base_assumptions", return_value=False): + result = can_annotate_variant_for_pathogenicity_evidence(mock_mapped_variant.variant) + + assert result is False + + def test_pathogenicity_range_check_returns_false_when_pathogenicity_ranges_check_fails(self, mock_mapped_variant): + with patch( + "mavedb.lib.annotation.eligibility._variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation", + return_value=False, + ): + result = can_annotate_variant_for_pathogenicity_evidence(mock_mapped_variant.variant) + + assert result is False + + def test_pathogenicity_range_check_returns_true_when_all_conditions_met( + self, + mock_mapped_variant_with_pathogenicity_calibration_score_set, + ): + assert ( + can_annotate_variant_for_pathogenicity_evidence( + mock_mapped_variant_with_pathogenicity_calibration_score_set.variant + ) + is True + ) + + +@pytest.mark.unit +class TestFunctionalAnnotationEligibilityUnit: + def test_functional_range_check_returns_false_when_base_assumptions_fail(self, mock_mapped_variant): + with patch( + "mavedb.lib.annotation.eligibility._can_annotate_variant_base_assumptions", + return_value=False, + ): + result = can_annotate_variant_for_functional_statement(mock_mapped_variant.variant) + + assert result is False + + def test_functional_range_check_returns_false_when_functional_classifications_check_fails( + self, mock_mapped_variant + ): + with patch( + "mavedb.lib.annotation.eligibility._variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation", + return_value=False, + ): + result = can_annotate_variant_for_functional_statement(mock_mapped_variant.variant) + + assert result is False + + def test_functional_range_check_returns_true_when_all_conditions_met( + self, + mock_mapped_variant_with_functional_calibration_score_set, + ): + assert ( + can_annotate_variant_for_functional_statement( + mock_mapped_variant_with_functional_calibration_score_set.variant + ) + is True + ) + + +@pytest.mark.integration +class TestAnnotationEligibilityIntegration: + def test_annotation_eligibility_returns_boolean_for_persisted_variant(self, setup_lib_db_with_mapped_variant): + # Make score presence explicit so a negative result is due to missing calibrations. + setup_lib_db_with_mapped_variant.variant.data = {"score_data": {"score": 1.0}} + + pathogenicity_allowed = can_annotate_variant_for_pathogenicity_evidence( + setup_lib_db_with_mapped_variant.variant + ) + functional_allowed = can_annotate_variant_for_functional_statement(setup_lib_db_with_mapped_variant.variant) + + # DB fixture score sets do not include calibrations by default, so both should be False. + assert setup_lib_db_with_mapped_variant.variant.score_set.score_calibrations == [] + assert pathogenicity_allowed is False + assert functional_allowed is False diff --git a/tests/lib/annotation/test_evidence_line.py b/tests/lib/annotation/test_evidence_line.py index fb6f33b74..640b19a5d 100644 --- a/tests/lib/annotation/test_evidence_line.py +++ b/tests/lib/annotation/test_evidence_line.py @@ -21,10 +21,10 @@ from mavedb.lib.annotation.classification import ExperimentalVariantFunctionalImpactClassification from mavedb.lib.annotation.evidence_line import acmg_evidence_line, functional_evidence_line from mavedb.lib.annotation.proposition import ( - mapped_variant_to_experimental_variant_clinical_impact_proposition, - mapped_variant_to_experimental_variant_functional_impact_proposition, + variant_functional_impact_proposition, + variant_pathogenicity_proposition, ) -from mavedb.lib.annotation.statement import mapped_variant_to_functional_statement +from mavedb.lib.annotation.statement import functional_statement @pytest.mark.unit @@ -49,23 +49,23 @@ class TestAcmgEvidenceLine: ) def test_acmg_evidence_line_with_met_valid_clinical_classification( self, - mock_mapped_variant_with_pathogenicity_calibration_score_set, + mock_annotation_context_with_pathogenicity_calibration_score_set, expected_outcome, expected_strength, expected_direction, ): """Test ACMG evidence line creation with met valid clinical classification.""" - mapped_variant = mock_mapped_variant_with_pathogenicity_calibration_score_set - score_calibration = mapped_variant.variant.score_set.score_calibrations[0] + context = mock_annotation_context_with_pathogenicity_calibration_score_set + score_calibration = context.variant.score_set.score_calibrations[0] with patch( "mavedb.lib.annotation.evidence_line.pathogenicity_classification_of_variant", return_value=(MagicMock(label="Test Range"), expected_outcome, expected_strength), ): - proposition = mapped_variant_to_experimental_variant_clinical_impact_proposition(mapped_variant) - study_result = variant_study_result(mapped_variant) - evidence = functional_evidence_line(mapped_variant, score_calibration, [study_result]) - result = acmg_evidence_line(mapped_variant, score_calibration, proposition, [evidence]) + proposition = variant_pathogenicity_proposition(context) + study_result = variant_study_result(context) + evidence = functional_evidence_line(context, score_calibration, [study_result]) + result = acmg_evidence_line(context, score_calibration, proposition, [evidence]) if expected_strength == StrengthOfEvidenceProvided.STRONG: expected_evidence_outcome = expected_outcome.value @@ -73,7 +73,7 @@ def test_acmg_evidence_line_with_met_valid_clinical_classification( expected_evidence_outcome = f"{expected_outcome.value}_{expected_strength.name.lower()}" assert isinstance(result, VariantPathogenicityEvidenceLine) - assert result.description == f"Pathogenicity evidence line for {mapped_variant.variant.urn}." + assert result.description == f"Pathogenicity evidence line for {context.variant.urn}." assert result.evidenceOutcome.primaryCoding.code.root == expected_evidence_outcome assert result.evidenceOutcome.primaryCoding.system == "ACMG Guidelines, 2015" assert result.evidenceOutcome.name == f"ACMG 2015 {expected_outcome.name} Criterion Met" @@ -87,11 +87,11 @@ def test_acmg_evidence_line_with_met_valid_clinical_classification( def test_acmg_evidence_line_with_not_met_clinical_classification( self, - mock_mapped_variant_with_pathogenicity_calibration_score_set, + mock_annotation_context_with_pathogenicity_calibration_score_set, ): """Test ACMG evidence line creation with not met clinical classification.""" - mapped_variant = mock_mapped_variant_with_pathogenicity_calibration_score_set - score_calibration = mapped_variant.variant.score_set.score_calibrations[0] + context = mock_annotation_context_with_pathogenicity_calibration_score_set + score_calibration = context.variant.score_set.score_calibrations[0] expected_outcome = VariantPathogenicityEvidenceLine.Criterion.PS3 expected_strength = None expected_evidence_outcome = f"{expected_outcome.value}_not_met" @@ -100,13 +100,13 @@ def test_acmg_evidence_line_with_not_met_clinical_classification( "mavedb.lib.annotation.evidence_line.pathogenicity_classification_of_variant", return_value=(MagicMock(label="Test Range"), expected_outcome, expected_strength), ): - proposition = mapped_variant_to_experimental_variant_clinical_impact_proposition(mapped_variant) - study_result = variant_study_result(mapped_variant) - evidence = functional_evidence_line(mapped_variant, score_calibration, [study_result]) - result = acmg_evidence_line(mapped_variant, score_calibration, proposition, [evidence]) + proposition = variant_pathogenicity_proposition(context) + study_result = variant_study_result(context) + evidence = functional_evidence_line(context, score_calibration, [study_result]) + result = acmg_evidence_line(context, score_calibration, proposition, [evidence]) assert isinstance(result, VariantPathogenicityEvidenceLine) - assert result.description == f"Pathogenicity evidence line for {mapped_variant.variant.urn}." + assert result.description == f"Pathogenicity evidence line for {context.variant.urn}." assert result.evidenceOutcome.primaryCoding.code.root == expected_evidence_outcome assert result.evidenceOutcome.primaryCoding.system == "ACMG Guidelines, 2015" assert result.evidenceOutcome.name == f"ACMG 2015 {expected_outcome.name} Criterion Not Met" @@ -117,19 +117,20 @@ def test_acmg_evidence_line_with_not_met_clinical_classification( assert result.targetProposition == proposition assert len(result.hasEvidenceItems) == 1 - def test_acmg_evidence_line_with_no_calibrations_raises_error(self, mock_mapped_variant): + def test_acmg_evidence_line_with_no_calibrations_raises_error(self, mock_annotation_context): """Test that ACMG evidence line creation raises error when no calibrations exist.""" - mock_mapped_variant.variant.score_set.score_calibrations = None + context = mock_annotation_context + context.variant.score_set.score_calibrations = None score_calibration = MagicMock() with pytest.raises(ValueError, match="does not have a score set with score calibrations"): - proposition = mapped_variant_to_experimental_variant_clinical_impact_proposition(mock_mapped_variant) - study_result = variant_study_result(mock_mapped_variant) - acmg_evidence_line(mock_mapped_variant, score_calibration, proposition, [study_result]) + proposition = variant_pathogenicity_proposition(context) + study_result = variant_study_result(context) + acmg_evidence_line(context, score_calibration, proposition, [study_result]) def test_acmg_evidence_line_accepts_statement_evidence_without_serialization_error( self, - mock_mapped_variant_with_pathogenicity_calibration_score_set, + mock_annotation_context_with_pathogenicity_calibration_score_set, ): """Regression test: VariantPathogenicityEvidenceLine must accept Statement instances directly. @@ -138,22 +139,22 @@ def test_acmg_evidence_line_accepts_statement_evidence_without_serialization_err nested VRS objects from those dicts (e.g. Allele with real genomic coordinates). Evidence model instances must be passed directly, not serialized. """ - mapped_variant = mock_mapped_variant_with_pathogenicity_calibration_score_set - score_calibration = mapped_variant.variant.score_set.score_calibrations[0] - - study_result = variant_study_result(mapped_variant) - functional_proposition = mapped_variant_to_experimental_variant_functional_impact_proposition(mapped_variant) - functional_evidence = functional_evidence_line(mapped_variant, score_calibration, [study_result]) - functional_statement = mapped_variant_to_functional_statement( - mapped_variant, + context = mock_annotation_context_with_pathogenicity_calibration_score_set + score_calibration = context.variant.score_set.score_calibrations[0] + + study_result = variant_study_result(context) + functional_proposition = variant_functional_impact_proposition(context) + functional_evidence = functional_evidence_line(context, score_calibration, [study_result]) + statement = functional_statement( + context, functional_proposition, [functional_evidence], score_calibration, ExperimentalVariantFunctionalImpactClassification.NORMAL, ) - clinical_proposition = mapped_variant_to_experimental_variant_clinical_impact_proposition(mapped_variant) + clinical_proposition = variant_pathogenicity_proposition(context) - result = acmg_evidence_line(mapped_variant, score_calibration, clinical_proposition, [functional_statement]) + result = acmg_evidence_line(context, score_calibration, clinical_proposition, [statement]) assert isinstance(result, VariantPathogenicityEvidenceLine) assert len(result.hasEvidenceItems) == 1 @@ -166,16 +167,16 @@ class TestFunctionalEvidenceLine: """Unit tests for functional evidence line creation.""" def test_functional_evidence_line_with_valid_functional_evidence( - self, mock_mapped_variant_with_functional_calibration_score_set + self, mock_annotation_context_with_functional_calibration_score_set ): """Test functional evidence line creation with valid evidence.""" - mapped_variant = mock_mapped_variant_with_functional_calibration_score_set - score_calibration = mapped_variant.variant.score_set.score_calibrations[0] - study_result = variant_study_result(mapped_variant) - result = functional_evidence_line(mapped_variant, score_calibration, [study_result]) + context = mock_annotation_context_with_functional_calibration_score_set + score_calibration = context.variant.score_set.score_calibrations[0] + study_result = variant_study_result(context) + result = functional_evidence_line(context, score_calibration, [study_result]) assert isinstance(result, EvidenceLine) - assert result.description == f"Functional evidence line for {mapped_variant.variant.urn}" + assert result.description == f"Functional evidence line for {context.variant.urn}" assert result.directionOfEvidenceProvided is not None assert result.specifiedBy assert result.contributions diff --git a/tests/lib/annotation/test_proposition.py b/tests/lib/annotation/test_proposition.py index cb883e0bc..b7c905c60 100644 --- a/tests/lib/annotation/test_proposition.py +++ b/tests/lib/annotation/test_proposition.py @@ -7,51 +7,164 @@ clinical and functional impact propositions. """ +from types import SimpleNamespace +from unittest.mock import patch + import pytest pytest.importorskip("psycopg2") +from ga4gh.cat_vrs.models import CategoricalVariant from ga4gh.va_spec.base.core import ( ExperimentalVariantFunctionalImpactProposition, VariantPathogenicityProposition, ) from ga4gh.vrs.models import MolecularVariation +from mavedb.lib.annotation.exceptions import MappingDataDoesntExistException from mavedb.lib.annotation.proposition import ( - mapped_variant_to_experimental_variant_clinical_impact_proposition, - mapped_variant_to_experimental_variant_functional_impact_proposition, + sequence_feature_for_variant, + variant_functional_impact_proposition, + variant_pathogenicity_proposition, ) +from mavedb.lib.cat_vrs import build_categorical_variant +from mavedb.models.allele import Allele +from mavedb.models.mapping_record_allele import MappingRecordAllele +from tests.helpers.constants import TEST_VALID_POST_MAPPED_VRS_ALLELE +from tests.lib.annotation.conftest import annotation_context_for + + +def _protein_categorical_variant() -> CategoricalVariant: + """A Cat-VRS ``CategoricalVariant`` for a protein-measured variant (defining protein + `encodes` nt). + + Built over transient links (no DB) so the proposition tests can assert the builders accept a + categorical subject — the shape a protein assay produces under Slice 5.1. + """ + links = [ + MappingRecordAllele( + is_authoritative=True, + allele=Allele(level="protein", vrs_digest="prot", post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE), + ), + MappingRecordAllele( + is_authoritative=False, + allele=Allele(level="cdna", vrs_digest="cdna", post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE), + ), + ] + transit = build_categorical_variant(links, name="urn:mavedb:test#1") + assert transit is not None + return transit.categorical_variant @pytest.mark.unit class TestExperimentalVariantClinicalImpactProposition: """Unit tests for experimental variant clinical impact proposition creation.""" - def test_mapped_variant_to_experimental_variant_clinical_impact_proposition(self, mock_mapped_variant): - """Test creation of clinical impact proposition from mapped variant.""" - result = mapped_variant_to_experimental_variant_clinical_impact_proposition(mock_mapped_variant) + def test_variant_pathogenicity_proposition(self, mock_annotation_context): + """Test creation of pathogenicity proposition from a variant annotation context.""" + result = variant_pathogenicity_proposition(mock_annotation_context) assert isinstance(result, VariantPathogenicityProposition) - assert result.description == f"Variant pathogenicity proposition for {mock_mapped_variant.variant.urn}." + assert result.description == f"Variant pathogenicity proposition for {mock_annotation_context.variant.urn}." assert isinstance(result.subjectVariant, MolecularVariation) assert result.predicate == "isCausalFor" assert result.objectCondition.root.conceptType == "Disease" assert result.objectCondition.root.primaryCoding.code.root == "C0012634" assert result.objectCondition.root.primaryCoding.system == "https://www.ncbi.nlm.nih.gov/medgen/" + def test_clinical_impact_proposition_carries_a_categorical_subject(self, mock_mapped_variant): + """A protein assay's categorical subject (Slice 5.1) is carried onto the proposition unchanged.""" + context = annotation_context_for(mock_mapped_variant, subject_variant=_protein_categorical_variant()) + result = variant_pathogenicity_proposition(context) + + assert isinstance(result.subjectVariant, CategoricalVariant) + @pytest.mark.unit class TestExperimentalVariantFunctionalImpactProposition: """Unit tests for experimental variant functional impact proposition creation.""" - def test_mapped_variant_to_experimental_variant_functional_impact_proposition(self, mock_mapped_variant): - """Test creation of functional impact proposition from mapped variant.""" - result = mapped_variant_to_experimental_variant_functional_impact_proposition(mock_mapped_variant) + def test_variant_functional_impact_proposition(self, mock_annotation_context): + """Test creation of functional impact proposition from a variant annotation context.""" + result = variant_functional_impact_proposition(mock_annotation_context) assert isinstance(result, ExperimentalVariantFunctionalImpactProposition) - assert result.description == f"Variant functional impact proposition for {mock_mapped_variant.variant.urn}." + assert result.description == f"Variant functional impact proposition for {mock_annotation_context.variant.urn}." assert isinstance(result.subjectVariant, MolecularVariation) assert result.predicate == "impactsFunctionOf" assert result.objectSequenceFeature.primaryCoding.code.root == "BRCA1" assert result.objectSequenceFeature.primaryCoding.system == "https://www.genenames.org/" assert result.experimentalContextQualifier is not None + + def test_functional_impact_proposition_carries_a_categorical_subject(self, mock_mapped_variant): + """A protein assay's categorical subject (Slice 5.1) is carried onto the proposition unchanged.""" + context = annotation_context_for(mock_mapped_variant, subject_variant=_protein_categorical_variant()) + result = variant_functional_impact_proposition(context) + + assert isinstance(result.subjectVariant, CategoricalVariant) + + +@pytest.mark.unit +class TestSequenceFeatureForVariantUnit: + """sequence_feature_for_variant — co-located with the propositions that consume it.""" + + def test_sequence_feature_raises_when_target_is_missing(self, mock_mapped_variant): + with patch("mavedb.lib.annotation.proposition.target_for_variant", return_value=None): + with pytest.raises(MappingDataDoesntExistException): + sequence_feature_for_variant(mock_mapped_variant.variant) + + def test_sequence_feature_returns_ensembl_identifier(self, mock_mapped_variant): + target = SimpleNamespace(mapped_hgnc_name=None, post_mapped_metadata={"x": "y"}, name="BRCA1") + + with patch("mavedb.lib.annotation.proposition.target_for_variant", return_value=target): + with patch( + "mavedb.lib.annotation.proposition.extract_ids_from_post_mapped_metadata", + return_value=["ENST00000357654"], + ): + feature, system = sequence_feature_for_variant(mock_mapped_variant.variant) + + assert feature == "ENST00000357654" + assert system == "https://www.ensembl.org/index.html" + + def test_sequence_feature_returns_refseq_identifier(self, mock_mapped_variant): + target = SimpleNamespace(mapped_hgnc_name=None, post_mapped_metadata={"x": "y"}, name="BRCA1") + + with patch("mavedb.lib.annotation.proposition.target_for_variant", return_value=target): + with patch( + "mavedb.lib.annotation.proposition.extract_ids_from_post_mapped_metadata", + return_value=["NM_000546.6"], + ): + feature, system = sequence_feature_for_variant(mock_mapped_variant.variant) + + assert feature == "NM_000546.6" + assert system == "https://www.ncbi.nlm.nih.gov/refseq/" + + def test_sequence_feature_returns_unknown_identifier_source(self, mock_mapped_variant): + target = SimpleNamespace(mapped_hgnc_name=None, post_mapped_metadata={"x": "y"}, name="BRCA1") + + with patch("mavedb.lib.annotation.proposition.target_for_variant", return_value=target): + with patch( + "mavedb.lib.annotation.proposition.extract_ids_from_post_mapped_metadata", + return_value=["CUSTOM_ID_1"], + ): + feature, system = sequence_feature_for_variant(mock_mapped_variant.variant) + + assert feature == "CUSTOM_ID_1" + assert system == "transcript or gene identifier of unknown source" + + def test_sequence_feature_falls_back_to_target_name(self, mock_mapped_variant): + target = SimpleNamespace(mapped_hgnc_name=None, post_mapped_metadata={}, name="TP53") + + with patch("mavedb.lib.annotation.proposition.target_for_variant", return_value=target): + with patch("mavedb.lib.annotation.proposition.extract_ids_from_post_mapped_metadata", return_value=[]): + feature, system = sequence_feature_for_variant(mock_mapped_variant.variant) + + assert feature == "TP53" + assert system == "https://www.mavedb.org/" + + def test_sequence_feature_raises_when_target_has_no_name_or_ids(self, mock_mapped_variant): + target = SimpleNamespace(mapped_hgnc_name=None, post_mapped_metadata={}, name=None) + + with patch("mavedb.lib.annotation.proposition.target_for_variant", return_value=target): + with patch("mavedb.lib.annotation.proposition.extract_ids_from_post_mapped_metadata", return_value=[]): + with pytest.raises(MappingDataDoesntExistException): + sequence_feature_for_variant(mock_mapped_variant.variant) diff --git a/tests/lib/annotation/test_statement.py b/tests/lib/annotation/test_statement.py index 5c36aeecf..7cd7e41fb 100644 --- a/tests/lib/annotation/test_statement.py +++ b/tests/lib/annotation/test_statement.py @@ -3,7 +3,7 @@ """ Tests for mavedb.lib.annotation.statement module. -This module tests statement creation functions for mapped variants, +This module tests statement creation functions for variants, focusing on functional impact statements and their classification. """ @@ -18,15 +18,13 @@ from mavedb.lib.annotation.annotate import variant_study_result from mavedb.lib.annotation.classification import ExperimentalVariantFunctionalImpactClassification from mavedb.lib.annotation.evidence_line import functional_evidence_line -from mavedb.lib.annotation.proposition import mapped_variant_to_experimental_variant_functional_impact_proposition -from mavedb.lib.annotation.statement import ( - mapped_variant_to_functional_statement, -) +from mavedb.lib.annotation.proposition import variant_functional_impact_proposition +from mavedb.lib.annotation.statement import functional_statement @pytest.mark.unit -class TestMappedVariantFunctionalStatement: - """Unit tests for mapped variant functional statement creation.""" +class TestFunctionalStatement: + """Unit tests for functional statement creation.""" @pytest.mark.parametrize( "classification, expected_direction", @@ -36,30 +34,26 @@ class TestMappedVariantFunctionalStatement: (ExperimentalVariantFunctionalImpactClassification.INDETERMINATE, Direction.NEUTRAL), ], ) - def test_mapped_variant_to_functional_statement( + def test_functional_statement( self, - mock_mapped_variant_with_functional_calibration_score_set, + mock_annotation_context_with_functional_calibration_score_set, classification, expected_direction, ): """Test functional statement creation with different classifications.""" - mapped_variant = mock_mapped_variant_with_functional_calibration_score_set - score_calibration = mapped_variant.variant.score_set.score_calibrations[0] + context = mock_annotation_context_with_functional_calibration_score_set + score_calibration = context.variant.score_set.score_calibrations[0] with patch( "mavedb.lib.annotation.evidence_line.functional_classification_of_variant", return_value=(MagicMock(label="Test Range"), classification), ): - proposition = mapped_variant_to_experimental_variant_functional_impact_proposition(mapped_variant) - evidence = functional_evidence_line( - mapped_variant, score_calibration, [variant_study_result(mapped_variant)] - ) - result = mapped_variant_to_functional_statement( - mapped_variant, proposition, [evidence], score_calibration, classification - ) + proposition = variant_functional_impact_proposition(context) + evidence = functional_evidence_line(context, score_calibration, [variant_study_result(context)]) + result = functional_statement(context, proposition, [evidence], score_calibration, classification) assert isinstance(result, Statement) - assert result.description == f"Variant functional impact statement for {mapped_variant.variant.urn}." + assert result.description == f"Variant functional impact statement for {context.variant.urn}." assert result.specifiedBy assert result.contributions assert len(result.contributions) == 3 # API, VRS, and score calibration contributions @@ -75,24 +69,22 @@ def test_mapped_variant_to_functional_statement( def test_no_calibrations_raises_value_error( self, - mock_mapped_variant_with_functional_calibration_score_set, + mock_annotation_context_with_functional_calibration_score_set, ): """Test that missing score calibrations raises ValueError in evidence line creation.""" - mapped_variant = mock_mapped_variant_with_functional_calibration_score_set - score_calibration = mapped_variant.variant.score_set.score_calibrations[0] + context = mock_annotation_context_with_functional_calibration_score_set + score_calibration = context.variant.score_set.score_calibrations[0] # Set calibrations to None to trigger the error - mapped_variant.variant.score_set.score_calibrations = None + context.variant.score_set.score_calibrations = None - proposition = mapped_variant_to_experimental_variant_functional_impact_proposition(mapped_variant) + proposition = variant_functional_impact_proposition(context) with pytest.raises(ValueError, match="does not have a score set with score calibrations"): # Use a dummy score calibration for evidence line creation, but it should not be reached due to the error - evidence = functional_evidence_line( - mapped_variant, score_calibration, [variant_study_result(mapped_variant)] - ) - mapped_variant_to_functional_statement( - mapped_variant, + evidence = functional_evidence_line(context, score_calibration, [variant_study_result(context)]) + functional_statement( + context, proposition, [evidence], score_calibration, @@ -101,8 +93,8 @@ def test_no_calibrations_raises_value_error( @pytest.mark.unit -class TestMappedVariantPathogenicityStatement: - """Unit tests for mapped variant pathogenicity statement ACMG classification mapping.""" +class TestPathogenicityStatement: + """Unit tests for pathogenicity statement ACMG classification mapping.""" def test_pathogenic_criterion_maps_to_pathogenic(self, mock_mapped_variant): """Test that pathogenic ACMG criterion (PS3) maps to PATHOGENIC classification.""" diff --git a/tests/lib/annotation/test_study_result.py b/tests/lib/annotation/test_study_result.py index 1cf8bbf63..356eaa1e2 100644 --- a/tests/lib/annotation/test_study_result.py +++ b/tests/lib/annotation/test_study_result.py @@ -14,24 +14,22 @@ from ga4gh.va_spec.base.core import ExperimentalVariantFunctionalImpactStudyResult from ga4gh.vrs.models import MolecularVariation -from mavedb.lib.annotation.document import mapped_variant_as_iri, variant_as_iri -from mavedb.lib.annotation.study_result import ( - mapped_variant_to_experimental_variant_impact_study_result, -) +from mavedb.lib.annotation.document import measured_allele_as_iri, variant_as_iri +from mavedb.lib.annotation.study_result import variant_impact_study_result @pytest.mark.unit class TestExperimentalVariantImpactStudyResult: """Unit tests for experimental variant impact study result creation.""" - def test_mapped_variant_to_experimental_variant_impact_study_result(self, mock_mapped_variant): - """Test creation of experimental variant impact study result from mapped variant.""" - result = mapped_variant_to_experimental_variant_impact_study_result(mock_mapped_variant) + def test_variant_impact_study_result(self, mock_annotation_context): + """Test creation of experimental variant impact study result from an annotation context.""" + result = variant_impact_study_result(mock_annotation_context) assert isinstance(result, ExperimentalVariantFunctionalImpactStudyResult) - assert result.description == f"Variant effect study result for {mock_mapped_variant.variant.urn}." + assert result.description == f"Variant effect study result for {mock_annotation_context.variant.urn}." assert isinstance(result.focusVariant, MolecularVariation) - assert result.functionalImpactScore == mock_mapped_variant.variant.data["score_data"]["score"] + assert result.functionalImpactScore == mock_annotation_context.variant.data["score_data"]["score"] # Verify all expected contribution types are present contribution_types = {c.name for c in result.contributions} expected_types = {"MaveDB API", "MaveDB VRS Mapper", "MaveDB Dataset Creator", "MaveDB Dataset Modifier"} @@ -39,18 +37,18 @@ def test_mapped_variant_to_experimental_variant_impact_study_result(self, mock_m # specifiedBy will be None when no primary publications exist assert result.sourceDataSet is not None assert result.reportedIn is not None - assert mapped_variant_as_iri(mock_mapped_variant) in result.reportedIn - assert variant_as_iri(mock_mapped_variant.variant) in result.reportedIn + assert measured_allele_as_iri(mock_annotation_context.measured_allele) in result.reportedIn + assert variant_as_iri(mock_annotation_context.variant) in result.reportedIn - def test_no_mapped_variant_is_filtered_properly(self, mock_mapped_variant): - """Test that study result handles missing mapped variant (no ClinGen allele ID).""" - mock_mapped_variant.clingen_allele_id = None - result = mapped_variant_to_experimental_variant_impact_study_result(mock_mapped_variant) + def test_no_clingen_allele_id_is_filtered_properly(self, mock_annotation_context): + """Test that study result handles a measured allele with no ClinGen allele ID.""" + mock_annotation_context.measured_allele.clingen_allele_id = None + result = variant_impact_study_result(mock_annotation_context) assert isinstance(result, ExperimentalVariantFunctionalImpactStudyResult) - assert result.description == f"Variant effect study result for {mock_mapped_variant.variant.urn}." + assert result.description == f"Variant effect study result for {mock_annotation_context.variant.urn}." assert isinstance(result.focusVariant, MolecularVariation) - assert result.functionalImpactScore == mock_mapped_variant.variant.data["score_data"]["score"] + assert result.functionalImpactScore == mock_annotation_context.variant.data["score_data"]["score"] # Verify all expected contribution types are present contribution_types = {c.name for c in result.contributions} expected_types = {"MaveDB API", "MaveDB VRS Mapper", "MaveDB Dataset Creator", "MaveDB Dataset Modifier"} @@ -58,5 +56,5 @@ def test_no_mapped_variant_is_filtered_properly(self, mock_mapped_variant): # specifiedBy will be None when no primary publications exist assert result.sourceDataSet is not None assert result.reportedIn is not None - assert variant_as_iri(mock_mapped_variant.variant) in result.reportedIn + assert variant_as_iri(mock_annotation_context.variant) in result.reportedIn assert len(result.reportedIn) == 1 diff --git a/tests/lib/annotation/test_util.py b/tests/lib/annotation/test_util.py deleted file mode 100644 index 515ae6286..000000000 --- a/tests/lib/annotation/test_util.py +++ /dev/null @@ -1,868 +0,0 @@ -# ruff: noqa: E402 - -""" -Tests for mavedb.lib.annotation.util module. - -This module tests utility functions used by annotation workflows, including -variation extraction, annotation eligibility checks, and sequence feature -resolution. -""" - -from copy import deepcopy -from types import SimpleNamespace -from unittest.mock import patch - -import pytest - -pytest.importorskip("psycopg2") - -from mavedb.lib.annotation.exceptions import MappingDataDoesntExistException -from mavedb.lib.annotation.util import ( - _can_annotate_variant_base_assumptions, - _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation, - can_annotate_variant_for_functional_statement, - can_annotate_variant_for_pathogenicity_evidence, - score_calibration_may_be_used_for_annotation, - select_strongest_functional_calibration, - select_strongest_pathogenicity_calibration, - sequence_feature_for_mapped_variant, - variation_from_mapped_variant, - vrs_object_from_mapped_variant, -) -from tests.helpers.constants import ( - TEST_SEQUENCE_LOCATION_ACCESSION, - TEST_VALID_POST_MAPPED_VRS_ALLELE, - TEST_VALID_POST_MAPPED_VRS_ALLELE_LENGTH_EXPRESSION, - TEST_VALID_POST_MAPPED_VRS_ALLELE_RLE, -) - - -@pytest.mark.unit -class TestVariationExtractionUnit: - @pytest.mark.parametrize( - "variation_version", - [{"variation": TEST_VALID_POST_MAPPED_VRS_ALLELE}, TEST_VALID_POST_MAPPED_VRS_ALLELE], - ids=["vrs13_wrapped_variation", "vrs2_direct_allele"], - ) - def test_variation_from_mapped_variant_post_mapped_variation(self, mock_mapped_variant, variation_version): - mock_mapped_variant.post_mapped = variation_version - - result = variation_from_mapped_variant(mock_mapped_variant).model_dump() - - assert result["location"]["id"] == TEST_SEQUENCE_LOCATION_ACCESSION - assert result["location"]["start"] == 5 - assert result["location"]["end"] == 6 - - def test_variation_from_mapped_variant_no_post_mapped(self, mock_mapped_variant): - mock_mapped_variant.post_mapped = None - - with pytest.raises(MappingDataDoesntExistException): - variation_from_mapped_variant(mock_mapped_variant) - - @pytest.mark.parametrize( - "allele_dict, expected_state_type, expected_state_fields", - [ - ( - TEST_VALID_POST_MAPPED_VRS_ALLELE, - "LiteralSequenceExpression", - {"sequence": "F"}, - ), - ( - TEST_VALID_POST_MAPPED_VRS_ALLELE_RLE, - "ReferenceLengthExpression", - {"length": 5, "repeatSubunitLength": 5}, - ), - ( - TEST_VALID_POST_MAPPED_VRS_ALLELE_LENGTH_EXPRESSION, - "LengthExpression", - {"length": 5}, - ), - ], - ids=["lse_state", "rle_state", "length_expression_state"], - ) - def test_allele_state_type_is_deserialized_correctly(self, allele_dict, expected_state_type, expected_state_fields): - result = vrs_object_from_mapped_variant(allele_dict).model_dump() - - assert result["state"]["type"] == expected_state_type - for k, v in expected_state_fields.items(): - assert result["state"][k] == v - - def test_unknown_allele_state_type_raises_value_error(self): - allele_dict = { - **TEST_VALID_POST_MAPPED_VRS_ALLELE, - "state": {"type": "UnknownFutureExpression", "someField": "someValue"}, - } - - with pytest.raises(ValueError, match="Unsupported VRS Allele state type"): - vrs_object_from_mapped_variant(allele_dict) - - @pytest.mark.parametrize( - "member_dict, expected_state_type", - [ - (TEST_VALID_POST_MAPPED_VRS_ALLELE, "LiteralSequenceExpression"), - (TEST_VALID_POST_MAPPED_VRS_ALLELE_RLE, "ReferenceLengthExpression"), - (TEST_VALID_POST_MAPPED_VRS_ALLELE_LENGTH_EXPRESSION, "LengthExpression"), - ], - ids=["lse_members", "rle_members", "length_expression_members"], - ) - def test_cis_phased_block_member_state_types_are_deserialized_correctly(self, member_dict, expected_state_type): - mapping_results = {"type": "CisPhasedBlock", "members": [member_dict, member_dict]} - - result = vrs_object_from_mapped_variant(mapping_results).model_dump() - - assert result["type"] == "CisPhasedBlock" - assert len(result["members"]) == 2 - assert result["members"][0]["state"]["type"] == expected_state_type - - -@pytest.mark.unit -class TestBaseAnnotationAssumptionsUnit: - def test_base_assumption_check_returns_false_when_score_is_none(self, mock_mapped_variant): - mock_mapped_variant.variant.data = {"score_data": {"score": None}} - - assert _can_annotate_variant_base_assumptions(mock_mapped_variant) is False - - def test_base_assumption_check_returns_true_when_all_conditions_met(self, mock_mapped_variant): - assert _can_annotate_variant_base_assumptions(mock_mapped_variant) is True - - -@pytest.mark.unit -class TestScoreCalibrationMayBeUsedForAnnotation: - def test_returns_false_for_research_use_only_when_not_allowed( - self, mock_mapped_variant_with_functional_calibration_score_set - ): - calibration = mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations[0] - calibration.research_use_only = True - - assert ( - score_calibration_may_be_used_for_annotation( - calibration, - annotation_type="functional", - allow_research_use_only_calibrations=False, - ) - is False - ) - - def test_returns_true_for_research_use_only_when_allowed( - self, mock_mapped_variant_with_functional_calibration_score_set - ): - calibration = mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations[0] - calibration.research_use_only = True - - assert ( - score_calibration_may_be_used_for_annotation( - calibration, - annotation_type="functional", - allow_research_use_only_calibrations=True, - ) - is True - ) - - def test_returns_false_when_functional_classifications_missing( - self, mock_mapped_variant_with_functional_calibration_score_set - ): - calibration = mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations[0] - calibration.functional_classifications = [] - - assert score_calibration_may_be_used_for_annotation(calibration, annotation_type="functional") is False - - def test_returns_false_for_pathogenicity_without_acmg_classifications( - self, mock_mapped_variant_with_pathogenicity_calibration_score_set - ): - calibration = mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations[ - 0 - ] - acmg_removed = [deepcopy(fc) for fc in calibration.functional_classifications] - for functional_classification in acmg_removed: - functional_classification["acmgClassification"] = None - calibration.functional_classifications = acmg_removed - - assert score_calibration_may_be_used_for_annotation(calibration, annotation_type="pathogenicity") is False - - def test_returns_true_for_pathogenicity_with_any_acmg_classification( - self, mock_mapped_variant_with_pathogenicity_calibration_score_set - ): - calibration = mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations[ - 0 - ] - - assert score_calibration_may_be_used_for_annotation(calibration, annotation_type="pathogenicity") is True - - -@pytest.mark.unit -class TestVariantScoreCalibrationsHaveRequiredCalibrationsAndRangesForAnnotation: - """ - Unit tests for the _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation function. - This function is used by both functional and pathogenicity annotation checks, so we test it separately here to avoid duplication in the tests for those checks. - """ - - @pytest.mark.parametrize("kind", ["functional", "pathogenicity"], ids=["functional", "pathogenicity"]) - def test_score_range_check_returns_false_when_calibrations_are_none(self, mock_mapped_variant, kind): - mock_mapped_variant.variant.score_set.score_calibrations = None - assert ( - _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation(mock_mapped_variant, kind) - is False - ) - - @pytest.mark.parametrize("kind", ["functional", "pathogenicity"], ids=["functional", "pathogenicity"]) - def test_score_range_check_returns_false_when_no_calibrations_present(self, mock_mapped_variant, kind): - mock_mapped_variant.variant.score_set.score_calibrations = [] - assert ( - _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation(mock_mapped_variant, kind) - is False - ) - - @pytest.mark.parametrize("annotation_type", ["functional", "pathogenicity"], ids=["functional", "pathogenicity"]) - def test_score_range_check_returns_false_when_all_calibrations_are_research_use_only_and_not_allowed( - self, mock_mapped_variant_with_functional_calibration_score_set, annotation_type - ): - """Test that research use only calibrations are excluded by default.""" - # Make all calibrations research use only - for ( - calibration - ) in mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations: - calibration.research_use_only = True - - assert ( - _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( - mock_mapped_variant_with_functional_calibration_score_set, annotation_type - ) - is False - ) - - @pytest.mark.parametrize( - "kind,variant_fixture", - [ - ("functional", "mock_mapped_variant_with_functional_calibration_score_set"), - ("pathogenicity", "mock_mapped_variant_with_pathogenicity_calibration_score_set"), - ], - ids=["functional_fixture", "pathogenicity_fixture"], - ) - def test_score_range_check_returns_true_when_research_use_only_calibrations_are_allowed( - self, kind, variant_fixture, request - ): - """Test that research use only calibrations are included when explicitly allowed.""" - mock_mapped_variant = request.getfixturevalue(variant_fixture) - # Make all calibrations research use only - for calibration in mock_mapped_variant.variant.score_set.score_calibrations: - calibration.primary = False - calibration.research_use_only = True - - assert ( - _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( - mock_mapped_variant, kind, allow_research_use_only_calibrations=True - ) - is True - ) - - @pytest.mark.parametrize( - "kind,variant_fixture", - [ - ("functional", "mock_mapped_variant_with_functional_calibration_score_set"), - ("pathogenicity", "mock_mapped_variant_with_pathogenicity_calibration_score_set"), - ], - ids=["functional_fixture", "pathogenicity_fixture"], - ) - def test_score_range_check_returns_false_when_calibrations_present_with_empty_ranges( - self, kind, variant_fixture, request - ): - mock_mapped_variant = request.getfixturevalue(variant_fixture) - - for calibration in mock_mapped_variant.variant.score_set.score_calibrations: - calibration.functional_classifications = None - - assert ( - _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation(mock_mapped_variant, kind) - is False - ) - - def test_pathogenicity_range_check_returns_false_when_no_acmg_calibration( - self, - mock_mapped_variant_with_pathogenicity_calibration_score_set, - ): - for ( - calibration - ) in mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations: - acmg_classification_removed = [deepcopy(r) for r in calibration.functional_classifications] - for fr in acmg_classification_removed: - fr["acmgClassification"] = None - - calibration.functional_classifications = acmg_classification_removed - - assert ( - _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( - mock_mapped_variant_with_pathogenicity_calibration_score_set, "pathogenicity" - ) - is False - ) - - def test_pathogenicity_range_check_returns_true_when_some_acmg_calibration( - self, - mock_mapped_variant_with_pathogenicity_calibration_score_set, - ): - for ( - calibration - ) in mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations: - acmg_classification_removed = [deepcopy(r) for r in calibration.functional_classifications] - acmg_classification_removed[0]["acmgClassification"] = None - - calibration.functional_classifications = acmg_classification_removed - - assert ( - _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( - mock_mapped_variant_with_pathogenicity_calibration_score_set, "pathogenicity" - ) - is True - ) - - @pytest.mark.parametrize( - "kind,variant_fixture", - [ - ("functional", "mock_mapped_variant_with_functional_calibration_score_set"), - ("pathogenicity", "mock_mapped_variant_with_pathogenicity_calibration_score_set"), - ], - ids=["functional_fixture", "pathogenicity_fixture"], - ) - def test_score_range_check_returns_true_when_calibration_kind_exists_with_ranges( - self, kind, variant_fixture, request - ): - mock_mapped_variant = request.getfixturevalue(variant_fixture) - - assert ( - _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation(mock_mapped_variant, kind) - is True - ) - - def test_score_range_check_returns_true_when_mixed_research_use_calibrations_exist_functional( - self, mock_mapped_variant_with_functional_calibration_score_set - ): - """Test behavior with mixed research use only and regular calibrations for functional annotation.""" - calibrations = mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations - - # If there's only one calibration, add another for testing - if len(calibrations) == 1: - # Create a copy of the existing calibration - new_calibration = deepcopy(calibrations[0]) - calibrations.append(new_calibration) - - # Make the first one research use only, leave the second as regular - calibrations[0].research_use_only = True - calibrations[1].research_use_only = False - - # Should return True because at least one non-research-only calibration has valid classifications - assert ( - _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( - mock_mapped_variant_with_functional_calibration_score_set, "functional" - ) - is True - ) - - def test_score_range_check_returns_true_when_mixed_research_use_calibrations_exist_pathogenicity( - self, mock_mapped_variant_with_pathogenicity_calibration_score_set - ): - """Test behavior with mixed research use only and regular calibrations for pathogenicity annotation.""" - calibrations = mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations - - # If there's only one calibration, add another for testing - if len(calibrations) == 1: - # Create a copy of the existing calibration - new_calibration = deepcopy(calibrations[0]) - calibrations.append(new_calibration) - - # Make the first one research use only, leave the second as regular - calibrations[0].research_use_only = True - calibrations[1].research_use_only = False - - # Should return True because at least one non-research-only calibration has valid classifications - assert ( - _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( - mock_mapped_variant_with_pathogenicity_calibration_score_set, "pathogenicity" - ) - is True - ) - - def test_score_range_check_handles_mixed_functional_classifications( - self, - mock_mapped_variant_with_functional_calibration_score_set, - ): - """Test behavior when some calibrations have functional classifications and some don't.""" - calibrations = mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations - - # If there's only one calibration, add another for testing - if len(calibrations) == 1: - new_calibration = deepcopy(calibrations[0]) - calibrations.append(new_calibration) - - # First calibration has functional classifications (should already exist) - # Second calibration has no functional classifications - calibrations[1].functional_classifications = None - - # Should return True because at least one calibration has valid functional classifications - assert ( - _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( - mock_mapped_variant_with_functional_calibration_score_set, "functional" - ) - is True - ) - - def test_pathogenicity_annotation_with_functional_classifications_but_no_acmg( - self, - mock_mapped_variant_with_functional_calibration_score_set, - ): - """Test that pathogenicity annotation fails when functional classifications exist but have no ACMG classifications.""" - calibrations = mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations - - # Remove ACMG classifications from all functional classifications - for calibration in calibrations: - if hasattr(calibration, "functional_classifications") and calibration.functional_classifications: - acmg_classification_removed = [deepcopy(fc) for fc in calibration.functional_classifications] - for fc in acmg_classification_removed: - if "acmgClassification" in fc: - fc["acmgClassification"] = None - calibration.functional_classifications = acmg_classification_removed - - # Should return False because no ACMG classifications exist - assert ( - _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( - mock_mapped_variant_with_functional_calibration_score_set, "pathogenicity" - ) - is False - ) - - def test_functional_annotation_with_empty_functional_classifications_list( - self, - mock_mapped_variant_with_functional_calibration_score_set, - ): - """Test that functional annotation fails when functional classifications list is empty.""" - calibrations = mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations - - # Set functional classifications to empty list - for calibration in calibrations: - calibration.functional_classifications = [] - - assert ( - _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( - mock_mapped_variant_with_functional_calibration_score_set, "functional" - ) - is False - ) - - -@pytest.mark.unit -class TestPathogenicityAnnotationEligibilityUnit: - def test_pathogenicity_range_check_returns_false_when_base_assumptions_fail(self, mock_mapped_variant): - with patch("mavedb.lib.annotation.util._can_annotate_variant_base_assumptions", return_value=False): - result = can_annotate_variant_for_pathogenicity_evidence(mock_mapped_variant) - - assert result is False - - def test_pathogenicity_range_check_returns_false_when_pathogenicity_ranges_check_fails(self, mock_mapped_variant): - with patch( - "mavedb.lib.annotation.util._variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation", - return_value=False, - ): - result = can_annotate_variant_for_pathogenicity_evidence(mock_mapped_variant) - - assert result is False - - def test_pathogenicity_range_check_returns_true_when_all_conditions_met( - self, - mock_mapped_variant_with_pathogenicity_calibration_score_set, - ): - assert ( - can_annotate_variant_for_pathogenicity_evidence( - mock_mapped_variant_with_pathogenicity_calibration_score_set - ) - is True - ) - - -@pytest.mark.unit -class TestFunctionalAnnotationEligibilityUnit: - def test_functional_range_check_returns_false_when_base_assumptions_fail(self, mock_mapped_variant): - with patch( - "mavedb.lib.annotation.util._can_annotate_variant_base_assumptions", - return_value=False, - ): - result = can_annotate_variant_for_functional_statement(mock_mapped_variant) - - assert result is False - - def test_functional_range_check_returns_false_when_functional_classifications_check_fails( - self, mock_mapped_variant - ): - with patch( - "mavedb.lib.annotation.util._variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation", - return_value=False, - ): - result = can_annotate_variant_for_functional_statement(mock_mapped_variant) - - assert result is False - - def test_functional_range_check_returns_true_when_all_conditions_met( - self, - mock_mapped_variant_with_functional_calibration_score_set, - ): - assert ( - can_annotate_variant_for_functional_statement(mock_mapped_variant_with_functional_calibration_score_set) - is True - ) - - -@pytest.mark.unit -class TestSequenceFeatureForMappedVariantUnit: - def test_sequence_feature_raises_when_target_is_missing(self, mock_mapped_variant): - with patch("mavedb.lib.annotation.util.target_for_variant", return_value=None): - with pytest.raises(MappingDataDoesntExistException): - sequence_feature_for_mapped_variant(mock_mapped_variant) - - def test_sequence_feature_returns_ensembl_identifier(self, mock_mapped_variant): - target = SimpleNamespace(mapped_hgnc_name=None, post_mapped_metadata={"x": "y"}, name="BRCA1") - - with patch("mavedb.lib.annotation.util.target_for_variant", return_value=target): - with patch( - "mavedb.lib.annotation.util.extract_ids_from_post_mapped_metadata", return_value=["ENST00000357654"] - ): - feature, system = sequence_feature_for_mapped_variant(mock_mapped_variant) - - assert feature == "ENST00000357654" - assert system == "https://www.ensembl.org/index.html" - - def test_sequence_feature_returns_refseq_identifier(self, mock_mapped_variant): - target = SimpleNamespace(mapped_hgnc_name=None, post_mapped_metadata={"x": "y"}, name="BRCA1") - - with patch("mavedb.lib.annotation.util.target_for_variant", return_value=target): - with patch( - "mavedb.lib.annotation.util.extract_ids_from_post_mapped_metadata", return_value=["NM_000546.6"] - ): - feature, system = sequence_feature_for_mapped_variant(mock_mapped_variant) - - assert feature == "NM_000546.6" - assert system == "https://www.ncbi.nlm.nih.gov/refseq/" - - def test_sequence_feature_returns_unknown_identifier_source(self, mock_mapped_variant): - target = SimpleNamespace(mapped_hgnc_name=None, post_mapped_metadata={"x": "y"}, name="BRCA1") - - with patch("mavedb.lib.annotation.util.target_for_variant", return_value=target): - with patch( - "mavedb.lib.annotation.util.extract_ids_from_post_mapped_metadata", return_value=["CUSTOM_ID_1"] - ): - feature, system = sequence_feature_for_mapped_variant(mock_mapped_variant) - - assert feature == "CUSTOM_ID_1" - assert system == "transcript or gene identifier of unknown source" - - def test_sequence_feature_falls_back_to_target_name(self, mock_mapped_variant): - target = SimpleNamespace(mapped_hgnc_name=None, post_mapped_metadata={}, name="TP53") - - with patch("mavedb.lib.annotation.util.target_for_variant", return_value=target): - with patch("mavedb.lib.annotation.util.extract_ids_from_post_mapped_metadata", return_value=[]): - feature, system = sequence_feature_for_mapped_variant(mock_mapped_variant) - - assert feature == "TP53" - assert system == "https://www.mavedb.org/" - - def test_sequence_feature_raises_when_target_has_no_name_or_ids(self, mock_mapped_variant): - target = SimpleNamespace(mapped_hgnc_name=None, post_mapped_metadata={}, name=None) - - with patch("mavedb.lib.annotation.util.target_for_variant", return_value=target): - with patch("mavedb.lib.annotation.util.extract_ids_from_post_mapped_metadata", return_value=[]): - with pytest.raises(MappingDataDoesntExistException): - sequence_feature_for_mapped_variant(mock_mapped_variant) - - -@pytest.mark.integration -class TestAnnotationUtilIntegration: - """Integration tests for utility helpers using persisted DB-backed variants.""" - - def test_variation_from_persisted_mapped_variant(self, setup_lib_db_with_mapped_variant): - setup_lib_db_with_mapped_variant.post_mapped = TEST_VALID_POST_MAPPED_VRS_ALLELE - variation = variation_from_mapped_variant(setup_lib_db_with_mapped_variant) - - assert variation is not None - assert variation.model_dump().get("type") in {"Allele", "CisPhasedBlock"} - - def test_annotation_eligibility_returns_boolean_for_persisted_variant(self, setup_lib_db_with_mapped_variant): - # Make score presence explicit so a negative result is due to missing calibrations. - setup_lib_db_with_mapped_variant.variant.data = {"score_data": {"score": 1.0}} - - pathogenicity_allowed = can_annotate_variant_for_pathogenicity_evidence(setup_lib_db_with_mapped_variant) - functional_allowed = can_annotate_variant_for_functional_statement(setup_lib_db_with_mapped_variant) - - # DB fixture score sets do not include calibrations by default, so both should be False. - assert setup_lib_db_with_mapped_variant.variant.score_set.score_calibrations == [] - assert pathogenicity_allowed is False - assert functional_allowed is False - - -@pytest.mark.unit -class TestSelectStrongestFunctionalCalibrationUnit: - """Unit tests for select_strongest_functional_calibration function.""" - - def test_returns_none_for_empty_calibrations(self, mock_mapped_variant): - """Test that empty calibration list returns None.""" - calibration, functional_range = select_strongest_functional_calibration(mock_mapped_variant, []) - assert calibration is None - assert functional_range is None - - def test_returns_single_calibration(self, mock_mapped_variant_with_functional_calibration_score_set): - """Test that single calibration is returned.""" - mapped_variant = mock_mapped_variant_with_functional_calibration_score_set - calibrations = mapped_variant.variant.score_set.score_calibrations - - calibration, functional_range = select_strongest_functional_calibration(mapped_variant, calibrations) - - assert calibration is not None - assert calibration == calibrations[0] - assert functional_range is not None - - def test_returns_first_when_all_agree(self, mock_mapped_variant_with_functional_calibration_score_set): - """Test that first calibration is returned when all have same classification.""" - from copy import deepcopy - - mapped_variant = mock_mapped_variant_with_functional_calibration_score_set - # Create multiple calibrations with same classification - calibration1 = mapped_variant.variant.score_set.score_calibrations[0] - calibration2 = deepcopy(calibration1) - calibration2.id = 999 - calibrations = [calibration1, calibration2] - - calibration, functional_range = select_strongest_functional_calibration(mapped_variant, calibrations) - - assert calibration is not None - assert calibration == calibrations[0] # Should return the first one - - def test_defaults_to_normal_on_conflict(self, mock_mapped_variant_with_functional_calibration_score_set): - """Test that normal classification is preferred when there are conflicts.""" - from copy import deepcopy - from unittest.mock import MagicMock, patch - - from mavedb.lib.annotation.classification import ExperimentalVariantFunctionalImpactClassification - - mapped_variant = mock_mapped_variant_with_functional_calibration_score_set - calibration1 = mapped_variant.variant.score_set.score_calibrations[0] - calibration2 = deepcopy(calibration1) - calibration2.id = 999 - calibrations = [calibration1, calibration2] - - # Mock to return different classifications - with patch( - "mavedb.lib.annotation.util.functional_classification_of_variant", - side_effect=[ - (MagicMock(label="Abnormal"), ExperimentalVariantFunctionalImpactClassification.ABNORMAL), - (MagicMock(label="Normal"), ExperimentalVariantFunctionalImpactClassification.NORMAL), - ], - ): - calibration, functional_range = select_strongest_functional_calibration(mapped_variant, calibrations) - - # Should return the normal classification (second one) - assert calibration == calibration2 - assert functional_range.label == "Normal" - - def test_returns_first_calibration_when_no_variants_in_ranges( - self, mock_mapped_variant_with_functional_calibration_score_set - ): - """Test that first calibration with None range is returned when variant is not in any functional range.""" - from unittest.mock import patch - - from mavedb.lib.annotation.classification import ExperimentalVariantFunctionalImpactClassification - - mapped_variant = mock_mapped_variant_with_functional_calibration_score_set - calibrations = mapped_variant.variant.score_set.score_calibrations - - # Mock to return None range but INDETERMINATE classification (variant not in any range) - with patch( - "mavedb.lib.annotation.util.functional_classification_of_variant", - return_value=(None, ExperimentalVariantFunctionalImpactClassification.INDETERMINATE), - ): - calibration, functional_range = select_strongest_functional_calibration(mapped_variant, calibrations) - - # Should return first calibration with None range (indicating variant not in any range) - assert calibration == calibrations[0] - assert functional_range is None - - -@pytest.mark.unit -class TestSelectStrongestPathogenicityCalibrationUnit: - """Unit tests for select_strongest_pathogenicity_calibration function.""" - - def test_returns_none_for_empty_calibrations(self, mock_mapped_variant): - """Test that empty calibration list returns None.""" - calibration, functional_range = select_strongest_pathogenicity_calibration(mock_mapped_variant, []) - assert calibration is None - assert functional_range is None - - def test_returns_single_calibration(self, mock_mapped_variant_with_pathogenicity_calibration_score_set): - """Test that single calibration is returned.""" - mapped_variant = mock_mapped_variant_with_pathogenicity_calibration_score_set - calibrations = mapped_variant.variant.score_set.score_calibrations - - calibration, functional_range = select_strongest_pathogenicity_calibration(mapped_variant, calibrations) - - assert calibration is not None - assert calibration == calibrations[0] - assert functional_range is not None - - def test_selects_strongest_evidence_strength(self, mock_mapped_variant_with_pathogenicity_calibration_score_set): - """Test that calibration with strongest evidence is selected.""" - from copy import deepcopy - from unittest.mock import MagicMock, patch - - from ga4gh.va_spec.acmg_2015 import VariantPathogenicityEvidenceLine - from ga4gh.va_spec.base.enums import StrengthOfEvidenceProvided - - mapped_variant = mock_mapped_variant_with_pathogenicity_calibration_score_set - calibration1 = mapped_variant.variant.score_set.score_calibrations[0] - calibration2 = deepcopy(calibration1) - calibration2.id = 999 - calibrations = [calibration1, calibration2] - - # Mock to return different evidence strengths - with patch( - "mavedb.lib.annotation.util.pathogenicity_classification_of_variant", - side_effect=[ - ( - MagicMock(label="Moderate"), - VariantPathogenicityEvidenceLine.Criterion.PS3, - StrengthOfEvidenceProvided.MODERATE, - ), - ( - MagicMock(label="Strong"), - VariantPathogenicityEvidenceLine.Criterion.PS3, - StrengthOfEvidenceProvided.VERY_STRONG, - ), - ], - ): - calibration, functional_range = select_strongest_pathogenicity_calibration(mapped_variant, calibrations) - - # Should return the one with VERY_STRONG evidence - assert calibration == calibration2 - assert functional_range.label == "Strong" - - def test_defaults_to_uncertain_on_tie(self, mock_mapped_variant_with_pathogenicity_calibration_score_set): - """Test that uncertain significance is returned when benign and pathogenic evidence tie.""" - from copy import deepcopy - from unittest.mock import MagicMock, patch - - from ga4gh.va_spec.acmg_2015 import VariantPathogenicityEvidenceLine - from ga4gh.va_spec.base.enums import StrengthOfEvidenceProvided - - mapped_variant = mock_mapped_variant_with_pathogenicity_calibration_score_set - calibration1 = mapped_variant.variant.score_set.score_calibrations[0] - calibration2 = deepcopy(calibration1) - calibration2.id = 999 - calibrations = [calibration1, calibration2] - - # Mock to return same evidence strength but different criteria (pathogenic vs benign) - with patch( - "mavedb.lib.annotation.util.pathogenicity_classification_of_variant", - side_effect=[ - ( - MagicMock(label="Pathogenic"), - VariantPathogenicityEvidenceLine.Criterion.PS3, - StrengthOfEvidenceProvided.STRONG, - ), - ( - MagicMock(label="Benign"), - VariantPathogenicityEvidenceLine.Criterion.BS3, - StrengthOfEvidenceProvided.STRONG, - ), - ], - ): - calibration, functional_range = select_strongest_pathogenicity_calibration(mapped_variant, calibrations) - - # Should return first calibration but None for range to indicate uncertain significance - assert calibration == calibration1 - assert functional_range is None - - def test_returns_classification_when_all_tied_are_same_type( - self, mock_mapped_variant_with_pathogenicity_calibration_score_set - ): - """Test that classification is returned normally when all tied candidates are the same type.""" - from copy import deepcopy - from unittest.mock import MagicMock, patch - - from ga4gh.va_spec.acmg_2015 import VariantPathogenicityEvidenceLine - from ga4gh.va_spec.base.enums import StrengthOfEvidenceProvided - - mapped_variant = mock_mapped_variant_with_pathogenicity_calibration_score_set - calibration1 = mapped_variant.variant.score_set.score_calibrations[0] - calibration2 = deepcopy(calibration1) - calibration2.id = 999 - calibrations = [calibration1, calibration2] - - # Mock to return same evidence strength and same type of criteria (both benign) - with patch( - "mavedb.lib.annotation.util.pathogenicity_classification_of_variant", - side_effect=[ - ( - MagicMock(label="Benign1"), - VariantPathogenicityEvidenceLine.Criterion.BS3, - StrengthOfEvidenceProvided.STRONG, - ), - ( - MagicMock(label="Benign2"), - VariantPathogenicityEvidenceLine.Criterion.BP1, - StrengthOfEvidenceProvided.STRONG, - ), - ], - ): - calibration, functional_range = select_strongest_pathogenicity_calibration(mapped_variant, calibrations) - - # Should return first calibration with its range (no conflict, so normal classification) - assert calibration == calibration1 - assert functional_range.label == "Benign1" - - def test_handles_none_evidence_strength(self, mock_mapped_variant_with_pathogenicity_calibration_score_set): - """Test that None evidence strength is handled correctly.""" - from copy import deepcopy - from unittest.mock import MagicMock, patch - - from ga4gh.va_spec.acmg_2015 import VariantPathogenicityEvidenceLine - from ga4gh.va_spec.base.enums import StrengthOfEvidenceProvided - - mapped_variant = mock_mapped_variant_with_pathogenicity_calibration_score_set - calibration1 = mapped_variant.variant.score_set.score_calibrations[0] - calibration2 = deepcopy(calibration1) - calibration2.id = 999 - calibrations = [calibration1, calibration2] - - # Mock with None and actual strength - with patch( - "mavedb.lib.annotation.util.pathogenicity_classification_of_variant", - side_effect=[ - (MagicMock(label="No Strength"), VariantPathogenicityEvidenceLine.Criterion.PS3, None), - ( - MagicMock(label="Moderate"), - VariantPathogenicityEvidenceLine.Criterion.PS3, - StrengthOfEvidenceProvided.MODERATE, - ), - ], - ): - calibration, functional_range = select_strongest_pathogenicity_calibration(mapped_variant, calibrations) - - # Should return the one with actual strength (second) - assert calibration == calibration2 - assert functional_range.label == "Moderate" - - def test_returns_first_calibration_when_no_variants_in_ranges( - self, mock_mapped_variant_with_pathogenicity_calibration_score_set - ): - """Test that first calibration with None range is returned when variant is not in any functional range.""" - from unittest.mock import patch - - from ga4gh.va_spec.acmg_2015 import VariantPathogenicityEvidenceLine - - mapped_variant = mock_mapped_variant_with_pathogenicity_calibration_score_set - calibrations = mapped_variant.variant.score_set.score_calibrations - - # Mock to return None range for all calibrations (variant not in any range) - with patch( - "mavedb.lib.annotation.util.pathogenicity_classification_of_variant", - return_value=(None, VariantPathogenicityEvidenceLine.Criterion.PS3, None), - ): - calibration, functional_range = select_strongest_pathogenicity_calibration(mapped_variant, calibrations) - - # Should return first calibration with None range (indicating variant not in any range) - assert calibration == calibrations[0] - assert functional_range is None diff --git a/tests/lib/test_vrs.py b/tests/lib/test_vrs.py new file mode 100644 index 000000000..fae8dda67 --- /dev/null +++ b/tests/lib/test_vrs.py @@ -0,0 +1,96 @@ +# ruff: noqa: E402 + +"""Tests for mavedb.lib.vrs — deserialization of stored post_mapped dicts into GA4GH VRS objects.""" + +from copy import deepcopy + +import pytest + +pytest.importorskip("psycopg2") + +from mavedb.lib.vrs import vrs_object_from_mapped_variant +from tests.helpers.constants import ( + TEST_SEQUENCE_LOCATION_ACCESSION, + TEST_VALID_POST_MAPPED_VRS_ALLELE, + TEST_VALID_POST_MAPPED_VRS_ALLELE_LENGTH_EXPRESSION, + TEST_VALID_POST_MAPPED_VRS_ALLELE_RLE, +) + + +@pytest.mark.unit +class TestVariationExtractionUnit: + @pytest.mark.parametrize( + "variation_version", + [{"variation": TEST_VALID_POST_MAPPED_VRS_ALLELE}, TEST_VALID_POST_MAPPED_VRS_ALLELE], + ids=["vrs13_wrapped_variation", "vrs2_direct_allele"], + ) + def test_vrs_object_from_post_mapped_variation(self, variation_version): + result = vrs_object_from_mapped_variant(variation_version).model_dump() + + assert result["location"]["id"] == TEST_SEQUENCE_LOCATION_ACCESSION + assert result["location"]["start"] == 5 + assert result["location"]["end"] == 6 + + @pytest.mark.parametrize( + "allele_dict, expected_state_type, expected_state_fields", + [ + ( + TEST_VALID_POST_MAPPED_VRS_ALLELE, + "LiteralSequenceExpression", + {"sequence": "F"}, + ), + ( + TEST_VALID_POST_MAPPED_VRS_ALLELE_RLE, + "ReferenceLengthExpression", + {"length": 5, "repeatSubunitLength": 5}, + ), + ( + TEST_VALID_POST_MAPPED_VRS_ALLELE_LENGTH_EXPRESSION, + "LengthExpression", + {"length": 5}, + ), + ], + ids=["lse_state", "rle_state", "length_expression_state"], + ) + def test_allele_state_type_is_deserialized_correctly(self, allele_dict, expected_state_type, expected_state_fields): + result = vrs_object_from_mapped_variant(allele_dict).model_dump() + + assert result["state"]["type"] == expected_state_type + for k, v in expected_state_fields.items(): + assert result["state"][k] == v + + def test_unknown_allele_state_type_raises_value_error(self): + allele_dict = { + **TEST_VALID_POST_MAPPED_VRS_ALLELE, + "state": {"type": "UnknownFutureExpression", "someField": "someValue"}, + } + + with pytest.raises(ValueError, match="Unsupported VRS Allele state type"): + vrs_object_from_mapped_variant(allele_dict) + + @pytest.mark.parametrize( + "member_dict, expected_state_type", + [ + (TEST_VALID_POST_MAPPED_VRS_ALLELE, "LiteralSequenceExpression"), + (TEST_VALID_POST_MAPPED_VRS_ALLELE_RLE, "ReferenceLengthExpression"), + (TEST_VALID_POST_MAPPED_VRS_ALLELE_LENGTH_EXPRESSION, "LengthExpression"), + ], + ids=["lse_members", "rle_members", "length_expression_members"], + ) + def test_cis_phased_block_member_state_types_are_deserialized_correctly(self, member_dict, expected_state_type): + mapping_results = {"type": "CisPhasedBlock", "members": [deepcopy(member_dict), deepcopy(member_dict)]} + + result = vrs_object_from_mapped_variant(mapping_results).model_dump() + + assert result["type"] == "CisPhasedBlock" + assert len(result["members"]) == 2 + assert result["members"][0]["state"]["type"] == expected_state_type + + +@pytest.mark.integration +class TestVrsIntegration: + def test_vrs_object_from_persisted_post_mapped(self, setup_lib_db_with_mapped_variant): + variation = vrs_object_from_mapped_variant(TEST_VALID_POST_MAPPED_VRS_ALLELE) + + assert variation is not None + assert variation.model_dump().get("type") in {"Allele", "CisPhasedBlock"} From a8363a6f0a3d52de81ec74655496ef17aa60e995 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Sat, 18 Jul 2026 10:25:24 -0700 Subject: [PATCH 090/252] feat(annotation): serve variant annotation from the Allele substrate Move annotation serving off the legacy MappedVariant model onto the MappingRecord/Allele substrate written by the mapping pipeline. New score sets, whose mapping data lands only on the new substrate, previously enumerated no annotatable variants and silently returned nothing. - Replace get_current_mapped_variants_for_annotation with get_annotatable_variants, querying live MappingRecord/Allele links and threading an as_of instant for temporal reconstruction. - Thread as_of end to end through the three streaming annotation endpoints: into get_annotatable_variants (set enumeration) as well as the per-variant context, so the set and its annotations share one instant and cannot drift. - Return 200 with an empty stream when a score set exists but has no annotatable variants (never mapped, or none live at as_of); 404 is reserved for an unresolvable URN or a permission failure, so as_of acts as a filter and never manufactures a 404. - Derive mavedb_vrs_contribution provenance from the live MappingRecord via VariantAnnotationContext instead of MappedVariant. - Add find_variants_by_vrs_identifier over Allele.vrs_digest, and repoint the public-data export onto get_annotatable_variants. --- src/mavedb/lib/alleles.py | 29 ++++ src/mavedb/lib/annotation/contribution.py | 12 +- src/mavedb/lib/score_sets.py | 79 ++++++---- src/mavedb/routers/score_sets.py | 180 +++++++++++++--------- src/mavedb/scripts/export_public_data.py | 12 +- tests/helpers/util/score_set.py | 34 ++++ tests/lib/annotation/test_contribution.py | 26 ++-- tests/routers/test_score_set.py | 110 +++++++++---- 8 files changed, 336 insertions(+), 146 deletions(-) diff --git a/src/mavedb/lib/alleles.py b/src/mavedb/lib/alleles.py index bf6a340d4..08f13ef0b 100644 --- a/src/mavedb/lib/alleles.py +++ b/src/mavedb/lib/alleles.py @@ -17,6 +17,7 @@ from mavedb.models.allele import Allele from mavedb.models.mapping_record import MappingRecord from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.variant import Variant def get_live_record_allele_links( @@ -51,6 +52,34 @@ def get_live_record_allele_links( ) +def find_variants_by_vrs_identifier( + db: Session, identifier: str, *, as_of: Optional[datetime] = None +) -> list[tuple[Variant, Allele]]: + """Resolve a GA4GH VRS identifier to the variants whose mapping links an allele bearing it. + + Matches ``identifier`` against a deduplicated allele's ``vrs_digest``, then walks ``Allele → + MappingRecordAllele → MappingRecord → Variant``. Because alleles are deduplicated by ``vrs_digest`` + and shared across variants/score sets, one identifier can resolve to several variants; the + paired allele carries the ClinGen id + level the caller surfaces alongside the URN. + + The live link + record set (``valid_to IS NULL`` / ``as_of``) is used to filter results. Returns distinct + ``(variant, matched_allele)`` pairs. Include an ``as_of`` timestamp to reconstruct the set as it stood + at a past instant. + """ + stmt = ( + select(Variant, Allele) + .select_from(Allele) + .join(MappingRecordAllele, MappingRecordAllele.allele_id == Allele.id) + .join(MappingRecord, MappingRecord.id == MappingRecordAllele.mapping_record_id) + .join(Variant, Variant.id == MappingRecord.variant_id) + .where(Allele.vrs_digest == identifier) + .where(MappingRecordAllele.live_at(as_of)) + .where(MappingRecord.live_at(as_of)) + ) + + return [(row[0], row[1]) for row in db.execute(stmt).unique().all()] + + def get_allele_translations(db: Session, allele_id: int, *, as_of: Optional[datetime] = None) -> list[Allele]: """Return the cross-layer equivalence set of ``allele_id``: every allele co-linked to a ``MappingRecord`` that links it (the anchor allele itself included), spanning the genomic, coding, diff --git a/src/mavedb/lib/annotation/contribution.py b/src/mavedb/lib/annotation/contribution.py index 4b0de881f..839721b5f 100644 --- a/src/mavedb/lib/annotation/contribution.py +++ b/src/mavedb/lib/annotation/contribution.py @@ -9,8 +9,8 @@ mavedb_user_agent, mavedb_vrs_agent, ) +from mavedb.lib.annotation.context import VariantAnnotationContext from mavedb.lib.types.annotation import ResourceWithCreationModificationDates -from mavedb.models.mapped_variant import MappedVariant from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.user import User @@ -31,17 +31,17 @@ def mavedb_api_contribution() -> Contribution: ) -def mavedb_vrs_contribution(mapped_variant: MappedVariant) -> Contribution: +def mavedb_vrs_contribution(context: VariantAnnotationContext) -> Contribution: """ Create a [VA Contribution](https://va-ga4gh.readthedocs.io/en/latest/core-information-model/entities/activities/contribution.html#contribution) - object from the provided mapped variant. + object from the variant's live mapping record (the VRS mapper's provenance). """ return Contribution( name="MaveDB VRS Mapper", description="Contribution from the MaveDB VRS mapping software", - # Guaranteed to be a str via DB constraints. - contributor=mavedb_vrs_agent(mapped_variant.mapping_api_version), # type: ignore - date=mapped_variant.mapped_date, # type: ignore + # Both guaranteed non-null via DB constraints on MappingRecord. + contributor=mavedb_vrs_agent(context.record.mapping_api_version), # type: ignore + date=context.record.mapped_date, # type: ignore activityType="human genome sequence mapping process", ) diff --git a/src/mavedb/lib/score_sets.py b/src/mavedb/lib/score_sets.py index 76cbab9e6..3c1559510 100644 --- a/src/mavedb/lib/score_sets.py +++ b/src/mavedb/lib/score_sets.py @@ -3,6 +3,7 @@ import logging import re from collections import Counter, defaultdict +from datetime import datetime from operator import attrgetter from typing import TYPE_CHECKING, Any, BinaryIO, Iterable, List, Optional, Sequence @@ -10,8 +11,8 @@ import pandas as pd from pandas.testing import assert_index_equal from sqlalchemy import Integer, and_, cast, func, or_, select -from sqlalchemy.orm import Query, Session, aliased, contains_eager, joinedload, selectinload from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Query, Session, aliased, contains_eager, joinedload, selectinload from mavedb.lib.exceptions import ValidationError from mavedb.lib.logging.context import logging_context, save_to_logging_context @@ -29,6 +30,9 @@ from mavedb.lib.validation.constants.general import null_values_list from mavedb.lib.validation.utilities import is_null as validate_is_null from mavedb.lib.variants import get_digest_from_post_mapped, get_hgvs_from_post_mapped, is_hgvs_g, is_hgvs_p +from mavedb.models.allele import Allele +from mavedb.models.clinical_control import ClinvarControl +from mavedb.models.clinical_control_mapped_variant import mapped_variants_clinical_controls_association_table from mavedb.models.contributor import Contributor from mavedb.models.controlled_keyword import ControlledKeyword from mavedb.models.doi_identifier import DoiIdentifier @@ -38,10 +42,10 @@ from mavedb.models.experiment_controlled_keyword import ExperimentControlledKeywordAssociation from mavedb.models.experiment_publication_identifier import ExperimentPublicationIdentifierAssociation from mavedb.models.experiment_set import ExperimentSet -from mavedb.models.clinical_control import ClinvarControl -from mavedb.models.clinical_control_mapped_variant import mapped_variants_clinical_controls_association_table from mavedb.models.gnomad_variant import GnomADVariant from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.publication_identifier import PublicationIdentifier from mavedb.models.refseq_identifier import RefseqIdentifier from mavedb.models.refseq_offset import RefseqOffset @@ -57,7 +61,7 @@ from mavedb.models.uniprot_offset import UniprotOffset from mavedb.models.user import User from mavedb.models.variant import Variant -from mavedb.view_models.search import ScoreSetsSearch, ControlledKeywordFilterOption +from mavedb.view_models.search import ControlledKeywordFilterOption, ScoreSetsSearch if TYPE_CHECKING: from mavedb.lib.permissions import Action @@ -554,35 +558,50 @@ def find_publish_or_private_superseded_score_set_tail( return score_set -def get_current_mapped_variants_for_annotation(db: Session, score_set: ScoreSet) -> Sequence[MappedVariant]: - """ - Load the current mapped variants for a score set with the relationships required to build VA-Spec - annotations eagerly loaded. - - This is the single source of truth for the eager-load shape shared by the annotated-variant - streaming endpoints and the public data export. The annotation builders reach through - ``MappedVariant.variant.score_set`` for publications, contributors, license, experiment, and score - calibrations, so each of those is loaded up front to avoid per-variant lazy loads. +def get_annotatable_variants( + db: Session, score_set: ScoreSet, *, as_of: Optional[datetime] = None +) -> Sequence[Variant]: + """Load a score set's annotatable variants from its variant's allele-graph, with the score-set + relationships the VA-Spec builders need eagerly loaded. + + A variant is *annotatable* when it has a live ``MappingRecord`` whose authoritative allele carries a + ``post_mapped`` VRS — exactly the condition under which :func:`annotation.context.variant_annotation_context` + yields a context. This is the single source of truth for the eager-load shape shared by the + annotated-variant streaming endpoints and the public data export. The annotation builders reach through + ``variant.score_set`` for publications, contributors, license, experiment, and score calibrations, so + each of those is loaded up front to avoid per-variant lazy loads. + + ``as_of`` reconstructs the annotatable set as it stood at a past instant: the same half-open + ``[valid_from, valid_to)`` predicate is applied to both the record and the authoritative link, so the + set is evaluated at one moment (matching :func:`annotation.context.variant_annotation_context`'s own + ``as_of``, so a per-variant context can be rebuilt at the same instant downstream). Defaults to the + currently-live rows. """ return ( - db.query(MappedVariant) - .join(MappedVariant.variant) - .join(Variant.score_set) - .filter(Variant.score_set_id == score_set.id) - .filter(MappedVariant.current.is_(True)) - .options( - contains_eager(MappedVariant.variant).contains_eager(Variant.score_set), - contains_eager(MappedVariant.variant) - .contains_eager(Variant.score_set) - .selectinload(ScoreSet.publication_identifier_associations), - contains_eager(MappedVariant.variant).contains_eager(Variant.score_set).selectinload(ScoreSet.created_by), - contains_eager(MappedVariant.variant).contains_eager(Variant.score_set).selectinload(ScoreSet.modified_by), - contains_eager(MappedVariant.variant).contains_eager(Variant.score_set).selectinload(ScoreSet.license), - contains_eager(MappedVariant.variant).contains_eager(Variant.score_set).selectinload(ScoreSet.experiment), - contains_eager(MappedVariant.variant) - .contains_eager(Variant.score_set) - .selectinload(ScoreSet.score_calibrations), + db.scalars( + select(Variant) + .join(MappingRecord, and_(MappingRecord.variant_id == Variant.id, MappingRecord.live_at(as_of))) + .join( + MappingRecordAllele, + and_( + MappingRecordAllele.mapping_record_id == MappingRecord.id, + MappingRecordAllele.is_authoritative.is_(True), + MappingRecordAllele.live_at(as_of), + ), + ) + .join(Allele, and_(Allele.id == MappingRecordAllele.allele_id, Allele.post_mapped.isnot(None))) + .where(Variant.score_set_id == score_set.id) + .options( + selectinload(Variant.score_set).selectinload(ScoreSet.publication_identifier_associations), + selectinload(Variant.score_set).selectinload(ScoreSet.created_by), + selectinload(Variant.score_set).selectinload(ScoreSet.modified_by), + selectinload(Variant.score_set).selectinload(ScoreSet.license), + selectinload(Variant.score_set).selectinload(ScoreSet.experiment), + selectinload(Variant.score_set).selectinload(ScoreSet.score_calibrations), + ) + .distinct() ) + .unique() .all() ) diff --git a/src/mavedb/routers/score_sets.py b/src/mavedb/routers/score_sets.py index 7a68e8550..506ed0b5b 100644 --- a/src/mavedb/routers/score_sets.py +++ b/src/mavedb/routers/score_sets.py @@ -27,12 +27,14 @@ variant_pathogenicity_statement, variant_study_result, ) +from mavedb.lib.annotation.context import variant_annotation_context from mavedb.lib.annotation.exceptions import MappingDataDoesntExistException from mavedb.lib.authorization import ( get_current_user, require_current_user, require_current_user_with_email, ) +from mavedb.lib.clinical_controls import get_clinical_control_options, get_clinical_controls_with_variant_urns from mavedb.lib.contributors import find_or_create_contributor from mavedb.lib.exceptions import MixedTargetError, NonexistentOrcidUserError from mavedb.lib.experiments import enrich_experiment_with_num_score_sets @@ -47,7 +49,6 @@ logging_context, save_to_logging_context, ) -from mavedb.lib.clinical_controls import get_clinical_control_options, get_clinical_controls_with_variant_urns from mavedb.lib.permissions import Action, assert_permission, has_permission from mavedb.lib.score_calibrations import create_score_calibration from mavedb.lib.score_set_variants import get_lean_score_set_variants @@ -56,7 +57,7 @@ csv_data_to_df, fetch_score_set_search_filter_options, find_meta_analyses_for_experiment_sets, - get_current_mapped_variants_for_annotation, + get_annotatable_variants, get_score_set_variants_as_csv, is_replaces_id_unique_violation, refresh_variant_urns, @@ -1185,10 +1186,13 @@ def get_score_set_mapped_variants( return mapped_variants -def _stream_generated_annotations(mapped_variants, annotation_function): +def _stream_generated_annotations(db, variants, annotation_function, as_of=None): """ Generator function to stream annotations as pure NDJSON data. + Builds each variant's :class:`VariantAnnotationContext` from the mapping-record substrate and passes it + to ``annotation_function``; variants with no live mapping data yield a null annotation. + Metadata should be provided via HTTP headers: - X-Total-Count: Total number of variants - X-Processing-Started: ISO timestamp when processing began @@ -1198,20 +1202,21 @@ def _stream_generated_annotations(mapped_variants, annotation_function): consumed via Server-Sent Events if needed. """ start_time = time.time() - total_variants = len(mapped_variants) + total_variants = len(variants) processed_count = 0 - logger.info(f"Starting streaming processing of {total_variants} mapped variants") + logger.info(f"Starting streaming processing of {total_variants} variants") - for i, mv in enumerate(mapped_variants): + for i, variant in enumerate(variants): try: - annotation = annotation_function(mv) + context = variant_annotation_context(db, variant, as_of=as_of) + annotation = annotation_function(context) if context is not None else None except MappingDataDoesntExistException: - logger.debug(f"Mapping data does not exist for variant {mv.variant.urn}.") + logger.debug(f"Mapping data does not exist for variant {variant.urn}.") annotation = None # Send pure result data (no wrapper) result = { - "variant_urn": mv.variant.urn, + "variant_urn": variant.urn, "annotation": annotation.model_dump(exclude_none=True) if annotation else None, } yield json.dumps(result, default=str) + "\n" @@ -1258,11 +1263,11 @@ def _stream_generated_annotations(mapped_variants, annotation_function): status_code=200, response_model=dict[str, Optional[VariantPathogenicityStatement]], response_model_exclude_none=True, - summary="Get pathogenicity statement annotations for mapped variants within a score set", + summary="Get pathogenicity statement annotations for variants within a score set", responses={ 200: { "content": {"application/x-ndjson": {}}, - "description": "Stream pathogenicity statement annotations for mapped variants.", + "description": "Stream pathogenicity statement annotations for variants.", }, **ACCESS_CONTROL_ERROR_RESPONSES, }, @@ -1270,22 +1275,31 @@ def _stream_generated_annotations(mapped_variants, annotation_function): def get_score_set_annotated_variants( *, urn: str, + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the molecular layer (Cat-VRS membership, VEP/gnomAD/ClinVar annotations) as it " + "stood at this instant, over the variant's fixed score. ISO 8601, ideally timezone-aware. " + "Content valid-time only — it never re-selects a score-set version, and scores/classifications " + "are as-of-invariant. Defaults to current." + ), + ), db: Session = Depends(deps.get_db), user_data: Optional[UserData] = Depends(get_current_user), ) -> Any: """ Retrieve annotated variants with pathogenicity statements for a given score set. - This endpoint streams pathogenicity evidence lines for all current mapped variants + This endpoint streams pathogenicity evidence lines for all current annotated variants associated with a specific score set. The response is returned as newline-delimited JSON (NDJSON) format for efficient processing of large datasets. NDJSON Response Format: - Each line in the response corresponds to a mapped variant and contains a JSON + Each line in the response corresponds to an annotated variant and contains a JSON object with the following structure: ``` { - "variant_urn": "", + "variant_urn": "", "annotation": { ... // Pathogenicity evidence line details } @@ -1301,23 +1315,27 @@ def get_score_set_annotated_variants( Returns: Any: StreamingResponse containing newline-delimited JSON with pathogenicity - evidence lines for each mapped variant. Response includes headers with + evidence lines for each annotated variant. Response includes headers with total count, processing start time, and stream type information. + A score set that exists but has no annotatable variants (never mapped, or none live at ``as_of``) + streams an empty body with ``X-Total-Count: 0`` — an empty collection, not a 404. + Raises: HTTPException: 404 error if the score set with the given URN is not found. - HTTPException: 404 error if no mapped variants are associated with the score set. HTTPException: 403 error if the user lacks READ permissions for the score set. Note: This function logs the request context and validates user permissions before - processing. Only current (non-historical) mapped variants are included in - the response. + processing. Use the `as_of` parameter to reconstruct the molecular layer as it stood at a specific + instant, over the variant's fixed score. The response is streamed to allow for efficient handling + of large datasets, and progress updates are logged for monitoring purposes. """ save_to_logging_context( { "requested_resource": urn, "resource_property": "annotated-variants/pathogenicity-statement", + "as_of": as_of, } ) @@ -1331,20 +1349,17 @@ def get_score_set_annotated_variants( assert_permission(user_data, score_set, Action.READ) - mapped_variants = get_current_mapped_variants_for_annotation(db, score_set) - - if not mapped_variants: - logger.info(msg="No mapped variants are associated with the requested score set.", extra=logging_context()) - raise HTTPException( - status_code=404, - detail=f"No mapped variants associated with score set URN {urn} were found. Could not construct evidence lines.", - ) + variants = get_annotatable_variants(db, score_set, as_of=as_of) + # An existing score set with no annotatable variants (never mapped, or none live at as_of) streams an + # empty NDJSON body with X-Total-Count: 0 — an empty collection, not a 404. 404 is reserved for an + # unresolvable URN or a permission failure, never for a filter (as_of) matching nothing. return StreamingResponse( - _stream_generated_annotations(mapped_variants, variant_pathogenicity_statement), + _stream_generated_annotations(db, variants, variant_pathogenicity_statement, as_of=as_of), media_type="application/x-ndjson", headers={ - "X-Total-Count": str(len(mapped_variants)), + "X-As-Of": as_of.isoformat() if as_of is not None else "current", + "X-Total-Count": str(len(variants)), "X-Processing-Started": datetime.now().isoformat(), "X-Stream-Type": "pathogenicity-evidence-line", "Access-Control-Expose-Headers": "X-Total-Count, X-Processing-Started, X-Stream-Type", @@ -1357,11 +1372,11 @@ def get_score_set_annotated_variants( status_code=200, response_model=dict[str, Optional[Statement]], response_model_exclude_none=True, - summary="Get functional impact statement annotations for mapped variants within a score set", + summary="Get functional impact statement annotations for annotated variants within a score set", responses={ 200: { "content": {"application/x-ndjson": {}}, - "description": "Stream functional impact statement annotations for mapped variants.", + "description": "Stream functional impact statement annotations for annotated variants.", }, **ACCESS_CONTROL_ERROR_RESPONSES, }, @@ -1369,22 +1384,31 @@ def get_score_set_annotated_variants( def get_score_set_annotated_variants_functional_statement( *, urn: str, + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the molecular layer (Cat-VRS membership, VEP/gnomAD/ClinVar annotations) as it " + "stood at this instant, over the variant's fixed score. ISO 8601, ideally timezone-aware. " + "Content valid-time only — it never re-selects a score-set version, and scores/classifications " + "are as-of-invariant. Defaults to current." + ), + ), db: Session = Depends(deps.get_db), user_data: Optional[UserData] = Depends(get_current_user), ): """ Retrieve functional impact statements for annotated variants in a score set. - This endpoint streams functional impact statements for all current mapped variants + This endpoint streams functional impact statements for all current annotated variants associated with a specific score set. The response is delivered as newline-delimited JSON (NDJSON) format. NDJSON Response Format: - Each line in the response corresponds to a mapped variant and contains a JSON + Each line in the response corresponds to an annotated variant and contains a JSON object with the following structure: ``` { - "variant_urn": "", + "variant_urn": "", "annotation": { ... // Functional impact statement details } @@ -1398,20 +1422,28 @@ def get_score_set_annotated_variants_functional_statement( Returns: StreamingResponse: NDJSON stream containing functional impact statements for each - mapped variant. Response includes headers with total count, processing start time, + annotated variant. Response includes headers with total count, processing start time, and stream type information. Raises: HTTPException: - 404 if the score set with the given URN is not found - - 404 if no mapped variants are associated with the score set + - 404 if no annotated variants are associated with the score set - 403 if the user lacks READ permission for the score set Note: - Only current (non-historical) mapped variants are included in the response. - The function requires appropriate read permissions on the score set. + The function requires appropriate read permissions on the score set. Use the `as_of` + parameter to reconstruct the molecular layer as it stood at a specific instant, over + the variant's fixed score. The response is streamed to allow for efficient handling of + large datasets, and progress updates are logged for monitoring purposes. """ - save_to_logging_context({"requested_resource": urn, "resource_property": "annotated-variants/functional-statement"}) + save_to_logging_context( + { + "requested_resource": urn, + "resource_property": "annotated-variants/functional-statement", + "as_of": as_of, + } + ) score_set = db.query(ScoreSet).filter(ScoreSet.urn == urn).first() if not score_set: @@ -1423,20 +1455,17 @@ def get_score_set_annotated_variants_functional_statement( assert_permission(user_data, score_set, Action.READ) - mapped_variants = get_current_mapped_variants_for_annotation(db, score_set) - - if not mapped_variants: - logger.info(msg="No mapped variants are associated with the requested score set.", extra=logging_context()) - raise HTTPException( - status_code=404, - detail=f"No mapped variants associated with score set URN {urn} were found. Could not construct functional impact statements.", - ) + variants = get_annotatable_variants(db, score_set, as_of=as_of) + # An existing score set with no annotatable variants (never mapped, or none live at as_of) streams an + # empty NDJSON body with X-Total-Count: 0 — an empty collection, not a 404. 404 is reserved for an + # unresolvable URN or a permission failure, never for a filter (as_of) matching nothing. return StreamingResponse( - _stream_generated_annotations(mapped_variants, variant_functional_impact_statement), + _stream_generated_annotations(db, variants, variant_functional_impact_statement, as_of=as_of), media_type="application/x-ndjson", headers={ - "X-Total-Count": str(len(mapped_variants)), + "X-As-Of": as_of.isoformat() if as_of is not None else "current", + "X-Total-Count": str(len(variants)), "X-Processing-Started": datetime.now().isoformat(), "X-Stream-Type": "functional-impact-statement", "Access-Control-Expose-Headers": "X-Total-Count, X-Processing-Started, X-Stream-Type", @@ -1449,11 +1478,11 @@ def get_score_set_annotated_variants_functional_statement( status_code=200, response_model=dict[str, Optional[ExperimentalVariantFunctionalImpactStudyResult]], response_model_exclude_none=True, - summary="Get functional study result annotations for mapped variants within a score set", + summary="Get functional study result annotations for annotated variants within a score set", responses={ 200: { "content": {"application/x-ndjson": {}}, - "description": "Stream functional study result annotations for mapped variants.", + "description": "Stream functional study result annotations for annotated variants.", }, **ACCESS_CONTROL_ERROR_RESPONSES, }, @@ -1461,22 +1490,31 @@ def get_score_set_annotated_variants_functional_statement( def get_score_set_annotated_variants_functional_study_result( *, urn: str, + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the molecular layer (Cat-VRS membership, VEP/gnomAD/ClinVar annotations) as it " + "stood at this instant, over the variant's fixed score. ISO 8601, ideally timezone-aware. " + "Content valid-time only — it never re-selects a score-set version, and scores/classifications " + "are as-of-invariant. Defaults to current." + ), + ), db: Session = Depends(deps.get_db), user_data: Optional[UserData] = Depends(get_current_user), ): """ Retrieve functional study results for annotated variants in a score set. - This endpoint streams functional study result annotations for all current mapped variants + This endpoint streams functional study result annotations for all current annotated variants associated with a specific score set. The results are returned as newline-delimited JSON (NDJSON) format for efficient streaming of large datasets. NDJSON Response Format: - Each line in the response corresponds to a mapped variant and contains a JSON + Each line in the response corresponds to a annotated variant and contains a JSON object with the following structure: ``` { - "variant_urn": "", + "variant_urn": "", "annotation": { ... // Functional study result details } @@ -1491,7 +1529,7 @@ def get_score_set_annotated_variants_functional_study_result( Returns: StreamingResponse: A streaming response containing functional study results in NDJSON format. Headers include: - - X-Total-Count: Total number of mapped variants being streamed + - X-Total-Count: Total number of annotated variants being streamed - X-Processing-Started: ISO timestamp when processing began - X-Stream-Type: Set to "functional-study-result" - Access-Control-Expose-Headers: Exposed headers for CORS @@ -1499,15 +1537,22 @@ def get_score_set_annotated_variants_functional_study_result( Raises: HTTPException: - 404 if the score set with the given URN is not found - - 404 if no mapped variants are associated with the score set + - 404 if no annotated variants are associated with the score set - 403 if the user lacks READ permission for the score set Notes: - - Only returns current mapped variants (MappedVariant.current == True) - - Eagerly loads related ScoreSet data including publications, users, license, and experiment - - Logs requests and errors for monitoring and debugging purposes + - The `as_of` parameter allows reconstruction of the molecular layer as it stood at a specific + instant, over the variant's fixed score. It is ISO 8601 formatted and ideally timezone-aware. + - The response is streamed to allow for efficient handling of large datasets, and progress updates + are logged for monitoring purposes. """ - save_to_logging_context({"requested_resource": urn, "resource_property": "annotated-variants/study-result"}) + save_to_logging_context( + { + "requested_resource": urn, + "resource_property": "annotated-variants/study-result", + "as_of": as_of, + } + ) score_set = db.query(ScoreSet).filter(ScoreSet.urn == urn).first() if not score_set: @@ -1519,20 +1564,17 @@ def get_score_set_annotated_variants_functional_study_result( assert_permission(user_data, score_set, Action.READ) - mapped_variants = get_current_mapped_variants_for_annotation(db, score_set) - - if not mapped_variants: - logger.info(msg="No mapped variants are associated with the requested score set.", extra=logging_context()) - raise HTTPException( - status_code=404, - detail=f"No mapped variants associated with score set URN {urn} were found. Could not construct study results.", - ) + variants = get_annotatable_variants(db, score_set, as_of=as_of) + # An existing score set with no annotatable variants (never mapped, or none live at as_of) streams an + # empty NDJSON body with X-Total-Count: 0 — an empty collection, not a 404. 404 is reserved for an + # unresolvable URN or a permission failure, never for a filter (as_of) matching nothing. return StreamingResponse( - _stream_generated_annotations(mapped_variants, variant_study_result), + _stream_generated_annotations(db, variants, variant_study_result, as_of=as_of), media_type="application/x-ndjson", headers={ - "X-Total-Count": str(len(mapped_variants)), + "X-As-Of": as_of.isoformat() if as_of is not None else "current", + "X-Total-Count": str(len(variants)), "X-Processing-Started": datetime.now().isoformat(), "X-Stream-Type": "functional-study-result", "Access-Control-Expose-Headers": "X-Total-Count, X-Processing-Started, X-Stream-Type", diff --git a/src/mavedb/scripts/export_public_data.py b/src/mavedb/scripts/export_public_data.py index cedc66608..07098cf4f 100644 --- a/src/mavedb/scripts/export_public_data.py +++ b/src/mavedb/scripts/export_public_data.py @@ -30,7 +30,8 @@ from sqlalchemy.orm import Session, joinedload, lazyload from mavedb.lib.annotation.annotate import variant_highest_level_annotation -from mavedb.lib.score_sets import get_current_mapped_variants_for_annotation, get_score_set_variants_as_csv +from mavedb.lib.annotation.context import variant_annotation_context +from mavedb.lib.score_sets import get_annotatable_variants, get_score_set_variants_as_csv from mavedb.models.experiment import Experiment from mavedb.models.experiment_set import ExperimentSet from mavedb.models.license import License @@ -211,16 +212,17 @@ def export_public_data(db: Session): # Write VA-Spec annotations NDJSON — mirrors the GET /api/v1/score-sets/{urn}/annotated-variants/* # streams, emitting one record per current mapped variant at its highest materialized VA level. - annotated_variants = get_current_mapped_variants_for_annotation(db, score_set) + annotatable_variants = get_annotatable_variants(db, score_set) va_lines = [] num_annotations = 0 - for mv in annotated_variants: - annotation = variant_highest_level_annotation(mv) + for variant in annotatable_variants: + context = variant_annotation_context(db, variant) + annotation = variant_highest_level_annotation(context) if context is not None else None if annotation is not None: num_annotations += 1 record = { - "variant_urn": mv.variant.urn, + "variant_urn": variant.urn, "annotation": annotation.model_dump(exclude_none=True) if annotation else None, } va_lines.append(json.dumps(record, default=str)) diff --git a/tests/helpers/util/score_set.py b/tests/helpers/util/score_set.py index 067c627fe..7e9022db7 100644 --- a/tests/helpers/util/score_set.py +++ b/tests/helpers/util/score_set.py @@ -205,6 +205,40 @@ def create_acc_score_set_with_variants( return score_set +def seed_annotation_substrate(db, score_set, *, skip_first=False): + """Give a score set's variants a live ``MappingRecord`` with one authoritative allele. + + The VA endpoints build a ``VariantAnnotationContext`` from the ``MappingRecord`` / ``Allele`` + substrate (not the legacy ``MappedVariant``), so a variant needs a live record with an + authoritative ``post_mapped`` allele before a study result / statement can be constructed. This + mirrors what the mapping pipeline writes. (Global seeding from ``mock_worker_vrs_mapping`` is + deferred to Slice 5.2/5.3, where it can be reconciled with the clinical-controls seeding.) + + With ``skip_first=True`` the first variant is left un-mapped on the new substrate (for the + "some variants were not mapped" cases). Returns the score set's variants in id order. + """ + variants = db.scalars( + select(VariantDbModel) + .join(ScoreSetDbModel) + .where(ScoreSetDbModel.urn == score_set["urn"]) + .order_by(VariantDbModel.id) + ).all() + for variant in variants[1:] if skip_first else variants: + seed_mapping_record( + db, + variant, + alleles=[ + AlleleSpec( + digest=f"va-allele-{variant.id}", + is_authoritative=True, + clingen_allele_id=f"CA{variant.id}", + post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X, + ) + ], + ) + return variants + + def link_clinical_controls_to_alleles(db, score_set): """Seed the new-model annotation graph for a score set's first two variants and link the two seeded ClinVar controls to their alleles via ``ClinvarAlleleLink``. diff --git a/tests/lib/annotation/test_contribution.py b/tests/lib/annotation/test_contribution.py index 51597db37..60990baf6 100644 --- a/tests/lib/annotation/test_contribution.py +++ b/tests/lib/annotation/test_contribution.py @@ -23,14 +23,17 @@ mavedb_score_calibration_contribution, mavedb_vrs_contribution, ) +from mavedb.lib.vrs import vrs_object_from_mapped_variant from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.user import User +from tests.helpers.constants import TEST_VALID_POST_MAPPED_VRS_ALLELE from tests.helpers.mocks.factories import ( create_mock_mapped_variant, create_mock_resource_with_dates, create_mock_score_calibration, create_mock_user, ) +from tests.lib.annotation.conftest import annotation_context_for @pytest.mark.unit @@ -85,14 +88,14 @@ class TestMavedbVrsContributionUnit: def test_returns_contribution_object(self): """Test that function returns proper Contribution object.""" mapped_variant = create_mock_mapped_variant() - contribution = mavedb_vrs_contribution(mapped_variant) + contribution = mavedb_vrs_contribution(annotation_context_for(mapped_variant)) assert isinstance(contribution, Contribution) def test_has_correct_name_and_description(self): """Test that contribution has correct name and description.""" mapped_variant = create_mock_mapped_variant() - contribution = mavedb_vrs_contribution(mapped_variant) + contribution = mavedb_vrs_contribution(annotation_context_for(mapped_variant)) assert contribution.name == "MaveDB VRS Mapper" assert contribution.description == "Contribution from the MaveDB VRS mapping software" @@ -100,7 +103,7 @@ def test_has_correct_name_and_description(self): def test_has_correct_activity_type(self): """Test that contribution has correct activity type.""" mapped_variant = create_mock_mapped_variant() - contribution = mavedb_vrs_contribution(mapped_variant) + contribution = mavedb_vrs_contribution(annotation_context_for(mapped_variant)) assert contribution.activityType == "human genome sequence mapping process" @@ -108,14 +111,14 @@ def test_uses_mapped_variant_date(self): """Test that contribution uses mapped variant date.""" test_date = datetime(2024, 3, 15, 12, 0, 0) mapped_variant = create_mock_mapped_variant(mapped_date=test_date) - contribution = mavedb_vrs_contribution(mapped_variant) + contribution = mavedb_vrs_contribution(annotation_context_for(mapped_variant)) assert contribution.date == test_date def test_has_contributor(self): """Test that contribution has a contributor.""" mapped_variant = create_mock_mapped_variant() - contribution = mavedb_vrs_contribution(mapped_variant) + contribution = mavedb_vrs_contribution(annotation_context_for(mapped_variant)) assert contribution.contributor is not None @@ -123,7 +126,7 @@ def test_has_contributor(self): def test_various_api_versions(self, api_version): """Test function works with various API versions.""" mapped_variant = create_mock_mapped_variant(mapping_api_version=api_version) - contribution = mavedb_vrs_contribution(mapped_variant) + contribution = mavedb_vrs_contribution(annotation_context_for(mapped_variant)) assert isinstance(contribution, Contribution) assert contribution.name == "MaveDB VRS Mapper" @@ -316,7 +319,12 @@ class TestContributionIntegration: def test_contributions_with_real_db_objects(self, session, setup_lib_db_with_mapped_variant): """Test contribution creation from persisted SQLAlchemy objects.""" mapped_variant = setup_lib_db_with_mapped_variant - vrs_contribution = mavedb_vrs_contribution(mapped_variant) + # The VRS contribution derives from the record's mapper provenance, not the subject; hand the + # context an explicit subject so it doesn't depend on this fixture's minimal post_mapped. + context = annotation_context_for( + mapped_variant, subject_variant=vrs_object_from_mapped_variant(TEST_VALID_POST_MAPPED_VRS_ALLELE) + ) + vrs_contribution = mavedb_vrs_contribution(context) creator = session.query(User).first() score_set = mapped_variant.variant.score_set @@ -350,7 +358,7 @@ def test_all_contributions_return_consistent_structure(self): api_contrib = mavedb_api_contribution() mapped_variant = create_mock_mapped_variant() - vrs_contrib = mavedb_vrs_contribution(mapped_variant) + vrs_contrib = mavedb_vrs_contribution(annotation_context_for(mapped_variant)) calibration = create_mock_score_calibration() cal_contrib = mavedb_score_calibration_contribution(calibration) @@ -376,7 +384,7 @@ def test_date_formatting_consistency(self): # VRS contribution mapped_variant = create_mock_mapped_variant(mapped_date=test_date) - vrs_contrib = mavedb_vrs_contribution(mapped_variant) + vrs_contrib = mavedb_vrs_contribution(annotation_context_for(mapped_variant)) assert vrs_contrib.date == test_date # Score calibration contribution diff --git a/tests/routers/test_score_set.py b/tests/routers/test_score_set.py index 942f032c7..11a07bf51 100644 --- a/tests/routers/test_score_set.py +++ b/tests/routers/test_score_set.py @@ -85,10 +85,10 @@ link_clinvar_control_to_mapped_variant, link_gnomad_variants_to_mapped_variants, publish_score_set, + seed_annotation_substrate, ) from tests.helpers.util.user import change_ownership from tests.helpers.util.variant import ( - clear_first_mapped_variant_post_mapped, create_mapped_variants_for_score_set, mock_worker_variant_insertion, ) @@ -4055,9 +4055,14 @@ def test_cannot_get_annotated_variants_for_nonexistent_score_set(client, setup_r @pytest.mark.parametrize("annotation_type", ["pathogenicity-statement", "functional-statement", "study-result"]) -def test_cannot_get_annotated_variants_for_score_set_with_no_mapped_variants( +def test_get_annotated_variants_for_score_set_with_no_mapped_variants_streams_empty( client, session, data_provider, data_files, setup_router_db, annotation_type ): + """A score set that exists but has no annotatable variants is an empty collection, not a 404. + + The route resolves (the URN names a real, readable score set), so it returns 200 with an empty NDJSON + body and ``X-Total-Count: 0``. 404 is reserved for an unresolvable URN or a permission failure. + """ experiment = create_experiment(client) score_set = create_seq_score_set_with_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" @@ -4082,13 +4087,47 @@ def test_cannot_get_annotated_variants_for_score_set_with_no_mapped_variants( assert "hgvs_splice" not in columns response = client.get(f"/api/v1/score-sets/{publish_score_set['urn']}/annotated-variants/{annotation_type}") - response_data = response.json() - assert response.status_code == 404 - assert ( - f"No mapped variants associated with score set URN {publish_score_set['urn']} were found" - in response_data["detail"] + assert response.status_code == 200 + assert response.headers["X-Total-Count"] == "0" + assert parse_ndjson_response(response) == [] + + +@pytest.mark.parametrize("annotation_type", ["pathogenicity-statement", "functional-statement", "study-result"]) +def test_annotated_variants_honor_as_of(client, session, data_provider, data_files, setup_router_db, annotation_type): + """The streaming annotation routes time-travel their annotatable set via ``as_of``. + + ``as_of`` is threaded into ``get_annotatable_variants`` (which allele links are live at that instant), + not only into the per-variant context. So a far-future instant sees the freshly-seeded substrate as + live (the full set streams, and the route echoes the instant on ``X-As-Of``); a far-past instant sees + no live mapping data (the annotatable set is empty). Either way the route returns 200 — ``as_of`` is a + filter, so an empty match is an empty collection, never a 404. This pins the wiring so the set + enumeration and the per-variant annotation cannot drift to different instants. + """ + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) + seed_annotation_substrate(session, score_set) + + url = f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/{annotation_type}" + + # Far future: the seeded mapping records are live, so the whole set streams and X-As-Of echoes back. + future = datetime(2999, 1, 1, tzinfo=timezone.utc) + future_response = client.get(url, params={"as_of": future.isoformat()}) + + assert future_response.status_code == 200 + assert datetime.fromisoformat(future_response.headers["X-As-Of"]) == future + assert len(parse_ndjson_response(future_response)) == score_set["numVariants"] + + # Far past: nothing was live yet, so the annotatable set is empty — an empty 200 stream, not a 404. + past = datetime(2000, 1, 1, tzinfo=timezone.utc) + past_response = client.get(url, params={"as_of": past.isoformat()}) + + assert past_response.status_code == 200 + assert datetime.fromisoformat(past_response.headers["X-As-Of"]) == past + assert past_response.headers["X-Total-Count"] == "0" + assert parse_ndjson_response(past_response) == [] # Tests that annotated variants of the correct type are returned when appropriate. The contents of these @@ -4119,6 +4158,7 @@ def test_get_annotated_pathogenicity_evidence_lines_for_score_set( create_publish_and_promote_score_calibration( client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) ) + seed_annotation_substrate(session, score_set) # The contents of the annotated variants objects should be tested in more detail elsewhere. response = client.get(f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/pathogenicity-statement") @@ -4150,6 +4190,10 @@ def test_nonetype_annotated_pathogenicity_evidence_lines_for_score_set_when_thre data_files / "scores.csv", ) + # Variants are mapped on the allele substrate, so they are annotatable; the null annotation comes from + # the missing calibration/thresholds, not from missing mapping. + seed_annotation_substrate(session, score_set) + response = client.get(f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/pathogenicity-statement") response_data = parse_ndjson_response(response) @@ -4173,6 +4217,9 @@ def test_nonetype_annotated_pathogenicity_evidence_lines_for_score_set_when_cali data_files / "scores.csv", ) + # Variants are mapped on the allele substrate (so annotatable); the null comes from the absent calibration. + seed_annotation_substrate(session, score_set) + response = client.get(f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/pathogenicity-statement") response_data = parse_ndjson_response(response) @@ -4209,21 +4256,21 @@ def test_get_annotated_pathogenicity_evidence_lines_for_score_set_when_some_vari client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) ) - first_var = clear_first_mapped_variant_post_mapped(session, score_set["urn"]) + first_var_urn = seed_annotation_substrate(session, score_set, skip_first=True)[0].urn response = client.get(f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/pathogenicity-statement") response_data = parse_ndjson_response(response) assert response.status_code == 200 - assert len(response_data) == score_set["numVariants"] + # The first variant is unmapped on the allele substrate, so it is not annotatable and is omitted from + # the stream entirely (rather than emitted with a null annotation). + assert len(response_data) == score_set["numVariants"] - 1 + assert first_var_urn not in {annotation_response.get("variant_urn") for annotation_response in response_data} for annotation_response in response_data: variant_urn = annotation_response.get("variant_urn") annotated_variant = annotation_response.get("annotation") - if variant_urn == first_var.urn: - assert annotated_variant is None - else: - assert f"Variant pathogenicity statement for {variant_urn}" in annotated_variant.get("description", "") + assert f"Variant pathogenicity statement for {variant_urn}" in annotated_variant.get("description", "") @pytest.mark.parametrize( @@ -4250,6 +4297,7 @@ def test_get_annotated_functional_impact_statement_for_score_set( create_publish_and_promote_score_calibration( client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) ) + seed_annotation_substrate(session, score_set) response = client.get(f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/functional-statement") response_data = parse_ndjson_response(response) @@ -4283,6 +4331,9 @@ def test_nonetype_annotated_functional_impact_statement_for_score_set_when_calib }, ) + # Mapped on the allele substrate (so annotatable); the null comes from the absent promoted calibration. + seed_annotation_substrate(session, score_set) + response = client.get(f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/functional-statement") response_data = parse_ndjson_response(response) @@ -4306,6 +4357,9 @@ def test_nonetype_annotated_functional_impact_statement_for_score_set_when_thres data_files / "scores.csv", ) + # Mapped on the allele substrate (so annotatable); the null comes from the absent calibration/ranges. + seed_annotation_substrate(session, score_set) + response = client.get(f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/functional-statement") response_data = parse_ndjson_response(response) @@ -4342,21 +4396,20 @@ def test_get_annotated_functional_impact_statement_for_score_set_when_some_varia client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) ) - first_var = clear_first_mapped_variant_post_mapped(session, score_set["urn"]) + first_var_urn = seed_annotation_substrate(session, score_set, skip_first=True)[0].urn response = client.get(f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/functional-statement") response_data = parse_ndjson_response(response) assert response.status_code == 200 - assert len(response_data) == score_set["numVariants"] + # The first variant is unmapped on the allele substrate, so it is not annotatable and is omitted from + # the stream entirely (rather than emitted with a null annotation). + assert len(response_data) == score_set["numVariants"] - 1 + assert first_var_urn not in {annotation_response.get("variant_urn") for annotation_response in response_data} for annotation_response in response_data: - variant_urn = annotation_response.get("variant_urn") annotated_variant = annotation_response.get("annotation") - if variant_urn == first_var.urn: - assert annotated_variant is None - else: - assert annotated_variant.get("type") == "Statement" + assert annotated_variant.get("type") == "Statement" @pytest.mark.parametrize( @@ -4375,6 +4428,7 @@ def test_get_annotated_functional_study_result_for_score_set( experiment["urn"], data_files / "scores.csv", ) + seed_annotation_substrate(session, score_set) response = client.get(f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/study-result") response_data = parse_ndjson_response(response) @@ -4407,6 +4461,7 @@ def test_annotated_functional_study_result_exists_for_score_set_when_thresholds_ "scoreRanges": camelize([TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED, TEST_PATHOGENICITY_SCORE_CALIBRATION]), }, ) + seed_annotation_substrate(session, score_set) response = client.get(f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/study-result") response_data = parse_ndjson_response(response) @@ -4439,6 +4494,7 @@ def test_annotated_functional_study_result_exists_for_score_set_when_ranges_not_ "scoreRanges": camelize([TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED, TEST_PATHOGENICITY_SCORE_CALIBRATION]), }, ) + seed_annotation_substrate(session, score_set) response = client.get(f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/study-result") response_data = parse_ndjson_response(response) @@ -4462,6 +4518,7 @@ def test_annotated_functional_study_result_exists_for_score_set_when_thresholds_ experiment["urn"], data_files / "scores.csv", ) + seed_annotation_substrate(session, score_set) response = client.get(f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/study-result") response_data = parse_ndjson_response(response) @@ -4495,21 +4552,20 @@ def test_annotated_functional_study_result_exists_for_score_set_when_some_varian }, ) - first_var = clear_first_mapped_variant_post_mapped(session, score_set["urn"]) + first_var_urn = seed_annotation_substrate(session, score_set, skip_first=True)[0].urn response = client.get(f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/study-result") response_data = parse_ndjson_response(response) assert response.status_code == 200 - assert len(response_data) == score_set["numVariants"] + # The first variant is unmapped on the allele substrate, so it is not annotatable and is omitted from + # the stream entirely (rather than emitted with a null annotation). + assert len(response_data) == score_set["numVariants"] - 1 + assert first_var_urn not in {annotation_response.get("variant_urn") for annotation_response in response_data} for annotation_response in response_data: - variant_urn = annotation_response.get("variant_urn") annotated_variant = annotation_response.get("annotation") - if variant_urn == first_var.urn: - assert annotated_variant is None - else: - assert annotated_variant.get("type") == "ExperimentalVariantFunctionalImpactStudyResult" + assert annotated_variant.get("type") == "ExperimentalVariantFunctionalImpactStudyResult" ######################################################################################################################## From 599c8e018a576356f25347684f47ae90d9e233d5 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Sat, 18 Jul 2026 10:42:03 -0700 Subject: [PATCH 091/252] feat(variants): relocate VA-Spec and VRS-lookup endpoints; drop /mapped-variants router Move the single-variant annotation surface onto the variants router, served from the MappingRecord/Allele substrate, and remove the legacy mapped_variant router entirely. - Add GET /variants/{urn}/va/{study-result,functional-statement, pathogenicity-statement}, each building a VA-Spec resource from the variant's live mapping context and threading as_of for temporal reconstruction. - Add GET /variants/vrs/{identifier}: resolve a GA4GH VRS id to the readable variants whose mapping links that allele, filtered by as_of. - Delete routers/mapped_variant.py and its registration in server_main. 404 semantics follow the collection-vs-resource split: the VRS lookup is collection-shaped, so no readable match returns 200 with an empty list (an absent id and a private-only match are indistinguishable, hiding existence); the single derived /va/* resources 404 when the variant is unknown or the statement does not exist at the requested instant. --- src/mavedb/routers/mapped_variant.py | 262 ---------- src/mavedb/routers/variants.py | 269 +++++++++- src/mavedb/server_main.py | 3 - src/mavedb/view_models/variant.py | 14 + tests/routers/test_mapped_variants.py | 620 ----------------------- tests/routers/test_variant.py | 20 +- tests/routers/test_variant_annotation.py | 352 +++++++++++++ 7 files changed, 639 insertions(+), 901 deletions(-) delete mode 100644 src/mavedb/routers/mapped_variant.py delete mode 100644 tests/routers/test_mapped_variants.py create mode 100644 tests/routers/test_variant_annotation.py diff --git a/src/mavedb/routers/mapped_variant.py b/src/mavedb/routers/mapped_variant.py deleted file mode 100644 index 7b97b304c..000000000 --- a/src/mavedb/routers/mapped_variant.py +++ /dev/null @@ -1,262 +0,0 @@ -import logging -from typing import Annotated, Any, Optional - -from fastapi import APIRouter, Depends, Path -from fastapi.exceptions import HTTPException -from ga4gh.core.identifiers import GA4GH_IR_REGEXP -from ga4gh.va_spec.acmg_2015 import VariantPathogenicityStatement -from ga4gh.va_spec.base.core import ExperimentalVariantFunctionalImpactStudyResult, Statement -from sqlalchemy import or_, select -from sqlalchemy.exc import MultipleResultsFound -from sqlalchemy.orm import Session - -from mavedb import deps -from mavedb.lib.annotation.annotate import ( - variant_functional_impact_statement, - variant_pathogenicity_statement, - variant_study_result, -) -from mavedb.lib.annotation.exceptions import MappingDataDoesntExistException -from mavedb.lib.authorization import get_current_user -from mavedb.lib.logging import LoggedRoute -from mavedb.lib.logging.context import ( - logging_context, - save_to_logging_context, -) -from mavedb.lib.permissions import Action, assert_permission, has_permission -from mavedb.lib.types.authentication import UserData -from mavedb.models.mapped_variant import MappedVariant -from mavedb.models.variant import Variant -from mavedb.routers.shared import ACCESS_CONTROL_ERROR_RESPONSES, PUBLIC_ERROR_RESPONSES, ROUTER_BASE_PREFIX -from mavedb.view_models import mapped_variant - -TAG_NAME = "Mapped Variants" - -logger = logging.getLogger(__name__) - - -async def fetch_mapped_variant_by_variant_urn(db: Session, user: Optional[UserData], urn: str) -> MappedVariant: - """ - We may combine this function back to show_mapped_variant if none of any new function call it. - Fetch one mapped variant by variant URN. - - :param db: An active database session. - :param urn: The variant URN. - :return: The mapped variant. - - :raises HTTPException: If the mapped variant is not found or if multiple variants are found. - """ - try: - item = ( - db.execute(select(MappedVariant).join(Variant).where(Variant.urn == urn, MappedVariant.current.is_(True))) - .scalars() - .one_or_none() - ) - except MultipleResultsFound: - logger.info( - msg="Could not fetch the requested mapped variant; Multiple such variants exist.", extra=logging_context() - ) - raise HTTPException(status_code=500, detail=f"Multiple variants with URN {urn} were found.") - - if not item: - logger.info( - msg="Could not fetch the requested mapped variant; No such mapped variants exist.", extra=logging_context() - ) - - raise HTTPException(status_code=404, detail=f"Mapped variant with URN {urn} not found") - - # Base mapped variant read permission on the score set in which it is contained. - assert_permission(user, item.variant.score_set, Action.READ) - return item - - -router = APIRouter( - prefix=f"{ROUTER_BASE_PREFIX}/mapped-variants", - tags=[TAG_NAME], - responses={**PUBLIC_ERROR_RESPONSES}, - route_class=LoggedRoute, -) - -metadata = { - "name": TAG_NAME, - "description": "Retrieve mapped variants and their associated variant annotations.", -} - - -@router.get( - "/{urn}", - status_code=200, - response_model=mapped_variant.MappedVariantWithMappingDetails, - responses={**ACCESS_CONTROL_ERROR_RESPONSES}, - summary="Fetch mapped variant by URN", -) -async def show_mapped_variant( - *, urn: str, db: Session = Depends(deps.get_db), user: Optional[UserData] = Depends(get_current_user) -) -> Any: - """ - Fetch a single mapped variant by URN. - """ - save_to_logging_context({"requested_resource": urn}) - - return await fetch_mapped_variant_by_variant_urn(db, user, urn) - - -@router.get( - "/{urn}/va/study-result", - status_code=200, - response_model=ExperimentalVariantFunctionalImpactStudyResult, - responses={**ACCESS_CONTROL_ERROR_RESPONSES}, - summary="Construct a VA-Spec StudyResult from a mapped variant", -) -async def show_mapped_variant_study_result( - *, urn: str, db: Session = Depends(deps.get_db), user: Optional[UserData] = Depends(get_current_user) -) -> ExperimentalVariantFunctionalImpactStudyResult: - """ - Construct a single VA-Spec StudyResult from a mapped variant by URN. - """ - save_to_logging_context({"requested_resource": urn}) - - mapped_variant = await fetch_mapped_variant_by_variant_urn(db, user, urn) - - try: - return variant_study_result(mapped_variant) - except MappingDataDoesntExistException as e: - logger.info( - msg=f"Could not construct a study result for mapped variant {urn}: {e}", - extra=logging_context(), - ) - raise HTTPException(status_code=404, detail=f"No study result exists for mapped variant {urn}: {e}") - - -# TODO#416: For now, this route supports only one statement per mapped variant. Eventually, we should support the possibility of multiple statements. -@router.get( - "/{urn}/va/functional-statement", - status_code=200, - response_model=Statement, - responses={**ACCESS_CONTROL_ERROR_RESPONSES}, - summary="Construct a VA-Spec Statement from a mapped variant", -) -async def show_mapped_variant_functional_impact_statement( - *, urn: str, db: Session = Depends(deps.get_db), user: Optional[UserData] = Depends(get_current_user) -) -> Statement: - """ - Construct a single VA-Spec Statement from a mapped variant by URN. - """ - save_to_logging_context({"requested_resource": urn}) - - mapped_variant = await fetch_mapped_variant_by_variant_urn(db, user, urn) - - try: - functional_impact = variant_functional_impact_statement(mapped_variant) - except MappingDataDoesntExistException as e: - logger.info( - msg="Could not construct a functional impact statement for this mapped variant; No mapping data exists for this score set.", - extra=logging_context(), - ) - raise HTTPException( - status_code=404, detail=f"No functional impact statement exists for mapped variant {urn}: {e}" - ) - - if not functional_impact: - logger.info( - msg="Could not construct a functional impact statement for this mapped variant. Variant does not have sufficient evidence to evaluate its functional impact.", - extra=logging_context(), - ) - raise HTTPException( - status_code=404, - detail=f"No functional impact statement exists for mapped variant {urn}. Variant does not have sufficient evidence to evaluate its functional impact.", - ) - - return functional_impact - - -# TODO#416: For now, this route supports only one evidence line per mapped variant. Eventually, we should support the possibility of multiple evidence lines. -@router.get( - "/{urn}/va/pathogenicity-statement", - status_code=200, - response_model=VariantPathogenicityStatement, - responses={**ACCESS_CONTROL_ERROR_RESPONSES}, - summary="Construct a VA-Spec EvidenceLine from a mapped variant", -) -async def show_mapped_variant_acmg_evidence_line( - *, urn: str, db: Session = Depends(deps.get_db), user: Optional[UserData] = Depends(get_current_user) -) -> VariantPathogenicityStatement: - """ - Construct a list of VA-Spec EvidenceLine(s) from a mapped variant by URN. - """ - save_to_logging_context({"requested_resource": urn}) - - mapped_variant = await fetch_mapped_variant_by_variant_urn(db, user, urn) - - try: - pathogenicity_statement = variant_pathogenicity_statement(mapped_variant) - except MappingDataDoesntExistException as e: - logger.info( - msg="Could not construct a pathogenicity statement for this mapped variant; No mapping data exists for this score set.", - extra=logging_context(), - ) - raise HTTPException(status_code=404, detail=f"No pathogenicity statement exists for mapped variant {urn}: {e}") - - if not pathogenicity_statement: - logger.info( - msg="Could not construct a pathogenicity statement for this mapped variant; Variant does not have sufficient evidence to evaluate its pathogenicity.", - extra=logging_context(), - ) - raise HTTPException( - status_code=404, - detail=f"No pathogenicity statement exists for mapped variant {urn}; Variant does not have sufficient evidence to evaluate its pathogenicity.", - ) - - return pathogenicity_statement - - -@router.get( - "/vrs/{identifier}", - status_code=200, - response_model=list[mapped_variant.MappedVariant], - responses={**ACCESS_CONTROL_ERROR_RESPONSES}, - summary="Fetch mapped variants by VRS identifier", -) -async def show_mapped_variants_by_identifier( - *, - identifier: Annotated[ - str, - Path( - description="String, a valid GA4GH digest based identifier.", - json_schema_extra={"example": "ga4gh:SQ.0123abcd"}, - regex=GA4GH_IR_REGEXP, - ), - ], - only_current: bool = True, - db: Session = Depends(deps.get_db), - user: Optional[UserData] = Depends(get_current_user), -) -> list[MappedVariant]: - """ - Fetch a single mapped variant by GA4GH identifier. - """ - query = select(MappedVariant).where( - or_(MappedVariant.pre_mapped["id"].astext == identifier, MappedVariant.post_mapped["id"].astext == identifier) - ) - - if only_current: - query = query.where(MappedVariant.current.is_(True)) - - items = db.scalars(query).all() - - permitted_items = [item for item in items if has_permission(user, item.variant.score_set, Action.READ).permitted] - if not permitted_items: - raise HTTPException(status_code=404, detail=f"No mapped variants with identifier {identifier} were found") - - return permitted_items - - -# for testing only -# @router.post("/map/{urn}", status_code=200, responses={404: {}, 500: {}}) -# async def map_score_set(*, urn: str, worker: ArqRedis = Depends(deps.get_worker)) -> Any: -# await worker.lpush(MAPPING_QUEUE_NAME, urn) # type: ignore -# await worker.enqueue_job( -# "variant_mapper_manager", -# None, -# None, -# None -# ) diff --git a/src/mavedb/routers/variants.py b/src/mavedb/routers/variants.py index d7cfdfc15..78144594c 100644 --- a/src/mavedb/routers/variants.py +++ b/src/mavedb/routers/variants.py @@ -1,14 +1,25 @@ import logging from datetime import datetime -from typing import Optional +from typing import Annotated, Optional -from fastapi import APIRouter, Depends, Query, Response +from fastapi import APIRouter, Depends, Path, Query, Response from fastapi.exceptions import HTTPException +from ga4gh.core.identifiers import GA4GH_IR_REGEXP +from ga4gh.va_spec.acmg_2015 import VariantPathogenicityStatement +from ga4gh.va_spec.base.core import ExperimentalVariantFunctionalImpactStudyResult, Statement from sqlalchemy import select from sqlalchemy.exc import MultipleResultsFound from sqlalchemy.orm import Session from mavedb import deps +from mavedb.lib.alleles import find_variants_by_vrs_identifier +from mavedb.lib.annotation.annotate import ( + variant_functional_impact_statement, + variant_pathogenicity_statement, + variant_study_result, +) +from mavedb.lib.annotation.context import variant_annotation_context +from mavedb.lib.annotation.exceptions import MappingDataDoesntExistException from mavedb.lib.authentication import get_current_user from mavedb.lib.logging import LoggedRoute from mavedb.lib.logging.context import logging_context, save_to_logging_context @@ -21,6 +32,7 @@ PUBLIC_ERROR_RESPONSES, ROUTER_BASE_PREFIX, ) +from mavedb.view_models.variant import VariantVrsMatch from mavedb.view_models.variant_detail import VariantDetail TAG_NAME = "Variants" @@ -40,6 +52,85 @@ } +def _fetch_readable_variant(db: Session, user_data: Optional[UserData], urn: str) -> Variant: + """Fetch a single variant by URN and assert the caller may read its score set. + + Raises 404 when no such variant exists, 500 when the URN is not unique (an invariant break), and the + standard 403 (via ``assert_permission``) when the score set is not readable. + """ + try: + variant = db.scalars(select(Variant).where(Variant.urn == urn)).one_or_none() + except MultipleResultsFound: + logger.info(msg="Could not fetch the requested variant; Multiple such variants exist.", extra=logging_context()) + raise HTTPException(status_code=500, detail=f"multiple variants with URN '{urn}' were found") + + if not variant: + logger.info(msg="Could not fetch the requested variant; No such variant exists.", extra=logging_context()) + raise HTTPException(status_code=404, detail=f"variant with URN '{urn}' not found") + + assert_permission(user_data, variant.score_set, Action.READ) + return variant + + +@router.get( + "/variants/vrs/{identifier}", + status_code=200, + response_model=list[VariantVrsMatch], + responses={**ACCESS_CONTROL_ERROR_RESPONSES}, + summary="Look up variants by VRS identifier", +) +def lookup_variants_by_vrs_identifier( + *, + response: Response, + identifier: Annotated[ + str, + Path( + description="A valid GA4GH digest-based identifier for the mapped allele.", + json_schema_extra={"example": "ga4gh:VA.0123abcd"}, + regex=GA4GH_IR_REGEXP, + ), + ], + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the molecular layer (Cat-VRS membership, VEP/gnomAD/ClinVar annotations) as it " + "stood at this instant, over the variant's fixed score. ISO 8601, ideally timezone-aware. " + "Content valid-time only — it never re-selects a score-set version, and scores/classifications " + "are as-of-invariant. Defaults to current." + ), + ), + db: Session = Depends(deps.get_db), + user_data: Optional[UserData] = Depends(get_current_user), +) -> list[VariantVrsMatch]: + """Resolve a GA4GH VRS identifier to the readable variants whose mapping links that allele. + + A deduplicated allele may be shared across score sets, so one identifier can resolve to several + variants. This is a lookup returning a collection: results are filtered to the score sets the caller + may read, and an empty list is returned when nothing readable matches. An absent identifier and a + match visible only in a private score set are deliberately indistinguishable (both yield ``[]``), so + the response never reveals a private allele's existence. + """ + save_to_logging_context({"requested_resource": identifier, "as_of": as_of}) + response.headers["X-As-Of"] = as_of.isoformat() if as_of is not None else "current" + + matches = find_variants_by_vrs_identifier(db, identifier, as_of=as_of) + permitted = [ + (variant, allele) + for variant, allele in matches + if has_permission(user_data, variant.score_set, Action.READ).permitted + ] + + return [ + VariantVrsMatch( + variant_urn=variant.urn or "", + clingen_allele_id=allele.clingen_allele_id, + vrs_id=(allele.post_mapped or {}).get("id"), + level=allele.level, + ) + for variant, allele in permitted + ] + + @router.get( "/variants/{urn}", status_code=200, @@ -72,17 +163,7 @@ def get_variant( """ save_to_logging_context({"requested_resource": urn, "as_of": as_of}) response.headers["X-As-Of"] = as_of.isoformat() if as_of is not None else "current" - try: - variant = db.scalars(select(Variant).where(Variant.urn == urn)).one_or_none() - except MultipleResultsFound: - logger.info(msg="Could not fetch the requested variant; Multiple such variants exist.", extra=logging_context()) - raise HTTPException(status_code=500, detail=f"multiple variants with URN '{urn}' were found") - - if not variant: - logger.info(msg="Could not fetch the requested variant; No such variant exists.", extra=logging_context()) - raise HTTPException(status_code=404, detail=f"variant with URN '{urn}' not found") - - assert_permission(user_data, variant.score_set, Action.READ) + variant = _fetch_readable_variant(db, user_data, urn) # Version standing: resolve the superseding version's visibility exactly as fetch_score_set_by_urn # does — a newer version the user cannot read is not leaked (the variant then reads as current). @@ -103,3 +184,165 @@ def get_variant( visible_calibration_ids=visible_calibration_ids, as_of=as_of, ) + + +@router.get( + "/variants/{urn}/va/study-result", + status_code=200, + response_model=ExperimentalVariantFunctionalImpactStudyResult, + responses={**ACCESS_CONTROL_ERROR_RESPONSES}, + summary="Construct a VA-Spec StudyResult for a variant", +) +def get_variant_study_result( + *, + response: Response, + urn: str, + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the molecular layer (Cat-VRS membership, VEP/gnomAD/ClinVar annotations) as it " + "stood at this instant, over the variant's fixed score. ISO 8601, ideally timezone-aware. " + "Content valid-time only — it never re-selects a score-set version, and scores/classifications " + "are as-of-invariant. Defaults to current." + ), + ), + db: Session = Depends(deps.get_db), + user_data: Optional[UserData] = Depends(get_current_user), +) -> ExperimentalVariantFunctionalImpactStudyResult: + """Construct a single VA-Spec StudyResult for a variant by URN, from its mapping substrate.""" + save_to_logging_context({"requested_resource": urn, "as_of": as_of}) + response.headers["X-As-Of"] = as_of.isoformat() if as_of is not None else "current" + + variant = _fetch_readable_variant(db, user_data, urn) + + context = variant_annotation_context(db, variant, as_of=as_of) + if context is None: + raise HTTPException( + status_code=404, detail=f"No study result exists for variant {urn}: no mapping data exists." + ) + + try: + return variant_study_result(context) + except MappingDataDoesntExistException as e: + logger.info(msg=f"Could not construct a study result for variant {urn}: {e}", extra=logging_context()) + raise HTTPException(status_code=404, detail=f"No study result exists for variant {urn}: {e}") + + +@router.get( + "/variants/{urn}/va/functional-statement", + status_code=200, + response_model=Statement, + responses={**ACCESS_CONTROL_ERROR_RESPONSES}, + summary="Construct a VA-Spec functional-impact Statement for a variant", +) +def get_variant_functional_impact_statement( + *, + response: Response, + urn: str, + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the molecular layer (Cat-VRS membership, VEP/gnomAD/ClinVar annotations) as it " + "stood at this instant, over the variant's fixed score. ISO 8601, ideally timezone-aware. " + "Content valid-time only — it never re-selects a score-set version, and scores/classifications " + "are as-of-invariant. Defaults to current." + ), + ), + db: Session = Depends(deps.get_db), + user_data: Optional[UserData] = Depends(get_current_user), +) -> Statement: + """Construct a single VA-Spec functional-impact Statement for a variant by URN.""" + save_to_logging_context({"requested_resource": urn, "as_of": as_of}) + response.headers["X-As-Of"] = as_of.isoformat() if as_of is not None else "current" + + variant = _fetch_readable_variant(db, user_data, urn) + + context = variant_annotation_context(db, variant, as_of=as_of) + if context is None: + raise HTTPException( + status_code=404, + detail=f"No functional impact statement exists for variant {urn}: no mapping data exists.", + ) + + try: + functional_impact = variant_functional_impact_statement(context) + except MappingDataDoesntExistException as e: + logger.info( + msg=f"Could not construct a functional impact statement for variant {urn}: {e}", extra=logging_context() + ) + raise HTTPException(status_code=404, detail=f"No functional impact statement exists for variant {urn}: {e}") + + if not functional_impact: + logger.info( + msg=f"Variant {urn} does not have sufficient evidence to evaluate its functional impact.", + extra=logging_context(), + ) + raise HTTPException( + status_code=404, + detail=( + f"No functional impact statement exists for variant {urn}. Variant does not have sufficient " + "evidence to evaluate its functional impact." + ), + ) + + return functional_impact + + +@router.get( + "/variants/{urn}/va/pathogenicity-statement", + status_code=200, + response_model=VariantPathogenicityStatement, + responses={**ACCESS_CONTROL_ERROR_RESPONSES}, + summary="Construct a VA-Spec pathogenicity Statement for a variant", +) +def get_variant_pathogenicity_statement( + *, + response: Response, + urn: str, + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the molecular layer (Cat-VRS membership, VEP/gnomAD/ClinVar annotations) as it " + "stood at this instant, over the variant's fixed score. ISO 8601, ideally timezone-aware. " + "Content valid-time only — it never re-selects a score-set version, and scores/classifications " + "are as-of-invariant. Defaults to current." + ), + ), + db: Session = Depends(deps.get_db), + user_data: Optional[UserData] = Depends(get_current_user), +) -> VariantPathogenicityStatement: + """Construct a single VA-Spec pathogenicity Statement for a variant by URN.""" + save_to_logging_context({"requested_resource": urn, "as_of": as_of}) + response.headers["X-As-Of"] = as_of.isoformat() if as_of is not None else "current" + + variant = _fetch_readable_variant(db, user_data, urn) + + context = variant_annotation_context(db, variant, as_of=as_of) + if context is None: + raise HTTPException( + status_code=404, + detail=f"No pathogenicity statement exists for variant {urn}: no mapping data exists.", + ) + + try: + pathogenicity_statement = variant_pathogenicity_statement(context) + except MappingDataDoesntExistException as e: + logger.info( + msg=f"Could not construct a pathogenicity statement for variant {urn}: {e}", extra=logging_context() + ) + raise HTTPException(status_code=404, detail=f"No pathogenicity statement exists for variant {urn}: {e}") + + if not pathogenicity_statement: + logger.info( + msg=f"Variant {urn} does not have sufficient evidence to evaluate its pathogenicity.", + extra=logging_context(), + ) + raise HTTPException( + status_code=404, + detail=( + f"No pathogenicity statement exists for variant {urn}; Variant does not have sufficient " + "evidence to evaluate its pathogenicity." + ), + ) + + return pathogenicity_statement diff --git a/src/mavedb/server_main.py b/src/mavedb/server_main.py index e79819999..12fa5bf35 100644 --- a/src/mavedb/server_main.py +++ b/src/mavedb/server_main.py @@ -51,7 +51,6 @@ hgvs, job_runs, licenses, - mapped_variant, orcid, permissions, pipelines, @@ -105,7 +104,6 @@ app.include_router(job_runs.router) app.include_router(licenses.router) # app.include_router(log.router) -app.include_router(mapped_variant.router) app.include_router(orcid.router) app.include_router(permissions.router) app.include_router(pipelines.router) @@ -253,7 +251,6 @@ def customize_openapi_schema(): hgvs.metadata, licenses.metadata, # log.metadata, - mapped_variant.metadata, orcid.metadata, permissions.metadata, publication_identifiers.metadata, diff --git a/src/mavedb/view_models/variant.py b/src/mavedb/view_models/variant.py index c32abcab6..48d4edb85 100644 --- a/src/mavedb/view_models/variant.py +++ b/src/mavedb/view_models/variant.py @@ -9,6 +9,20 @@ from mavedb.view_models.mapped_variant import SavedMappedVariant +class VariantVrsMatch(BaseModel): + """One variant whose mapping links an allele bearing the queried VRS identifier. + + The ``GET /variants/vrs/{identifier}`` lookup result. The ClinGen id + level ride from the matched + (deduplicated) allele; the URN from the variant it is linked to. Replaces the legacy MappedVariant VRS + lookup on the new Allele substrate (#743, item 3.5). + """ + + variant_urn: str + clingen_allele_id: Optional[str] = None + vrs_id: Optional[str] = None + level: Optional[str] = None + + class VariantEffectMeasurementBase(BaseModel): """Properties shared by most variant effect measurement view models""" diff --git a/tests/routers/test_mapped_variants.py b/tests/routers/test_mapped_variants.py deleted file mode 100644 index 51f45452f..000000000 --- a/tests/routers/test_mapped_variants.py +++ /dev/null @@ -1,620 +0,0 @@ -# ruff: noqa: E402 - -import json - -import pytest - -from tests.helpers.util.user import change_ownership - -arq = pytest.importorskip("arq") -cdot = pytest.importorskip("cdot") -fastapi = pytest.importorskip("fastapi") - -from urllib.parse import quote_plus - -from ga4gh.va_spec.acmg_2015 import VariantPathogenicityStatement -from ga4gh.va_spec.base.core import ExperimentalVariantFunctionalImpactStudyResult, Statement -from sqlalchemy import select -from sqlalchemy.orm.session import make_transient - -from mavedb.models.mapped_variant import MappedVariant -from mavedb.models.score_set import ScoreSet as ScoreSetDbModel -from mavedb.models.variant import Variant -from mavedb.view_models.mapped_variant import SavedMappedVariant -from tests.helpers.constants import ( - TEST_BIORXIV_IDENTIFIER, - TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED, - TEST_PUBMED_IDENTIFIER, -) -from tests.helpers.util.common import deepcamelize -from tests.helpers.util.experiment import create_experiment -from tests.helpers.util.score_calibration import create_publish_and_promote_score_calibration -from tests.helpers.util.score_set import ( - create_seq_score_set_with_mapped_variants, - create_seq_score_set_with_variants, -) - - -def test_show_mapped_variant(client, session, data_provider, data_files, setup_router_db): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}") - response_data = response.json() - - assert response.status_code == 200 - assert response_data["id"] == 1 - - SavedMappedVariant.model_validate_json(json.dumps(response_data)) - - -def test_cannot_show_mapped_variant_when_multiple_exist(client, session, data_provider, data_files, setup_router_db): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - item = session.scalar(select(MappedVariant).join(Variant).where(Variant.urn == f'{score_set["urn"]}#1')) - assert item is not None - - session.expunge(item) - make_transient(item) - item.id = None - session.add(item) - session.commit() - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}") - response_data = response.json() - - assert response.status_code == 500 - assert response_data["detail"] == f"Multiple variants with URN {score_set['urn']}#1 were found." - - -def test_cannot_show_mapped_variant_when_none_exists(client, session, data_provider, data_files, setup_router_db): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}") - response_data = response.json() - - assert response.status_code == 404 - assert response_data["detail"] == f"Mapped variant with URN {score_set['urn']}#1 not found" - - -def test_show_mapped_variant_study_result(client, session, data_provider, data_files, setup_router_db): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}/va/study-result") - response_data = response.json() - - assert response.status_code == 200 - assert response_data["description"] == f"Variant effect study result for {score_set['urn']}#1." - - ExperimentalVariantFunctionalImpactStudyResult.model_validate_json(json.dumps(response_data)) - - -def test_cannot_show_mapped_variant_study_result_when_multiple_exist( - client, session, data_provider, data_files, setup_router_db -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - item = session.scalar(select(MappedVariant).join(Variant).where(Variant.urn == f'{score_set["urn"]}#1')) - assert item is not None - - session.expunge(item) - make_transient(item) - item.id = None - session.add(item) - session.commit() - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}/va/study-result") - response_data = response.json() - - assert response.status_code == 500 - assert response_data["detail"] == f"Multiple variants with URN {score_set['urn']}#1 were found." - - -def test_cannot_show_mapped_variant_study_result_when_none_exists( - client, session, data_provider, data_files, setup_router_db -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}/va/study-result") - response_data = response.json() - - assert response.status_code == 404 - assert response_data["detail"] == f"Mapped variant with URN {score_set['urn']}#1 not found" - - -def test_cannot_show_mapped_variant_study_result_when_no_mapping_data_exists( - client, session, data_provider, data_files, setup_router_db -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - item = session.scalar(select(MappedVariant).join(Variant).where(Variant.urn == f'{score_set["urn"]}#1')) - assert item is not None - - item.post_mapped = None - session.add(item) - session.commit() - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}/va/study-result") - response_data = response.json() - - assert response.status_code == 404 - assert ( - f"No study result exists for mapped variant {score_set['urn']}#1: Variant {score_set['urn']}#1 does not have a post mapped variant." - in response_data["detail"] - ) - - -@pytest.mark.parametrize( - "mock_publication_fetch", - [ - [ - {"dbName": "PubMed", "identifier": f"{TEST_PUBMED_IDENTIFIER}"}, - {"dbName": "bioRxiv", "identifier": TEST_BIORXIV_IDENTIFIER}, - ] - ], - indirect=["mock_publication_fetch"], -) -def test_show_mapped_variant_functional_impact_statement( - client, session, data_provider, data_files, setup_router_db, mock_publication_fetch -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - create_publish_and_promote_score_calibration( - client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) - ) - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}/va/functional-statement") - response_data = response.json() - - assert response.status_code == 200 - assert response_data["description"] == f"Variant functional impact statement for {score_set['urn']}#1." - - Statement.model_validate_json(json.dumps(response_data)) - - -def test_cannot_show_mapped_variant_functional_impact_statement_when_multiple_exist( - client, session, data_provider, data_files, setup_router_db -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - item = session.scalar(select(MappedVariant).join(Variant).where(Variant.urn == f'{score_set["urn"]}#1')) - assert item is not None - - session.expunge(item) - make_transient(item) - item.id = None - session.add(item) - session.commit() - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}/va/functional-statement") - response_data = response.json() - - assert response.status_code == 500 - assert response_data["detail"] == f"Multiple variants with URN {score_set['urn']}#1 were found." - - -def test_cannot_show_mapped_variant_functional_impact_statement_when_none_exists( - client, session, data_provider, data_files, setup_router_db -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}/va/functional-statement") - response_data = response.json() - - assert response.status_code == 404 - assert response_data["detail"] == f"Mapped variant with URN {score_set['urn']}#1 not found" - - -@pytest.mark.parametrize( - "mock_publication_fetch", - [ - [ - {"dbName": "PubMed", "identifier": f"{TEST_PUBMED_IDENTIFIER}"}, - {"dbName": "bioRxiv", "identifier": TEST_BIORXIV_IDENTIFIER}, - ] - ], - indirect=["mock_publication_fetch"], -) -def test_cannot_show_mapped_variant_functional_impact_statement_when_no_mapping_data_exists( - client, session, data_provider, data_files, setup_router_db, mock_publication_fetch -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - create_publish_and_promote_score_calibration( - client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) - ) - - item = session.scalar(select(MappedVariant).join(Variant).where(Variant.urn == f'{score_set["urn"]}#1')) - assert item is not None - - item.post_mapped = None - session.add(item) - session.commit() - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}/va/functional-statement") - response_data = response.json() - - assert response.status_code == 404 - assert ( - f"No functional impact statement exists for mapped variant {score_set['urn']}#1: Variant {score_set['urn']}#1 does not have a post mapped variant." - in response_data["detail"] - ) - - -def test_cannot_show_mapped_variant_functional_impact_statement_when_insufficient_functional_evidence( - client, session, data_provider, data_files, setup_router_db -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - # insufficient evidence = no (primary) calibrations - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}/va/functional-statement") - response_data = response.json() - - assert response.status_code == 404 - assert ( - f"No functional impact statement exists for mapped variant {score_set['urn']}#1. Variant does not have sufficient evidence to evaluate its functional impact" - in response_data["detail"] - ) - - -@pytest.mark.parametrize( - "mock_publication_fetch", - [ - [ - {"dbName": "PubMed", "identifier": f"{TEST_PUBMED_IDENTIFIER}"}, - {"dbName": "bioRxiv", "identifier": TEST_BIORXIV_IDENTIFIER}, - ] - ], - indirect=["mock_publication_fetch"], -) -def test_show_mapped_variant_clinical_evidence_line( - client, session, data_provider, data_files, setup_router_db, mock_publication_fetch -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - create_publish_and_promote_score_calibration( - client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) - ) - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#2')}/va/pathogenicity-statement") - response_data = response.json() - - assert response.status_code == 200 - assert f"Variant pathogenicity statement for {score_set['urn']}#2" in response_data["description"] - - VariantPathogenicityStatement.model_validate_json(json.dumps(response_data)) - - -def test_cannot_show_mapped_variant_clinical_evidence_line_when_multiple_exist( - client, session, data_provider, data_files, setup_router_db -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - item = session.scalar(select(MappedVariant).join(Variant).where(Variant.urn == f'{score_set["urn"]}#1')) - assert item is not None - - session.expunge(item) - make_transient(item) - item.id = None - session.add(item) - session.commit() - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}/va/pathogenicity-statement") - response_data = response.json() - - assert response.status_code == 500 - assert response_data["detail"] == f"Multiple variants with URN {score_set['urn']}#1 were found." - - -def test_cannot_show_mapped_variant_clinical_evidence_line_when_none_exists( - client, session, data_provider, data_files, setup_router_db -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}/va/pathogenicity-statement") - response_data = response.json() - - assert response.status_code == 404 - assert response_data["detail"] == f"Mapped variant with URN {score_set['urn']}#1 not found" - - -@pytest.mark.parametrize( - "mock_publication_fetch", - [ - [ - {"dbName": "PubMed", "identifier": f"{TEST_PUBMED_IDENTIFIER}"}, - {"dbName": "bioRxiv", "identifier": TEST_BIORXIV_IDENTIFIER}, - ] - ], - indirect=["mock_publication_fetch"], -) -def test_cannot_show_mapped_variant_clinical_evidence_line_when_no_mapping_data_exists( - client, session, data_provider, data_files, setup_router_db, mock_publication_fetch -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - create_publish_and_promote_score_calibration( - client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) - ) - - item = session.scalar(select(MappedVariant).join(Variant).where(Variant.urn == f'{score_set["urn"]}#1')) - assert item is not None - - item.post_mapped = None - session.add(item) - session.commit() - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}/va/pathogenicity-statement") - response_data = response.json() - - assert response.status_code == 404 - assert ( - f"No pathogenicity statement exists for mapped variant {score_set['urn']}#1: Variant {score_set['urn']}#1 does not have a post mapped variant." - in response_data["detail"] - ) - - -def test_cannot_show_mapped_variant_clinical_evidence_line_when_insufficient_pathogenicity_evidence( - client, session, data_provider, data_files, setup_router_db -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}/va/pathogenicity-statement") - response_data = response.json() - - assert response.status_code == 404 - assert ( - f"No pathogenicity statement exists for mapped variant {score_set['urn']}#1; Variant does not have sufficient evidence to evaluate its pathogenicity" - in response_data["detail"] - ) - - -def test_show_mapped_variants_by_ga4gh_identifier(client, session, data_provider, data_files, setup_router_db): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - mapped_variant = session.scalar(select(MappedVariant).join(Variant).where(Variant.urn == f'{score_set["urn"]}#1')) - assert mapped_variant is not None - - response = client.get(f"/api/v1/mapped-variants/vrs/{quote_plus(mapped_variant.pre_mapped['id'])}") - response_data = response.json() - - assert response.status_code == 200 - assert len(response_data) == score_set["numVariants"] - - for response_data in response_data: - assert response_data["preMapped"]["id"] == mapped_variant.pre_mapped["id"] - SavedMappedVariant.model_validate_json(json.dumps(response_data)) - - -def test_cannot_show_mapped_variants_by_ga4gh_identifier_when_none_exist( - client, session, data_provider, data_files, setup_router_db -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - mapped_variant = session.scalar(select(MappedVariant).join(Variant).where(Variant.urn == f'{score_set["urn"]}#1')) - assert mapped_variant is not None - - fake_mapped_variant_id = mapped_variant.pre_mapped["id"] - fake_mapped_variant_id = fake_mapped_variant_id[:-3] + "aaa" # Modify the ID to ensure it doesn't exist - - response = client.get(f"/api/v1/mapped-variants/vrs/{quote_plus(fake_mapped_variant_id)}") - assert response.status_code == 404 - - -def test_show_mapped_variants_by_ga4gh_identifier_with_non_current_variants( - client, session, data_provider, data_files, setup_router_db -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - mapped_variant = session.scalar(select(MappedVariant).join(Variant).where(Variant.urn == f'{score_set["urn"]}#1')) - assert mapped_variant is not None - - # Set the mapped variant to non-current - mapped_variant.current = False - session.add(mapped_variant) - session.commit() - - response = client.get( - f"/api/v1/mapped-variants/vrs/{quote_plus(mapped_variant.pre_mapped['id'])}?only_current=false" - ) - response_data = response.json() - - assert response.status_code == 200 - assert len(response_data) == score_set["numVariants"] - - for response_data in response_data: - assert response_data["preMapped"]["id"] == mapped_variant.pre_mapped["id"] - SavedMappedVariant.model_validate_json(json.dumps(response_data)) - - -def test_show_mapped_variants_by_ga4gh_identifier_with_only_current_variants( - client, session, data_provider, data_files, setup_router_db -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - mapped_variant = session.scalar(select(MappedVariant).join(Variant).where(Variant.urn == f'{score_set["urn"]}#1')) - mapped_variant2 = session.scalar(select(MappedVariant).join(Variant).where(Variant.urn == f'{score_set["urn"]}#2')) - assert mapped_variant is not None - assert mapped_variant2 is not None - - # Set the mapped variant to non-current - mapped_variant2.current = False - mapped_variant2.pre_mapped = mapped_variant.pre_mapped # Ensure both pre mapped blobs match besides current status - session.add(mapped_variant) - session.commit() - - response = client.get( - f"/api/v1/mapped-variants/vrs/{quote_plus(mapped_variant.pre_mapped['id'])}?only_current=true" - ) - response_data = response.json() - - assert response.status_code == 200 - assert len(response_data) == score_set["numVariants"] - 1 - - for response_data in response_data: - assert response_data["preMapped"]["id"] == mapped_variant.pre_mapped["id"] - SavedMappedVariant.model_validate_json(json.dumps(response_data)) - - -def test_cannot_show_mapped_variants_by_ga4gh_identifier_without_score_set_permissions( - client, session, data_provider, data_files, setup_router_db -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - change_ownership(session, score_set["urn"], ScoreSetDbModel) - - mapped_variant = session.scalar(select(MappedVariant).join(Variant).where(Variant.urn == f'{score_set["urn"]}#1')) - assert mapped_variant is not None - - response = client.get(f"/api/v1/mapped-variants/vrs/{quote_plus(mapped_variant.pre_mapped['id'])}") - assert response.status_code == 404 diff --git a/tests/routers/test_variant.py b/tests/routers/test_variant.py index ea1c01086..dae7fd50e 100644 --- a/tests/routers/test_variant.py +++ b/tests/routers/test_variant.py @@ -54,7 +54,8 @@ def _post_mapped() -> dict: def _seed_mapping(session, variant_urn): """Give a variant a live coding-measured mapping record whose authoritative allele carries a digest + ClinGen id + a live VEP consequence, its genomic projection sibling (shared - projection_group), and a protein apex — enough to build Cat-VRS and exercise the projection axes.""" + projection_group), a protein apex, and a synonymous cousin (a nt encoder of the same consequence in + a different projection group) — enough to build Cat-VRS and exercise the projection + convergent axes.""" variant = session.scalar(select(VariantDbModel).where(VariantDbModel.urn == variant_urn)) record = MappingRecord( variant_id=variant.id, @@ -78,13 +79,16 @@ def _seed_mapping(session, variant_urn): protein = Allele( vrs_digest="prot-digest", level="protein", post_mapped=_post_mapped(), hgvs_p="NP_000537.3:p.Ala406Thr" ) - session.add_all([measured, genomic, protein]) + cousin = Allele( + vrs_digest="cousin-digest", level="cdna", post_mapped=_post_mapped(), hgvs_c="NM_000546.6:c.1218C>T" + ) + session.add_all([measured, genomic, protein, cousin]) session.commit() session.add_all( [ # The measured cdna link and its genomic projection share a projection_group; the protein - # apex is in no pair (group None). + # apex is in no pair (group None); the cousin sits in its own projection group. MappingRecordAllele( mapping_record_id=record.id, allele_id=measured.id, is_authoritative=True, projection_group=0 ), @@ -92,6 +96,9 @@ def _seed_mapping(session, variant_urn): mapping_record_id=record.id, allele_id=genomic.id, is_authoritative=False, projection_group=0 ), MappingRecordAllele(mapping_record_id=record.id, allele_id=protein.id, is_authoritative=False), + MappingRecordAllele( + mapping_record_id=record.id, allele_id=cousin.id, is_authoritative=False, projection_group=1 + ), VepAlleleConsequence( allele_id=measured.id, functional_consequence="missense_variant", @@ -125,6 +132,9 @@ def test_get_variant_detail_envelope(client, session, data_provider, data_files, assert body["mode"] == "projection" # Spec-pure Cat-VRS carries its own field names (no camelization of the nested GA4GH object). assert body["molecularRepresentation"]["type"] == "CategoricalVariant" + # State A: the full-closure Cat-VRS member set agrees with the sidecar (every linked allele is a + # member, the cousin included), so the envelope is internally consistent. + assert len(body["molecularRepresentation"]["members"]) == len(body["alleles"]) # The alleles identity sidecar serializes camelCase, keyed by digest; the protein member is a # translation of the defining coding allele (which itself has no relation to itself). assert body["alleles"]["cdna-digest"]["level"] == "cdna" @@ -142,6 +152,10 @@ def test_get_variant_detail_envelope(client, session, data_provider, data_files, # The apex is a deterministic projection here but pairs with nothing (projectionOf dropped as null). assert body["alleles"]["prot-digest"]["derivation"] == "projection" assert "projectionOf" not in body["alleles"]["prot-digest"] + # The synonymous cousin (different projection group) surfaces as a member wearing co_encodes and is + # labelled `convergent` (a distinct change sharing the consequence, not an ambiguous candidate). + assert body["alleles"]["cousin-digest"]["relation"] == "co_encodes" + assert body["alleles"]["cousin-digest"]["derivation"] == "convergent" assert body["annotations"]["cdna-digest"]["vep"]["consequence"] == "missense_variant" assert body["isCurrent"] is True assert "supersededByScoreSet" not in body # dropped by exclude_none when current diff --git a/tests/routers/test_variant_annotation.py b/tests/routers/test_variant_annotation.py new file mode 100644 index 000000000..54d7fadf8 --- /dev/null +++ b/tests/routers/test_variant_annotation.py @@ -0,0 +1,352 @@ +# ruff: noqa: E402 +"""Router tests for the variant annotation surface relocated onto the Allele substrate. + +The VA-Spec statement endpoints (``/variants/{urn}/va/*``) and the VRS-identifier lookup +(``/variants/vrs/{identifier}``) moved off the legacy ``/mapped-variants`` router in #743; they build +from the ``MappingRecord`` / ``Allele`` substrate (seeded by ``seed_annotation_substrate``), never the +legacy ``MappedVariant``. +""" + +import json +from datetime import datetime, timezone + +import pytest + +from tests.helpers.util.user import change_ownership + +arq = pytest.importorskip("arq") +cdot = pytest.importorskip("cdot") +fastapi = pytest.importorskip("fastapi") + +from urllib.parse import quote_plus + +from ga4gh.va_spec.acmg_2015 import VariantPathogenicityStatement +from ga4gh.va_spec.base.core import ExperimentalVariantFunctionalImpactStudyResult, Statement + +from sqlalchemy import select + +from mavedb.models.score_set import ScoreSet as ScoreSetDbModel +from mavedb.models.variant import Variant as VariantDbModel +from tests.helpers.constants import ( + TEST_BIORXIV_IDENTIFIER, + TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED, + TEST_GA4GH_IDENTIFIER, + TEST_PUBMED_IDENTIFIER, + TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X, +) +from tests.helpers.util.annotation import AlleleSpec, seed_mapping_record +from tests.helpers.util.common import deepcamelize +from tests.helpers.util.experiment import create_experiment +from tests.helpers.util.score_calibration import create_publish_and_promote_score_calibration +from tests.helpers.util.score_set import ( + create_seq_score_set_with_mapped_variants, + create_seq_score_set_with_variants, + seed_annotation_substrate, +) + +_PUBLICATION_FETCH = [ + {"dbName": "PubMed", "identifier": f"{TEST_PUBMED_IDENTIFIER}"}, + {"dbName": "bioRxiv", "identifier": TEST_BIORXIV_IDENTIFIER}, +] + + +# --- StudyResult ----------------------------------------------------------------------------------- + + +def test_variant_study_result(client, session, data_provider, data_files, setup_router_db): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + seed_annotation_substrate(session, score_set) + + response = client.get(f"/api/v1/variants/{quote_plus(score_set['urn'] + '#1')}/va/study-result") + response_data = response.json() + + assert response.status_code == 200 + assert response_data["description"] == f"Variant effect study result for {score_set['urn']}#1." + ExperimentalVariantFunctionalImpactStudyResult.model_validate_json(json.dumps(response_data)) + + +def test_variant_study_result_404_when_variant_missing(client, session, data_provider, data_files, setup_router_db): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + + urn = f"{score_set['urn']}#404" + response = client.get(f"/api/v1/variants/{quote_plus(urn)}/va/study-result") + + assert response.status_code == 404 + assert response.json()["detail"] == f"variant with URN '{urn}' not found" + + +def test_variant_study_result_404_when_no_mapping_data(client, session, data_provider, data_files, setup_router_db): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + + # The variant exists but has no mapping record seeded on the new substrate. + urn = f"{score_set['urn']}#1" + response = client.get(f"/api/v1/variants/{quote_plus(urn)}/va/study-result") + + assert response.status_code == 404 + assert f"No study result exists for variant {urn}: no mapping data exists." in response.json()["detail"] + + +# --- Functional-impact Statement ------------------------------------------------------------------- + + +@pytest.mark.parametrize("mock_publication_fetch", [_PUBLICATION_FETCH], indirect=["mock_publication_fetch"]) +def test_variant_functional_impact_statement( + client, session, data_provider, data_files, setup_router_db, mock_publication_fetch +): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + create_publish_and_promote_score_calibration( + client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) + ) + seed_annotation_substrate(session, score_set) + + response = client.get(f"/api/v1/variants/{quote_plus(score_set['urn'] + '#1')}/va/functional-statement") + response_data = response.json() + + assert response.status_code == 200 + assert response_data["description"] == f"Variant functional impact statement for {score_set['urn']}#1." + Statement.model_validate_json(json.dumps(response_data)) + + +def test_variant_functional_impact_statement_404_when_insufficient_evidence( + client, session, data_provider, data_files, setup_router_db +): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + # Mapped on the new substrate, but no (primary) calibration => insufficient evidence. + seed_annotation_substrate(session, score_set) + + urn = f"{score_set['urn']}#1" + response = client.get(f"/api/v1/variants/{quote_plus(urn)}/va/functional-statement") + + assert response.status_code == 404 + assert ( + f"No functional impact statement exists for variant {urn}. Variant does not have sufficient evidence" + in response.json()["detail"] + ) + + +# --- Pathogenicity Statement ----------------------------------------------------------------------- + + +@pytest.mark.parametrize("mock_publication_fetch", [_PUBLICATION_FETCH], indirect=["mock_publication_fetch"]) +def test_variant_pathogenicity_statement( + client, session, data_provider, data_files, setup_router_db, mock_publication_fetch +): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + create_publish_and_promote_score_calibration( + client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) + ) + seed_annotation_substrate(session, score_set) + + response = client.get(f"/api/v1/variants/{quote_plus(score_set['urn'] + '#2')}/va/pathogenicity-statement") + response_data = response.json() + + assert response.status_code == 200 + assert f"Variant pathogenicity statement for {score_set['urn']}#2" in response_data["description"] + VariantPathogenicityStatement.model_validate_json(json.dumps(response_data)) + + +def test_variant_pathogenicity_statement_404_when_insufficient_evidence( + client, session, data_provider, data_files, setup_router_db +): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + seed_annotation_substrate(session, score_set) + + urn = f"{score_set['urn']}#1" + response = client.get(f"/api/v1/variants/{quote_plus(urn)}/va/pathogenicity-statement") + + assert response.status_code == 404 + assert ( + f"No pathogenicity statement exists for variant {urn}; Variant does not have sufficient evidence" + in response.json()["detail"] + ) + + +# --- VRS-identifier lookup ------------------------------------------------------------------------- + + +def test_lookup_variants_by_vrs_identifier(client, session, data_provider, data_files, setup_router_db): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + variant = session.scalar(select(VariantDbModel).where(VariantDbModel.urn == f"{score_set['urn']}#1")) + # vrs_digest stores the full GA4GH VRS id in production (the mapper writes post_mapped["id"] into it), + # so the lookup matches the identifier against vrs_digest. The allele's ClinGen id rides alongside the URN. + seed_mapping_record( + session, + variant, + assay_level="cdna", + alleles=[ + AlleleSpec( + digest=TEST_GA4GH_IDENTIFIER, + level="cdna", + is_authoritative=True, + clingen_allele_id="CA1", + post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X, + ) + ], + ) + + response = client.get(f"/api/v1/variants/vrs/{quote_plus(TEST_GA4GH_IDENTIFIER)}") + response_data = response.json() + + assert response.status_code == 200 + assert len(response_data) == 1 + assert response_data[0]["variantUrn"] == f"{score_set['urn']}#1" + assert response_data[0]["clingenAlleleId"] == "CA1" + assert response_data[0]["vrsId"] == TEST_GA4GH_IDENTIFIER + + +def test_lookup_variants_by_vrs_identifier_empty_when_none_match( + client, session, data_provider, data_files, setup_router_db +): + """The lookup is collection-shaped: no match is an empty list, not a 404.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + seed_annotation_substrate(session, score_set) + + # A well-formed GA4GH id (32-char digest) that no seeded allele carries. + response = client.get(f"/api/v1/variants/vrs/{quote_plus('ga4gh:VA.AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA')}") + + assert response.status_code == 200 + assert response.json() == [] + + +def test_lookup_variants_by_vrs_identifier_empty_without_read_permission( + client, session, data_provider, data_files, setup_router_db +): + """Existence-hiding: an identifier that matches only a private score set is filtered to an empty list, + indistinguishable from a genuine no-match — so the response never reveals the private allele exists.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + variant = session.scalar(select(VariantDbModel).where(VariantDbModel.urn == f"{score_set['urn']}#1")) + # The identifier matches a real allele (by vrs_digest); the permission filter drops it to an empty list. + seed_mapping_record( + session, + variant, + assay_level="cdna", + alleles=[AlleleSpec(digest=TEST_GA4GH_IDENTIFIER, level="cdna", is_authoritative=True)], + ) + change_ownership(session, score_set["urn"], ScoreSetDbModel) + + response = client.get(f"/api/v1/variants/vrs/{quote_plus(TEST_GA4GH_IDENTIFIER)}") + + assert response.status_code == 200 + assert response.json() == [] + + +# --- as_of temporal behavior ------------------------------------------------------------------------ + + +@pytest.mark.parametrize("va_endpoint", ["study-result", "functional-statement", "pathogenicity-statement"]) +def test_va_endpoint_404_when_no_mapping_data_at_as_of( + client, session, data_provider, data_files, setup_router_db, va_endpoint +): + """A past instant with no live mapping yields the single-resource 404 — the statement did not exist then. + + ``as_of`` is threaded into ``variant_annotation_context``, so a far-past instant sees no live mapping + record and the derived VA statement cannot be constructed. This is the deliberate contrast with the + score-set *collection* endpoints: an empty collection is 200 + empty, but a single derived resource + that does not exist at the requested instant is a 404. + """ + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + seed_annotation_substrate(session, score_set) + + urn = f"{score_set['urn']}#1" + past = datetime(2000, 1, 1, tzinfo=timezone.utc) + response = client.get(f"/api/v1/variants/{quote_plus(urn)}/va/{va_endpoint}", params={"as_of": past.isoformat()}) + + assert response.status_code == 404 + assert "no mapping data exists" in response.json()["detail"] + + +def test_variant_study_result_honors_as_of(client, session, data_provider, data_files, setup_router_db): + """The study-result route reconstructs its subject at ``as_of`` and echoes the instant on ``X-As-Of``. + + Study result needs only live mapping (no calibration), so it cleanly exercises the positive path: a + far-future instant still sees the freshly-seeded substrate as live and returns 200. Omitting ``as_of`` + reports ``current``. + """ + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + seed_annotation_substrate(session, score_set) + + url = f"/api/v1/variants/{quote_plus(score_set['urn'] + '#1')}/va/study-result" + + # No as_of: the header reports the current standing. + current_response = client.get(url) + assert current_response.status_code == 200 + assert current_response.headers["X-As-Of"] == "current" + + # Far future: the seeded mapping is live, so the subject reconstructs and the instant echoes back. + future = datetime(2999, 1, 1, tzinfo=timezone.utc) + future_response = client.get(url, params={"as_of": future.isoformat()}) + assert future_response.status_code == 200 + assert datetime.fromisoformat(future_response.headers["X-As-Of"]) == future + + +def test_lookup_variants_by_vrs_identifier_honors_as_of(client, session, data_provider, data_files, setup_router_db): + """The VRS lookup filters its matches by ``as_of`` (the allele link must be live at that instant).""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + variant = session.scalar(select(VariantDbModel).where(VariantDbModel.urn == f"{score_set['urn']}#1")) + seed_mapping_record( + session, + variant, + assay_level="cdna", + alleles=[ + AlleleSpec( + digest=TEST_GA4GH_IDENTIFIER, + level="cdna", + is_authoritative=True, + clingen_allele_id="CA1", + post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X, + ) + ], + ) + url = f"/api/v1/variants/vrs/{quote_plus(TEST_GA4GH_IDENTIFIER)}" + + # Far future: the allele link is live, so the identifier resolves and the instant echoes back. + future = datetime(2999, 1, 1, tzinfo=timezone.utc) + future_response = client.get(url, params={"as_of": future.isoformat()}) + assert future_response.status_code == 200 + assert len(future_response.json()) == 1 + assert datetime.fromisoformat(future_response.headers["X-As-Of"]) == future + + # Far past: nothing was live yet, so no allele matches — an empty collection, not a 404. + past = datetime(2000, 1, 1, tzinfo=timezone.utc) + past_response = client.get(url, params={"as_of": past.isoformat()}) + assert past_response.status_code == 200 + assert past_response.json() == [] From 275a75a4a3c50ccea0a3ef7d188d97e59359ce83 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Sat, 18 Jul 2026 13:24:50 -0700 Subject: [PATCH 092/252] feat(alleles): add allele-detail endpoint (digest / CAID / PAID) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add GET /alleles/{identifier}, the allele-grain sibling of GET /variants/{urn}: anchor identity + the full cross-layer equivalence class (each member labelled relative to the focus — projection / candidate / convergent) + digest-keyed annotations. One path overloaded across a VRS digest, a nucleotide CAID, or a protein PAID via a custom Starlette convertor. Public and measurement-agnostic; no Cat-VRS (that stays rooted at the measured allele on the variant view). Unify the molecular core with variant detail: extract a shared AlleleIdentity (lib + view model) used by both endpoints' `alleles` map, and mark the anchored allele with `isFocus` — replacing the `derivation: "authoritative"` value on variant detail (there is no authoritative *variant* at the allele grain). Factor the nucleotide-level set into SequenceLevel.NUCLEOTIDE_LEVELS, shared by cat_vrs and allele_detail. --- src/mavedb/lib/allele_detail.py | 188 ++++++++++++++++ src/mavedb/lib/allele_identity.py | 68 ++++++ src/mavedb/lib/cat_vrs.py | 11 +- src/mavedb/lib/variant_detail.py | 78 +------ src/mavedb/models/enums/sequence_level.py | 14 ++ src/mavedb/routers/alleles.py | 129 +++++++++++ src/mavedb/server_main.py | 3 + src/mavedb/view_models/allele_annotation.py | 6 +- src/mavedb/view_models/allele_detail.py | 39 ++++ src/mavedb/view_models/allele_identity.py | 29 +++ src/mavedb/view_models/variant_detail.py | 29 +-- tests/lib/test_allele_detail.py | 203 ++++++++++++++++++ tests/lib/test_variant_detail.py | 21 +- tests/routers/test_allele.py | 224 ++++++++++++++++++++ tests/routers/test_variant.py | 8 +- 15 files changed, 933 insertions(+), 117 deletions(-) create mode 100644 src/mavedb/lib/allele_detail.py create mode 100644 src/mavedb/lib/allele_identity.py create mode 100644 src/mavedb/routers/alleles.py create mode 100644 src/mavedb/view_models/allele_detail.py create mode 100644 src/mavedb/view_models/allele_identity.py create mode 100644 tests/lib/test_allele_detail.py create mode 100644 tests/routers/test_allele.py diff --git a/src/mavedb/lib/allele_detail.py b/src/mavedb/lib/allele_detail.py new file mode 100644 index 000000000..14603f509 --- /dev/null +++ b/src/mavedb/lib/allele_detail.py @@ -0,0 +1,188 @@ +"""The allele-detail view backing ``GET /alleles/{digest:CAID}``. + +The allele-grain sibling of :mod:`lib.variant_detail`. Where the variant view anchors on a +*measurement*, this anchors on an *allele* (a VRS digest, or a CAID naming the nt-canonical change) +and serves that allele's identity, its full cross-layer **equivalence class**, and the digest-keyed +annotations that ride alongside. + +It is measurement-agnostic, and that shapes what it does *not* carry: + +- No score, counts, classification, or version standing — those are properties of a measured variant, + not of an allele (an allele is deduplicated by digest and shared across every score set mapping to it). +- No spec-pure Cat-VRS. The ``CategoricalVariant`` / ``DefiningAlleleConstraint`` is emitted only by the + measurement view, rooted at the measured allele. Here the members are labelled on a flat map + *relative to the focus allele*. + +**The equivalence class is the full cross-record union** (:func:`lib.alleles.get_allele_translations`): +every allele co-linked to any live record touching the focus. That is the discoverable "everything +related to this change" set. Each member is then labelled relative to the focus — which the graph +determines even without a measurement, because candidate-ness is *directional* (protein→nt is +many-to-one ⇒ candidate; nt→protein is deterministic ⇒ consequence; nt↔nt is a faithful coordinate +transform). "Less power than a measurement" only means we cannot say which nt was *measured* — but we +can still say faithful vs. candidate vs. convergent from the graph and the focus. + +``as_of`` reconstructs the molecular layer — class membership + annotations — at the past instant. The +focus allele's own identity is immutable and content-addressed. ``as_of`` defaults to currently-live rows. +""" + +from dataclasses import dataclass +from datetime import datetime +from typing import Any, Optional + +from sqlalchemy import select +from sqlalchemy.orm import Session + +from mavedb.lib.allele_annotations import AlleleAnnotations, get_allele_annotations +from mavedb.lib.allele_identity import AlleleDerivation, AlleleIdentity +from mavedb.lib.alleles import get_allele_translations +from mavedb.lib.cat_vrs import CatVrsRelation +from mavedb.lib.vrs import vrs_object_from_mapped_variant +from mavedb.models.allele import Allele +from mavedb.models.enums.sequence_level import NUCLEOTIDE_LEVELS, SequenceLevel +from mavedb.models.mapping_record_allele import MappingRecordAllele + + +@dataclass(frozen=True) +class AlleleDetail: + """The assembled allele-detail envelope (transit; serialized by ``view_models.allele_detail``).""" + + # Flat anchor identity — the queried allele's own molecular facts (for a CAID fetch, its + # representative allele; ``clingenAlleleId`` carries the CAID either way). + digest: str + level: Optional[str] + hgvs: Optional[str] # reference-frame HGVS (exactly one of hgvs_g/c/p, depending on level) + clingen_allele_id: Optional[str] + vrs: Optional[dict[str, Any]] # spec-pure GA4GH VRS variation from post_mapped + + # The full cross-layer equivalence class, keyed by VRS digest and sharing keys with `annotations`. + # Each member is an `AlleleIdentity` labelled relative to the focus. The focus allele(s) carry + # `is_focus=True`. + alleles: dict[str, AlleleIdentity] + + # External annotations, keyed by VRS digest, for every member of the class. + annotations: dict[str, AlleleAnnotations] + + +def _focus_projection_pairs(db: Session, focus_alleles: list[Allele], *, as_of: Optional[datetime]) -> dict[str, str]: + """A symmetric ``digest -> sibling digest`` map for each focus allele's c↔g coordinate partner. + + Read from any one live record of the focus: the partner is the ≤1 *other* allele sharing the focus + allele's ``projection_group`` (a projection group is a c↔g pair). This is the one place the + focus/partner-vs-convergent-cousin distinction needs the link-level ``projection_group`` — every + other member is classified by level alone. + """ + pairs: dict[str, str] = {} + for focus in focus_alleles: + if focus.vrs_digest is None: + continue + link = db.scalar( + select(MappingRecordAllele) + .where(MappingRecordAllele.allele_id == focus.id) + .where(MappingRecordAllele.projection_group.isnot(None)) + .where(MappingRecordAllele.live_at(as_of)) + .limit(1) + ) + if link is None: + continue + sibling_digests = db.scalars( + select(Allele.vrs_digest) + .select_from(MappingRecordAllele) + .join(Allele, Allele.id == MappingRecordAllele.allele_id) + .where(MappingRecordAllele.mapping_record_id == link.mapping_record_id) + .where(MappingRecordAllele.projection_group == link.projection_group) + .where(MappingRecordAllele.allele_id != focus.id) + .where(MappingRecordAllele.live_at(as_of)) + ).all() + for sibling in sibling_digests: + if sibling is not None: + pairs[focus.vrs_digest] = sibling + pairs.setdefault(sibling, focus.vrs_digest) + return pairs + + +def _member_label( + *, focus_is_protein: bool, member_level: Optional[str], is_coordinate_partner: bool +) -> tuple[Optional[CatVrsRelation], Optional[AlleleDerivation]]: + """The (relation, derivation) for a non-focus member, read *relative to the focus*. + + - **protein focus** → every nt member is a reverse-translation ``candidate`` that ``encodes`` it. + - **nt focus** → the protein consequence is ``translation_of`` (a deterministic ``projection``); the + focus's coordinate partner is ``coordinate_representation_of`` (also ``projection``); any other nt + is a synonymous cousin — ``co_encodes`` / ``convergent`` (a distinct change sharing the consequence). + """ + if focus_is_protein: + if member_level in NUCLEOTIDE_LEVELS: + return CatVrsRelation.ENCODES, AlleleDerivation.CANDIDATE + return None, None + + if member_level == SequenceLevel.protein.value: + return CatVrsRelation.TRANSLATION_OF, AlleleDerivation.PROJECTION + if member_level in NUCLEOTIDE_LEVELS: + if is_coordinate_partner: + return CatVrsRelation.COORDINATE_REPRESENTATION_OF, AlleleDerivation.PROJECTION + return CatVrsRelation.CO_ENCODES, AlleleDerivation.CONVERGENT + return None, None + + +def get_allele_detail( + db: Session, anchor: Allele, *, focus_digests: set[str], as_of: Optional[datetime] = None +) -> AlleleDetail: + """Assemble the allele-detail envelope anchored on ``anchor``. + + ``focus_digests`` is the set of VRS digests that are the *focus* of this view — a single digest for + a by-digest fetch, or the (genomic + coding) representations of a CAID for a by-CAID fetch. The + equivalence class is the record-agnostic union co-linked to any live record touching ``anchor`` + (:func:`get_allele_translations`, which includes ``anchor``); an orphan allele with no live links + falls back to itself. Every member is labelled relative to the focus (see :func:`_member_label`). + ``as_of`` reconstructs the molecular layer; see the module docstring. + """ + equivalence_class = get_allele_translations(db, anchor.id, as_of=as_of) or [anchor] + + annotations = get_allele_annotations(db, equivalence_class, as_of=as_of) + + focus_alleles = [a for a in equivalence_class if a.vrs_digest in focus_digests] or [anchor] + focus_is_protein = any(a.level == SequenceLevel.protein.value for a in focus_alleles) + + pairs = _focus_projection_pairs(db, focus_alleles, as_of=as_of) + # The focus's coordinate partner(s) that are not themselves focus (for a CAID fetch the c↔g pair is + # jointly the focus, so this is empty and the partners are simply flagged is_focus instead). + coordinate_partner_digests = {pairs[d] for d in focus_digests if d in pairs} - focus_digests + + alleles: dict[str, AlleleIdentity] = {} + for member in equivalence_class: + digest = member.vrs_digest + if digest is None: + continue + + if digest in focus_digests: + relation, derivation = None, None + else: + rel, der = _member_label( + focus_is_protein=focus_is_protein, + member_level=member.level, + is_coordinate_partner=digest in coordinate_partner_digests, + ) + relation = rel.value if rel is not None else None + derivation = der.value if der is not None else None + + alleles[digest] = AlleleIdentity( + level=member.level, + hgvs=member.hgvs_g or member.hgvs_c or member.hgvs_p, + clingen_allele_id=member.clingen_allele_id, + is_focus=digest in focus_digests, + relation=relation, + derivation=derivation, + projection_of=pairs.get(digest), + ) + + return AlleleDetail( + digest=anchor.vrs_digest or "", + level=anchor.level, + hgvs=anchor.hgvs_g or anchor.hgvs_c or anchor.hgvs_p, + clingen_allele_id=anchor.clingen_allele_id, + vrs=vrs_object_from_mapped_variant(anchor.post_mapped).model_dump(mode="json", exclude_none=True) + if anchor.post_mapped is not None + else None, + alleles=alleles, + annotations=annotations, + ) diff --git a/src/mavedb/lib/allele_identity.py b/src/mavedb/lib/allele_identity.py new file mode 100644 index 000000000..7f4c12a74 --- /dev/null +++ b/src/mavedb/lib/allele_identity.py @@ -0,0 +1,68 @@ +"""The MaveDB molecular-identity view of an allele — shared by variant detail and allele detail. + +Both serving views present a set of alleles keyed by VRS digest, each labelled by how it relates to +the allele the view is anchored on (its *focus*). The two views differ only in what the focus is: +``GET /variants/{urn}`` focuses the **measured** allele, ``GET /alleles/{digest}`` focuses the +**queried** allele. The identity shape is shared and its labels are always read *relative to the +focus*. + +This module owns that shared shape (:class:`AlleleIdentity`) and its provenance vocabulary +(:class:`AlleleDerivation`). The Cat-VRS structural relation codes live in :mod:`lib.cat_vrs` +(``CatVrsRelation``); this module carries only the derivation (confidence/provenance) axis, which is +orthogonal. +""" + +from dataclasses import dataclass +from enum import Enum +from typing import Optional + + +class AlleleDerivation(str, Enum): + """How an allele's representation was arrived at, *relative to the focus allele*: its + confidence/provenance axis. Orthogonal to the Cat-VRS structural ``relation`` axis: ``relation`` + says which level relates to which, ``derivation`` says how much to trust the representation. + + There is deliberately **no** ``authoritative`` value: the focus allele is marked by + :attr:`AlleleIdentity.is_focus`, not by a derivation. That keeps the axis meaningful without a + measurement — on the allele view there is no *measured* allele, only the queried focus. + """ + + # Deterministic and precise, given (assembly, transcript): the focus's coordinate partner + # (nucleotide↔nucleotide) and its protein consequence (nucleotide→protein). + PROJECTION = "projection" + # Reverse-translation of a *protein* focus: genuinely ambiguous. One member of the fanned-out + # equivalence class, not a precise and deterministic pair. + CANDIDATE = "candidate" + # A distinct, precisely-known nucleotide change that *converges* on the same protein consequence + # a different codon produces. Not ambiguous like a candidate nor a projection *of* the focus. A + # separate, unmeasured variant that simply shares the consequence. + CONVERGENT = "convergent" + + +@dataclass(frozen=True) +class AlleleIdentity: + """The MaveDB molecular-identity facts for one allele in a view's ``alleles`` map, keyed by VRS + digest. ``level`` + reference-frame ``hgvs`` (exactly one of the allele's genomic/coding/protein + columns) are what the UI labels by — never the digest. + + Four axes, all read *relative to the view's focus allele*: + + - ``is_focus``: this allele is the one the view is anchored on (the measured allele on variant + detail; the queried allele/CAID on allele detail). Its ``relation``/``derivation`` are ``None`` — + it is the reference point the others are described against. + - ``relation`` (Cat-VRS, structural): member→focus relation (``coordinate_representation_of`` / + ``translation_of`` / ``encodes`` / ``co_encodes``). ``None`` for the focus itself. + - ``derivation`` (provenance): :class:`AlleleDerivation` — projection / candidate / convergent. + Orthogonal to ``relation``; never conflate them. + - ``projection_of`` (provenance): the VRS digest of this allele's c↔g projection sibling, when + known. ``None`` for the protein apex, pre-reverse-translation data, or where the pairing is not + resolved in this view. + """ + + level: Optional[str] + hgvs: Optional[str] + clingen_allele_id: Optional[str] + is_focus: bool + relation: Optional[str] + derivation: Optional[str] + projection_of: Optional[str] diff --git a/src/mavedb/lib/cat_vrs.py b/src/mavedb/lib/cat_vrs.py index b9ad6d39c..02e1d1d03 100644 --- a/src/mavedb/lib/cat_vrs.py +++ b/src/mavedb/lib/cat_vrs.py @@ -31,7 +31,7 @@ from mavedb.lib.logging.context import logging_context from mavedb.lib.vrs import vrs_object_from_mapped_variant from mavedb.models.allele import Allele -from mavedb.models.enums.sequence_level import SequenceLevel +from mavedb.models.enums.sequence_level import NUCLEOTIDE_LEVELS, SequenceLevel from mavedb.models.mapping_record_allele import MappingRecordAllele logger = logging.getLogger(__name__) @@ -63,9 +63,6 @@ class CatVrsMode(str, Enum): REVERSE_TRANSLATION = "reverse_translation" # Mode 2 — protein measured; score is implied. -_NUCLEOTIDE_LEVELS = {SequenceLevel.genomic.value, SequenceLevel.cdna.value} - - @dataclass class CategoricalVariantTransit: """The spec-pure Cat-VRS object plus the MaveDB layer that rides beside it.""" @@ -87,7 +84,7 @@ def is_convergent_cousin( - The member is in a different projection group than the defining (measured) allele. """ return ( - member_level in _NUCLEOTIDE_LEVELS + member_level in NUCLEOTIDE_LEVELS and member_group is not None and defining_group is not None and member_group != defining_group @@ -109,12 +106,12 @@ def _relation_for( """ # Protein measured. Every nt member encodes it, the protein member is the defining. Grouping is irrelevant. if defining_level == SequenceLevel.protein.value: - return CatVrsRelation.ENCODES if member_level in _NUCLEOTIDE_LEVELS else None + return CatVrsRelation.ENCODES if member_level in NUCLEOTIDE_LEVELS else None # Nt measured (genomic or cdna). if member_level == SequenceLevel.protein.value: return CatVrsRelation.TRANSLATION_OF - if member_level in _NUCLEOTIDE_LEVELS: + if member_level in NUCLEOTIDE_LEVELS: if is_convergent_cousin(member_level, member_group, defining_group=defining_group): return CatVrsRelation.CO_ENCODES return CatVrsRelation.COORDINATE_REPRESENTATION_OF diff --git a/src/mavedb/lib/variant_detail.py b/src/mavedb/lib/variant_detail.py index 8f06c7cc5..567fcf226 100644 --- a/src/mavedb/lib/variant_detail.py +++ b/src/mavedb/lib/variant_detail.py @@ -18,13 +18,13 @@ from dataclasses import dataclass from datetime import datetime -from enum import Enum from typing import Any, Optional from sqlalchemy import select from sqlalchemy.orm import Session from mavedb.lib.allele_annotations import AlleleAnnotations, get_allele_annotations +from mavedb.lib.allele_identity import AlleleDerivation, AlleleIdentity from mavedb.lib.alleles import get_live_record_allele_links from mavedb.lib.cat_vrs import categorical_variant_for_variant, is_convergent_cousin from mavedb.lib.score_calibrations import calibration_preference_key @@ -54,63 +54,6 @@ class VariantClassificationRecord: classification: ScoreCalibrationFunctionalClassification -class AlleleDerivation(str, Enum): - """How an allele's representation was arrived at — its confidence/provenance axis. - - Distinct from (orthogonal to) the Cat-VRS ``relation`` axis: ``relation`` is *structural* (which - level relates to which, member→defining), whereas ``derivation`` is *epistemic* (how much to trust - the representation). See the design's "Semantics note — two axes, keep them separate". Derived at - serialization time from ``is_authoritative`` + the assay level — no stored column. - """ - - # The assay's actual measurement (the authoritative link). Precise by definition. - AUTHORITATIVE = "authoritative" - # Deterministic and precise. Derived from the measured change itself: nucleotide↔nucleotide (its - # coordinate partner) and nucleotide→protein (its consequence) are both deterministic given - # (assembly, transcript). - PROJECTION = "projection" - # Reverse-translation output of a *protein* measurement — genuinely ambiguous (many synonymous - # codons). One member of the fanned-out equivalence class, not a precise coordinate. - CANDIDATE = "candidate" - # A distinct, precisely-known nucleotide change that *converges* on the measured protein consequence - # from a different codon (a synonymous cousin under a nucleotide assay). Not ambiguous like a - # candidate, and not a projection *of* the measured change — a separate, unmeasured variant that - # merely shares the consequence. Pairs with the ``co_encodes`` Cat-VRS relation. - CONVERGENT = "convergent" - - -@dataclass(frozen=True) -class AlleleIdentity: - """The MaveDB molecular-identity facts for one of the variant's linked alleles. - - Rides alongside the spec-pure Cat-VRS keyed by VRS digest (the ``alleles`` map). ``level`` and - ``hgvs`` (the reference-frame HGVS, exactly one of the allele's genomic/coding/protein columns) are - what the UI labels the per-level annotation panel by — *never* the digest. - - Three axes ride here, deliberately independent: - - - ``relation`` (Cat-VRS, structural): this allele's member→defining relation - (``is_protein_of`` / ``coordinate_representation_of`` / …). ``None`` when it *is* the measured - allele, or when the allele is not a Cat-VRS member. Sourced from - ``cat_vrs.CategoricalVariantTransit.member_relations``. - - ``derivation`` (provenance): :class:`AlleleDerivation` — authoritative / projection / candidate / - convergent. Orthogonal to ``relation``; never conflate the two (a protein member of a nucleotide - assay has ``relation=translation_of`` but ``derivation=projection``; a synonymous cousin has - ``relation=co_encodes`` and ``derivation=convergent``; the nucleotide fan-out of a protein assay is - ``derivation=candidate``). - - ``projection_of`` (provenance): the VRS digest of this allele's projection sibling — the ≤1 *other* - member of its ``projection_group`` (a c↔g pair). ``None`` for the protein apex (group ``NULL``), - for pre-reverse-translation data, and where a level's projection failed. - """ - - level: Optional[str] - hgvs: Optional[str] - clingen_allele_id: Optional[str] - relation: Optional[str] - derivation: Optional[str] - projection_of: Optional[str] - - @dataclass(frozen=True) class VariantDetail: """The assembled variant-detail envelope (transit; serialized by ``view_models.variant``).""" @@ -177,13 +120,11 @@ def _classifications_for_variant( ] -def _derivation_for( - *, is_authoritative: bool, assay_level: Optional[SequenceLevel], is_cousin: bool -) -> AlleleDerivation: - """The provenance of a linked allele's representation, from ``is_authoritative`` + the assay level. +def _derivation_for(*, assay_level: Optional[SequenceLevel], is_cousin: bool) -> AlleleDerivation: + """The provenance of a *non-focus* linked allele's representation, from the assay level. - The measured allele is ``authoritative``. Every *other* allele's provenance turns on the protein → - codon boundary, which surfaces three ways: + The measured allele is the focus (``is_focus=True``, no derivation). Every *other* allele's + provenance turns on the protein → codon boundary, which surfaces three ways: - **protein assay** (``assay_level`` protein) → ``candidate`` for the whole derived (nucleotide) fan-out: reverse translation is genuinely ambiguous (which codon was measured is unknown). @@ -199,8 +140,6 @@ def _derivation_for( ``coordinate_representation_of`` / ``translation_of`` ↔ ``projection``, protein-assay ``encodes`` ↔ ``candidate``. """ - if is_authoritative: - return AlleleDerivation.AUTHORITATIVE if assay_level == SequenceLevel.protein.value: return AlleleDerivation.CANDIDATE if is_cousin: @@ -301,10 +240,11 @@ def get_variant_detail( level=allele.level, hgvs=allele.hgvs_g or allele.hgvs_c or allele.hgvs_p, clingen_allele_id=allele.clingen_allele_id, + is_focus=link.is_authoritative, relation=relation.value if relation is not None else None, - derivation=_derivation_for( - is_authoritative=link.is_authoritative, assay_level=assay_level, is_cousin=is_cousin - ).value, + derivation=( + None if link.is_authoritative else _derivation_for(assay_level=assay_level, is_cousin=is_cousin).value + ), projection_of=projection_of, ) diff --git a/src/mavedb/models/enums/sequence_level.py b/src/mavedb/models/enums/sequence_level.py index 016c36f96..62875f99f 100644 --- a/src/mavedb/models/enums/sequence_level.py +++ b/src/mavedb/models/enums/sequence_level.py @@ -25,6 +25,16 @@ def from_wire(cls, code: str) -> "SequenceLevel": except KeyError as exc: raise ValueError(f"Unknown sequence level wire code: {code!r}") from exc + @classmethod + def nucleotide_levels(cls) -> set["SequenceLevel"]: + """Return the subset of levels that are nucleotide (genomic or coding).""" + return {cls.genomic, cls.cdna} + + @classmethod + def protein_levels(cls) -> set["SequenceLevel"]: + """Return the subset of levels that are protein (protein only).""" + return {cls.protein} + # Module-level so it doesn't get mistaken for an enum member. _WIRE_TO_LEVEL: dict[str, SequenceLevel] = { @@ -32,3 +42,7 @@ def from_wire(cls, code: str) -> "SequenceLevel": "c": SequenceLevel.cdna, "g": SequenceLevel.genomic, } + + +NUCLEOTIDE_LEVELS = {level.value for level in SequenceLevel.nucleotide_levels()} +PROTEIN_LEVELS = {level.value for level in SequenceLevel.protein_levels()} diff --git a/src/mavedb/routers/alleles.py b/src/mavedb/routers/alleles.py new file mode 100644 index 000000000..abd575718 --- /dev/null +++ b/src/mavedb/routers/alleles.py @@ -0,0 +1,129 @@ +import logging +from datetime import datetime +from typing import Optional + +from fastapi import APIRouter, Depends, Path, Query, Response +from fastapi.exceptions import HTTPException +from sqlalchemy import select +from sqlalchemy.orm import Session +from starlette.convertors import Convertor, register_url_convertor + +from mavedb import deps +from mavedb.lib.allele_detail import get_allele_detail +from mavedb.lib.logging import LoggedRoute +from mavedb.lib.logging.context import save_to_logging_context +from mavedb.models.allele import Allele +from mavedb.models.enums.sequence_level import SequenceLevel +from mavedb.routers.shared import ( + ACCESS_CONTROL_ERROR_RESPONSES, + PUBLIC_ERROR_RESPONSES, + ROUTER_BASE_PREFIX, +) +from mavedb.view_models.allele_detail import AlleleDetail + +TAG_NAME = "Alleles" + +logger = logging.getLogger(__name__) + + +# One resource, three identifier forms — a GA4GH VRS digest (one allele), a ClinGen nucleotide id (CAID, +# the nt-canonical change) or protein id (PAID). Rather than split the path or add a query param, overload +# a single `/alleles/{identifier}` with a custom Starlette convertor (the same pattern the publication +# router uses for DOI/PubMed/…). The convertor matches any of the three forms; the handler dispatches by +# form. In the OpenAPI spec this is still just a string path parameter. +class AlleleIdentifierConverter(Convertor): + # A GA4GH IR (ga4gh:.<32 base64url>) OR a ClinGen allele id (CA…/PA… + digits). + # NOTE: Multivariant CAIDs (CA…/PA… + digits + `,` + CA…/PA… + digits) are intentionally excluded. + regex = r"(?:ga4gh:[^.]+\.[0-9A-Za-z_\-]{32})|(?:[CP]A[0-9]+)" + + def convert(self, value: str) -> str: + return value + + def to_string(self, value: str) -> str: + return str(value) + + +register_url_convertor("allele_identifier", AlleleIdentifierConverter()) + +router = APIRouter( + prefix=f"{ROUTER_BASE_PREFIX}", + tags=[TAG_NAME], + responses={**PUBLIC_ERROR_RESPONSES}, + route_class=LoggedRoute, +) + +metadata = { + "name": TAG_NAME, + "description": "Retrieve deduplicated alleles — identity, cross-layer equivalence class, and " + "external annotations — by VRS digest or ClinGen allele id (CAID / PAID).", +} + + +def _representative(alleles: list[Allele]) -> Allele: + """The allele whose identity fills the flat anchor fields for a ClinGen-id fetch. A CAID names one nt + change in up to two frames (genomic + coding) — prefer the coding frame, then genomic; a PAID matches + the single protein allele, which falls through to ``alleles[0]``.""" + by_level = {a.level: a for a in alleles} + return by_level.get(SequenceLevel.cdna.value) or by_level.get(SequenceLevel.genomic.value) or alleles[0] + + +@router.get( + "/alleles/{identifier:allele_identifier}", + status_code=200, + response_model=AlleleDetail, + responses={**ACCESS_CONTROL_ERROR_RESPONSES}, + response_model_exclude_none=True, + summary="Fetch allele detail by VRS digest, CAID, or PAID", +) +def get_allele( + *, + response: Response, + identifier: str = Path( + description="A GA4GH VRS digest (one allele), or a ClinGen allele id — a nucleotide CAID (the " + "nt-canonical change, its genomic + coding frames) or a protein PAID.", + json_schema_extra={"example": "ga4gh:VA.0123abcd"}, + ), + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the molecular layer (equivalence-class membership + VEP/gnomAD/ClinVar " + "annotations) as it stood at this instant. ISO 8601, ideally timezone-aware. The focus " + "allele's own identity is content-addressed and immutable, so it is unaffected. Defaults to " + "current." + ), + ), + db: Session = Depends(deps.get_db), +): + """Fetch the detail envelope for a deduplicated allele, by any of its identifiers. + + The allele-grain sibling of ``GET /variants/{urn}``. Flat anchor identity (digest, level, HGVS, + ClinGen id, spec-pure VRS) plus the cross-layer equivalence class (each member labelled relative to + the focus) and a digest-keyed annotation map. The ``identifier`` may be: + + - a **VRS digest** (``ga4gh:VA.…``) — focuses that one allele. + - a **CAID** (``CA…``) — the nt-canonical change; The coding frame is the preferential focus, + falling back to the genomic frame if no coding frame exists. + - a **PAID** (``PA…``) — the protein change; the protein allele is focused and its nucleotide + equivalents surface as reverse-translation candidates. + + This is a **public molecular resource**. It carries no score-set-level information. No scores, + classifications, measurements, or version standing. Only the allele's own identity, its cross-layer + equivalence class, and public reference annotations (VEP / gnomAD / ClinVar). + """ + save_to_logging_context({"requested_resource": identifier, "as_of": as_of}) + response.headers["X-As-Of"] = as_of.isoformat() if as_of is not None else "current" + + if identifier.startswith("ga4gh:"): + allele = db.scalar(select(Allele).where(Allele.vrs_digest == identifier)) + if allele is None: + raise HTTPException(status_code=404, detail=f"allele with VRS digest '{identifier}' not found") + return get_allele_detail(db, allele, focus_digests={identifier}, as_of=as_of) + + # A ClinGen allele id (CAID or PAID): resolve to its allele(s) — the nt-canonical change's genomic + + # coding frames for a CAID, or the single protein allele for a PAID — and focus all of them. + matches = list(db.scalars(select(Allele).where(Allele.clingen_allele_id == identifier)).all()) + if not matches: + raise HTTPException(status_code=404, detail=f"no allele with ClinGen allele id '{identifier}' found") + + focus_digests = {a.vrs_digest for a in matches if a.vrs_digest is not None} + return get_allele_detail(db, _representative(matches), focus_digests=focus_digests, as_of=as_of) diff --git a/src/mavedb/server_main.py b/src/mavedb/server_main.py index 12fa5bf35..d3e8c4221 100644 --- a/src/mavedb/server_main.py +++ b/src/mavedb/server_main.py @@ -39,6 +39,7 @@ from mavedb.models import * # noqa: F403 from mavedb.routers import ( access_keys, + alleles, alphafold, api_information, clingen_alleles, @@ -92,6 +93,7 @@ ) app.add_middleware(GZipMiddleware, minimum_size=1000, compresslevel=5) app.include_router(access_keys.router) +app.include_router(alleles.router) app.include_router(api_information.router) app.include_router(clingen_alleles.router) app.include_router(collections.router) @@ -240,6 +242,7 @@ def customize_openapi_schema(): openapi_schema["tags"] = [ access_keys.metadata, + alleles.metadata, api_information.metadata, clingen_alleles.metadata, collections.metadata, diff --git a/src/mavedb/view_models/allele_annotation.py b/src/mavedb/view_models/allele_annotation.py index b538d4639..a995fabcb 100644 --- a/src/mavedb/view_models/allele_annotation.py +++ b/src/mavedb/view_models/allele_annotation.py @@ -2,9 +2,9 @@ The pydantic serialization boundary over the ``lib.allele_annotations`` transit dataclasses (``from_attributes`` coerces them directly). These are the sparse, digest-keyed annotation blocks -that ride *alongside* the spec-pure Cat-VRS on ``GET /variants/{urn}`` and are the allele's own block -on ``GET /alleles/{digest}`` — so they live here, shared by both serving views rather than in either -one's file. +that ride *alongside* the spec-pure Cat-VRS on ``GET /variants/{urn}`` and alongside the cross-layer +equivalence class on ``GET /alleles/{digest}`` — so they live here, shared by both serving views rather +than in either one's file. """ from typing import Optional diff --git a/src/mavedb/view_models/allele_detail.py b/src/mavedb/view_models/allele_detail.py new file mode 100644 index 000000000..67ec74bd3 --- /dev/null +++ b/src/mavedb/view_models/allele_detail.py @@ -0,0 +1,39 @@ +"""Response view models for the allele-detail view (``GET /alleles/{digest|CAID}``). + +The pydantic serialization boundary over the :mod:`lib.allele_detail` transit dataclasses. The +allele-grain sibling of ``view_models.variant_detail``. The ``alleles`` map reuses the shared +:class:`AlleleIdentity` (``view_models.allele_identity``) and the digest-keyed +:class:`AlleleAnnotations` block (``view_models.allele_annotation``) — both shared with the +variant view. +""" + +from typing import Any, Optional + +from mavedb.view_models.allele_annotation import AlleleAnnotations +from mavedb.view_models.allele_identity import AlleleIdentity +from mavedb.view_models.base.base import BaseModel + + +class AlleleDetail(BaseModel): + """The allele-detail envelope (``GET /alleles/{digest|CAID}``). + + Flat anchor-identity fields (``digest`` / ``level`` / ``hgvs`` / ``clingenAlleleId`` and the spec-pure + GA4GH ``vrs`` variation) plus the MaveDB layer riding alongside. This layer, keyed by VRS digest, + contains the ``alleles`` map: the full cross-layer equivalence class. Each entry is an ``AlleleIdentity``, + labelled relative to the focus (``isFocus`` marks the queried allele / the CAID's representations) and + the digest-keyed ``annotations`` map. The two maps share keys. Measurement-agnostic: no score, + classification, or version standing, and no re-anchored Cat-VRS (those belong to ``GET /variants/{urn}``). + Absent fields are omitted. + """ + + digest: str + level: Optional[str] = None + hgvs: Optional[str] = None + clingen_allele_id: Optional[str] = None + vrs: Optional[dict[str, Any]] = None + + alleles: dict[str, AlleleIdentity] = {} + annotations: dict[str, AlleleAnnotations] = {} + + class Config: + from_attributes = True diff --git a/src/mavedb/view_models/allele_identity.py b/src/mavedb/view_models/allele_identity.py new file mode 100644 index 000000000..b77743df5 --- /dev/null +++ b/src/mavedb/view_models/allele_identity.py @@ -0,0 +1,29 @@ +"""Response view model for the shared allele molecular-identity shape. + +The pydantic boundary over :class:`lib.allele_identity.AlleleIdentity` (``from_attributes`` coerces it; +aliases camelize per the shared base config). Shared by both serving views — the ``alleles`` map on +``GET /variants/{urn}`` and on ``GET /alleles/{digest}``. See the lib docstring for the focus-relative +semantics of each axis. +""" + +from typing import Optional + +from mavedb.view_models.base.base import BaseModel + + +class AlleleIdentity(BaseModel): + """One allele in a view's ``alleles`` map, keyed by VRS digest and labelled relative to the view's + focus allele. ``isFocus`` marks the anchor (measured allele / queried allele); ``relation`` and + ``derivation`` describe every other member's structural + provenance relationship to it, and are + absent on the focus itself.""" + + level: Optional[str] = None + hgvs: Optional[str] = None + clingen_allele_id: Optional[str] = None + is_focus: bool + relation: Optional[str] = None + derivation: Optional[str] = None + projection_of: Optional[str] = None + + class Config: + from_attributes = True diff --git a/src/mavedb/view_models/variant_detail.py b/src/mavedb/view_models/variant_detail.py index 31f5518aa..d9bc08344 100644 --- a/src/mavedb/view_models/variant_detail.py +++ b/src/mavedb/view_models/variant_detail.py @@ -10,6 +10,7 @@ from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.view_models.allele_annotation import AlleleAnnotations +from mavedb.view_models.allele_identity import AlleleIdentity from mavedb.view_models.base.base import BaseModel from mavedb.view_models.score_calibration import SavedFunctionalClassification @@ -30,34 +31,6 @@ class Config: from_attributes = True -class AlleleIdentity(BaseModel): - """The MaveDB molecular-identity facts for one of the variant's linked alleles. - - An entry of the ``alleles`` sidecar, keyed by VRS digest. ``level`` + ``hgvs`` (the reference-frame - HGVS) are what the UI labels the per-level annotation panel by — never the digest. - - Three independent axes: - - ``relation`` (Cat-VRS, structural): member→defining relation; ``null`` when it *is* the measured - allele, or when the allele is not a Cat-VRS member. - - ``derivation`` (provenance): ``authoritative`` (measured) / ``projection`` (deterministic, - precise, derived from the measured change) / ``candidate`` (protein-assay reverse-translation, - ambiguous) / ``convergent`` (a distinct, precise nucleotide change that converges on the measured - protein consequence). Orthogonal to ``relation``. Do not conflate them. - - ``projectionOf`` (provenance): the VRS digest of this allele's projection sibling (the paired c↔g member of - its projection pair group); ``null`` for the protein apex and pre-reverse-translation data. - """ - - level: Optional[str] = None - hgvs: Optional[str] = None - clingen_allele_id: Optional[str] = None - relation: Optional[str] = None - derivation: Optional[str] = None - projection_of: Optional[str] = None - - class Config: - from_attributes = True - - class VariantDetail(BaseModel): """The assayed variant-detail envelope (``GET /variants/{urn}``). diff --git a/tests/lib/test_allele_detail.py b/tests/lib/test_allele_detail.py new file mode 100644 index 000000000..75375b7a4 --- /dev/null +++ b/tests/lib/test_allele_detail.py @@ -0,0 +1,203 @@ +# ruff: noqa: E402 +"""Integration tests for the allele-detail assembly backing ``GET /alleles/{digest}`` (+ CAID). + +``get_allele_detail`` is the allele-grain sibling of ``get_variant_detail``: it anchors on an allele +(a digest, or a CAID's genomic+coding pair) and serves the **full cross-layer equivalence class** +(``get_allele_translations``), with each member labelled *relative to the focus* — faithful +(``projection``) vs. convergent cousin vs. reverse-translation ``candidate`` — plus the digest-keyed +annotations. These tests pin the labelling in both directions (nt focus / protein focus), the full-class +membership incl. cousins, the CAID focus set, the orphan fallback, and ``as_of``. +""" + +from datetime import datetime, timezone + +import pytest + +pytest.importorskip("psycopg2") + +from sqlalchemy import select + +from mavedb.lib.allele_detail import get_allele_detail +from mavedb.models.allele import Allele +from mavedb.models.variant import Variant +from tests.helpers.constants import TEST_MINIMAL_VARIANT +from tests.helpers.util.annotation import AlleleSpec, seed_mapping_record + +T0 = datetime(2020, 1, 1, tzinfo=timezone.utc) +T1 = datetime(2021, 1, 1, tzinfo=timezone.utc) +T2 = datetime(2022, 1, 1, tzinfo=timezone.utc) + +_VALID_DIGEST = "0123456789abcdefghijABCDEFGHIJ_-" + + +def _post_mapped() -> dict: + return { + "id": f"ga4gh:VA.{_VALID_DIGEST}", + "type": "Allele", + "state": {"type": "LiteralSequenceExpression", "sequence": "F"}, + "digest": _VALID_DIGEST, + "location": { + "id": f"ga4gh:SL.{_VALID_DIGEST}", + "end": 6, + "type": "SequenceLocation", + "start": 5, + "digest": _VALID_DIGEST, + "sequenceReference": { + "type": "SequenceReference", + "label": "NP_000000.0", + "refgetAccession": "SQ.0123456789abcdefghijABCDEFGHIJ_-", + }, + }, + } + + +def _variant(session, score_set, suffix): + variant = Variant(**TEST_MINIMAL_VARIANT, urn=f"{score_set.urn}#{suffix}", score_set_id=score_set.id) + session.add(variant) + session.commit() + return variant + + +def _allele(session, digest) -> Allele: + allele = session.scalar(select(Allele).where(Allele.vrs_digest == digest)) + assert allele is not None, f"allele {digest!r} not seeded" + return allele + + +def _coding_measured_specs(): + """A coding-measured record: authoritative cdna (+VEP+ClinGen), its genomic projection partner + (shared projection_group 0), the protein apex, and a synonymous cousin (own projection_group 1).""" + return [ + AlleleSpec( + digest="cdna-d", + level="cdna", + is_authoritative=True, + clingen_allele_id="CA1", + hgvs_c="NM_000546.6:c.1216G>A", + post_mapped=_post_mapped(), + vep_consequence="missense_variant", + projection_group=0, + ), + AlleleSpec( + digest="gen-d", + level="genomic", + clingen_allele_id="CA1", + hgvs_g="NC_000017.11:g.7676154C>T", + post_mapped=_post_mapped(), + projection_group=0, + ), + AlleleSpec(digest="prot-d", level="protein", hgvs_p="NP_000537.3:p.Ala406Thr", post_mapped=_post_mapped()), + AlleleSpec( + digest="cousin-d", + level="cdna", + hgvs_c="NM_000546.6:c.1218C>T", + post_mapped=_post_mapped(), + projection_group=1, + ), + ] + + +@pytest.mark.integration +def test_nt_focus_labels_the_full_class_relative_to_the_queried_allele(session, setup_lib_db_with_score_set): + """Fetching the measured coding allele: the whole class comes back, each member labelled relative to + it — its genomic is a faithful coordinate projection, the protein its consequence, the cousin convergent.""" + variant = _variant(session, setup_lib_db_with_score_set, 1) + seed_mapping_record(session, variant, alleles=_coding_measured_specs(), assay_level="cdna") + + detail = get_allele_detail(session, _allele(session, "cdna-d"), focus_digests={"cdna-d"}) + + assert detail.digest == "cdna-d" + assert detail.level == "cdna" + assert detail.vrs is not None and detail.vrs["type"] == "Allele" + assert set(detail.alleles) == {"cdna-d", "gen-d", "prot-d", "cousin-d"} # full class, cousin included + + focus = detail.alleles["cdna-d"] + assert focus.is_focus is True and focus.relation is None and focus.derivation is None + assert focus.projection_of == "gen-d" + + genomic = detail.alleles["gen-d"] + assert genomic.is_focus is False + assert genomic.relation == "coordinate_representation_of" + assert genomic.derivation == "projection" + assert genomic.projection_of == "cdna-d" + + protein = detail.alleles["prot-d"] + assert protein.relation == "translation_of" + assert protein.derivation == "projection" + + cousin = detail.alleles["cousin-d"] + assert cousin.relation == "co_encodes" + assert cousin.derivation == "convergent" # distinct change sharing the consequence, not the coordinate partner + + +@pytest.mark.integration +def test_protein_focus_labels_every_nucleotide_as_a_candidate(session, setup_lib_db_with_score_set): + """Fetching the protein apex: walking down is ambiguous, so every nt member is a reverse-translation + candidate (the 'less power without a measurement' case, still labelled precisely by direction).""" + variant = _variant(session, setup_lib_db_with_score_set, 1) + seed_mapping_record(session, variant, alleles=_coding_measured_specs(), assay_level="cdna") + + detail = get_allele_detail(session, _allele(session, "prot-d"), focus_digests={"prot-d"}) + + assert detail.alleles["prot-d"].is_focus is True + for nt in ("cdna-d", "gen-d", "cousin-d"): + assert detail.alleles[nt].relation == "encodes", nt + assert detail.alleles[nt].derivation == "candidate", nt + + +@pytest.mark.integration +def test_annotations_join_by_digest(session, setup_lib_db_with_score_set): + variant = _variant(session, setup_lib_db_with_score_set, 1) + seed_mapping_record(session, variant, alleles=_coding_measured_specs(), assay_level="cdna") + + detail = get_allele_detail(session, _allele(session, "cdna-d"), focus_digests={"cdna-d"}) + + assert set(detail.annotations) == set(detail.alleles) + assert detail.annotations["cdna-d"].vep is not None + assert detail.annotations["cdna-d"].vep.consequence == "missense_variant" + assert detail.annotations["gen-d"].vep is None # a member with no annotations still gets an empty block + + +@pytest.mark.integration +def test_caid_focus_flags_both_frames(session, setup_lib_db_with_score_set): + """A CAID names the nt change in both frames: fetched by CAID, the genomic + coding are both isFocus + (each other's coordinate partner), and only the true cousin is convergent.""" + variant = _variant(session, setup_lib_db_with_score_set, 1) + seed_mapping_record(session, variant, alleles=_coding_measured_specs(), assay_level="cdna") + + # The CAID resolves to the {cdna, genomic} pair (both carry clingen_allele_id="CA1"). + detail = get_allele_detail(session, _allele(session, "cdna-d"), focus_digests={"cdna-d", "gen-d"}) + + assert detail.alleles["cdna-d"].is_focus is True + assert detail.alleles["gen-d"].is_focus is True + assert detail.alleles["prot-d"].relation == "translation_of" + assert detail.alleles["cousin-d"].derivation == "convergent" + + +@pytest.mark.integration +def test_orphan_allele_falls_back_to_itself(session, setup_lib_db_with_score_set): + orphan = Allele(vrs_digest="orphan-d", level="cdna", hgvs_c="NM:c.9A>G", post_mapped=_post_mapped()) + session.add(orphan) + session.commit() + + detail = get_allele_detail(session, orphan, focus_digests={"orphan-d"}) + + assert set(detail.alleles) == {"orphan-d"} + assert detail.alleles["orphan-d"].is_focus is True + assert set(detail.annotations) == {"orphan-d"} + + +@pytest.mark.integration +def test_as_of_reconstructs_membership(session, setup_lib_db_with_score_set): + variant = _variant(session, setup_lib_db_with_score_set, 1) + seed_mapping_record(session, variant, alleles=_coding_measured_specs(), assay_level="cdna", valid_from=T1) + + anchor = _allele(session, "cdna-d") + + before = get_allele_detail(session, anchor, focus_digests={"cdna-d"}, as_of=T0) + assert set(before.alleles) == {"cdna-d"} # links not yet live → class collapses to the focus + assert before.annotations["cdna-d"].vep is None + + after = get_allele_detail(session, anchor, focus_digests={"cdna-d"}, as_of=T2) + assert set(after.alleles) == {"cdna-d", "gen-d", "prot-d", "cousin-d"} + assert after.annotations["cdna-d"].vep is not None diff --git a/tests/lib/test_variant_detail.py b/tests/lib/test_variant_detail.py index 4c8a3384c..8950dd6ef 100644 --- a/tests/lib/test_variant_detail.py +++ b/tests/lib/test_variant_detail.py @@ -185,10 +185,13 @@ def test_full_envelope_for_a_coding_assay(session, setup_lib_db_with_score_set): assert measured_identity.hgvs == "NM_000546.6:c.1216G>A" # coalesced from hgvs_c assert measured_identity.clingen_allele_id == "CA123" assert measured_identity.relation is None # the defining allele has no relation to itself - # The measured allele is authoritative and pairs with its genomic projection sibling. - assert measured_identity.derivation == "authoritative" + # The measured allele is the focus; derivation describes the *other* members relative to it, so the + # focus carries none. It pairs with its genomic projection sibling. + assert measured_identity.is_focus is True + assert measured_identity.derivation is None assert measured_identity.projection_of == "gen-digest" genomic_identity = detail.alleles["gen-digest"] + assert genomic_identity.is_focus is False # Nucleotide assay: the genomic sibling is a precise projection, paired back to the measured cdna. assert genomic_identity.level == "genomic" assert genomic_identity.derivation == "projection" @@ -245,11 +248,12 @@ def test_protein_assay_targets_the_protein_frame(session, setup_lib_db_with_scor assert detail.reference_hgvs == "NP_000537.3:p.Ala406Thr" assert detail.assay_level_digest == "prot-digest" assert detail.mode == "reverse_translation" - # Defining protein allele has no relation to itself, is authoritative, and is in no c/g pair. + # Defining protein allele is the focus (no relation/derivation to itself), and is in no c/g pair. prot = detail.alleles["prot-digest"] assert prot.level == "protein" assert prot.relation is None - assert prot.derivation == "authoritative" + assert prot.is_focus is True + assert prot.derivation is None assert prot.projection_of is None # The coding member `encodes` the protein (structural relation) but is an ambiguous `candidate` # (provenance) — the two axes disagree by design — and pairs to its genomic projection. @@ -265,9 +269,9 @@ def test_protein_assay_targets_the_protein_frame(session, setup_lib_db_with_scor @pytest.mark.integration def test_pre_reverse_translation_data_degrades_projection_of_to_null(session, setup_lib_db_with_score_set): - """Links written before reverse translation ran carry a NULL projection_group: derivation is still - computed (authoritative / projection from is_authoritative + assay level), but projection_of degrades - to null — nothing pairs until the data is re-processed.""" + """Links written before reverse translation ran carry a NULL projection_group: focus/derivation are + still computed (is_focus from is_authoritative; projection from the assay level), but projection_of + degrades to null — nothing pairs until the data is re-processed.""" score_set = setup_lib_db_with_score_set variant = _variant(session, score_set, 1, hgvs_nt="c.1216G>A") record = _record(session, variant, assay_level="cdna", hgvs_assay_level="NM_000546.6:c.1216G>A") @@ -278,7 +282,8 @@ def test_pre_reverse_translation_data_degrades_projection_of_to_null(session, se detail = get_variant_detail(session, variant) - assert detail.alleles["cdna-digest"].derivation == "authoritative" + assert detail.alleles["cdna-digest"].is_focus is True + assert detail.alleles["cdna-digest"].derivation is None assert detail.alleles["cdna-digest"].projection_of is None # no group -> no sibling assert detail.alleles["prot-digest"].derivation == "projection" assert detail.alleles["prot-digest"].projection_of is None diff --git a/tests/routers/test_allele.py b/tests/routers/test_allele.py new file mode 100644 index 000000000..7ce80c0c9 --- /dev/null +++ b/tests/routers/test_allele.py @@ -0,0 +1,224 @@ +# ruff: noqa: E402 +"""Router tests for the allele-detail endpoints (``GET /alleles/{digest}`` and ``?clingenAlleleId=``). + +Exercise the HTTP surface end to end: the envelope serializes and validates, the equivalence class is +labelled relative to the focus, CAID fetch focuses the nt-canonical change, ``as_of`` is echoed and +reconstructs membership, an unknown id is a plain 404 — and, unlike ``GET /variants/{urn}``, the +endpoint is **public with no privacy gate**: an allele reachable only through a private score set is +still served. +""" + +import pytest + +arq = pytest.importorskip("arq") +cdot = pytest.importorskip("cdot") +fastapi = pytest.importorskip("fastapi") + +from mavedb.models.score_set import ScoreSet as ScoreSetDbModel +from mavedb.view_models.allele_detail import AlleleDetail +from tests.helpers.dependency_overrider import DependencyOverrider +from tests.helpers.util.annotation import AlleleSpec, seed_mapping_record +from tests.helpers.util.experiment import create_experiment +from tests.helpers.util.score_set import create_seq_score_set_with_variants +from tests.helpers.util.user import change_ownership + +_VALID_DIGEST = "0123456789abcdefghijABCDEFGHIJ_-" + + +def _ir(tag: str) -> str: + """A valid GA4GH IR (``ga4gh:VA.`` + exactly 32 base64url chars), tagged for readability.""" + return f"ga4gh:VA.{tag.ljust(32, '0')}" + + +CDNA = _ir("cdna") +GEN = _ir("gen") +PROT = _ir("prot") +COUSIN = _ir("cousin") + + +def _post_mapped() -> dict: + return { + "id": f"ga4gh:VA.{_VALID_DIGEST}", + "type": "Allele", + "state": {"type": "LiteralSequenceExpression", "sequence": "F"}, + "digest": _VALID_DIGEST, + "location": { + "id": f"ga4gh:SL.{_VALID_DIGEST}", + "end": 6, + "type": "SequenceLocation", + "start": 5, + "digest": _VALID_DIGEST, + "sequenceReference": { + "type": "SequenceReference", + "label": "NP_000000.0", + "refgetAccession": "SQ.0123456789abcdefghijABCDEFGHIJ_-", + }, + }, + } + + +def _seed(session, variant_urn): + """Coding-measured record: authoritative cdna (+ClinGen CA1 +VEP), its genomic partner (also CA1), + the protein apex, and a synonymous cousin in its own projection group.""" + seed_mapping_record( + session, + variant_urn, + alleles=[ + AlleleSpec( + digest=CDNA, + level="cdna", + is_authoritative=True, + clingen_allele_id="CA1", + hgvs_c="NM_000546.6:c.1216G>A", + post_mapped=_post_mapped(), + vep_consequence="missense_variant", + projection_group=0, + ), + AlleleSpec( + digest=GEN, + level="genomic", + clingen_allele_id="CA1", + hgvs_g="NC_000017.11:g.7676154C>T", + post_mapped=_post_mapped(), + projection_group=0, + ), + AlleleSpec( + digest=PROT, + level="protein", + clingen_allele_id="PA1", + hgvs_p="NP_000537.3:p.Ala406Thr", + post_mapped=_post_mapped(), + ), + AlleleSpec( + digest=COUSIN, + level="cdna", + hgvs_c="NM_000546.6:c.1218C>T", + post_mapped=_post_mapped(), + projection_group=1, + ), + ], + assay_level="cdna", + ) + + +def test_get_allele_detail_envelope(client, session, data_provider, data_files, setup_router_db): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + _seed(session, f"{score_set['urn']}#1") + + response = client.get(f"/api/v1/alleles/{CDNA}") + + assert response.status_code == 200 + body = response.json() + AlleleDetail.model_validate(body) + + assert body["digest"] == CDNA + assert body["level"] == "cdna" + assert body["hgvs"] == "NM_000546.6:c.1216G>A" + assert body["clingenAlleleId"] == "CA1" + assert body["vrs"]["type"] == "Allele" + # Full equivalence class, keyed by digest, labelled relative to the queried allele (camelCase). + assert set(body["alleles"]) == {CDNA, GEN, PROT, COUSIN} + assert body["alleles"][CDNA]["isFocus"] is True + assert "derivation" not in body["alleles"][CDNA] # focus carries none; dropped by exclude_none + assert body["alleles"][GEN]["relation"] == "coordinate_representation_of" + assert body["alleles"][GEN]["derivation"] == "projection" + assert body["alleles"][PROT]["relation"] == "translation_of" + assert body["alleles"][COUSIN]["relation"] == "co_encodes" + assert body["alleles"][COUSIN]["derivation"] == "convergent" + # Annotations share keys with the class and join by digest. + assert set(body["annotations"]) == {CDNA, GEN, PROT, COUSIN} + assert body["annotations"][CDNA]["vep"]["consequence"] == "missense_variant" + # Measurement-agnostic: no variant-detail measurement fields. + assert "molecularRepresentation" not in body + assert "classifications" not in body + assert "isCurrent" not in body + assert response.headers["X-As-Of"] == "current" + + +def test_get_allele_detail_by_caid(client, session, data_provider, data_files, setup_router_db): + """CAID fetch on the overloaded `/alleles/{identifier}` path focuses the nt-canonical change (both + genomic + coding frames isFocus).""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + _seed(session, f"{score_set['urn']}#1") + + response = client.get("/api/v1/alleles/CA1") + + assert response.status_code == 200 + body = response.json() + AlleleDetail.model_validate(body) + assert body["clingenAlleleId"] == "CA1" + assert body["alleles"][CDNA]["isFocus"] is True + assert body["alleles"][GEN]["isFocus"] is True # the CAID's other frame + assert body["alleles"][COUSIN]["derivation"] == "convergent" + + +def test_get_allele_detail_by_paid(client, session, data_provider, data_files, setup_router_db): + """PAID fetch focuses the protein allele; its nucleotide equivalents surface as candidates — the + same `/alleles/{identifier}` path, no separate route or param.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + _seed(session, f"{score_set['urn']}#1") + + response = client.get("/api/v1/alleles/PA1") + + assert response.status_code == 200 + body = response.json() + AlleleDetail.model_validate(body) + assert body["clingenAlleleId"] == "PA1" + assert body["alleles"][PROT]["isFocus"] is True + # Walking down from the protein is ambiguous: every nucleotide member is a candidate. + for nt in (CDNA, GEN, COUSIN): + assert body["alleles"][nt]["derivation"] == "candidate", nt + + +def test_get_allele_detail_unknown_digest_is_404(client, setup_router_db): + response = client.get(f"/api/v1/alleles/{_ir('nonexistent')}") + assert response.status_code == 404 + + +def test_get_allele_detail_unknown_caid_is_404(client, setup_router_db): + response = client.get("/api/v1/alleles/CA999999") + assert response.status_code == 404 + + +def test_get_allele_detail_is_public_no_privacy_gate( + client, session, data_provider, data_files, setup_router_db, anonymous_app_overrides +): + """The decision: no privacy gate. An allele reachable only through a private score set is still + served to an anonymous caller — contrast ``GET /variants/{urn}``, which 404s the same variant.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + _seed(session, f"{score_set['urn']}#1") + change_ownership(session, score_set["urn"], ScoreSetDbModel) + + with DependencyOverrider(anonymous_app_overrides): + response = client.get(f"/api/v1/alleles/{CDNA}") + + assert response.status_code == 200 + assert response.json()["digest"] == CDNA + + +def test_get_allele_detail_echoes_as_of_header(client, session, data_provider, data_files, setup_router_db): + """``as_of`` is echoed and reconstructs membership: before the record was live the class collapses + to the focus (the allele row itself is content-addressed, so it still resolves).""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + _seed(session, f"{score_set['urn']}#1") + + response = client.get(f"/api/v1/alleles/{CDNA}", params={"as_of": "2020-01-01T00:00:00Z"}) + + assert response.status_code == 200 + assert response.headers["X-As-Of"] == "2020-01-01T00:00:00+00:00" + assert set(response.json()["alleles"]) == {CDNA} diff --git a/tests/routers/test_variant.py b/tests/routers/test_variant.py index dae7fd50e..a422bbe24 100644 --- a/tests/routers/test_variant.py +++ b/tests/routers/test_variant.py @@ -141,9 +141,13 @@ def test_get_variant_detail_envelope(client, session, data_provider, data_files, assert body["alleles"]["cdna-digest"]["hgvs"] == "NM_000546.6:c.1216G>A" assert body["alleles"]["cdna-digest"]["clingenAlleleId"] == "CA123" assert "relation" not in body["alleles"]["cdna-digest"] # null relation dropped by exclude_none - # Provenance axis (derivation) + the projection pairing (projectionOf), camelCased. - assert body["alleles"]["cdna-digest"]["derivation"] == "authoritative" + # The measured allele is the focus (isFocus); it carries no derivation (that axis describes the + # *other* members relative to it), and pairs with its genomic projection sibling (projectionOf). + assert body["alleles"]["cdna-digest"]["isFocus"] is True + assert "derivation" not in body["alleles"]["cdna-digest"] # null derivation dropped by exclude_none assert body["alleles"]["cdna-digest"]["projectionOf"] == "gen-digest" + assert body["alleles"]["gen-digest"]["isFocus"] is False + assert body["alleles"]["gen-digest"]["relation"] == "coordinate_representation_of" assert body["alleles"]["gen-digest"]["derivation"] == "projection" assert body["alleles"]["gen-digest"]["projectionOf"] == "cdna-digest" assert body["alleles"]["prot-digest"]["level"] == "protein" From 193c55f4dc99af580a7389328c32df2a65d6920a Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Mon, 20 Jul 2026 10:27:08 -0700 Subject: [PATCH 093/252] refactor(workflow): extract shared pipeline-kickoff helper from score_sets router score_sets.py had its own private `enqueue_pipeline_entrypoint` helper for building `Pipeline`/`JobRun` records and enqueuing the `start_pipeline` ARQ job, duplicated at both of its call sites. Move that logic into `mavedb.lib.workflow.kickoff.enqueue_pipeline_for_score_set` so the `run_pipeline` script and other future callers share the same enqueue contract instead of reimplementing it. - Restore the discard-pipeline-on-enqueue-failure safety net that the router's original helper had but the extracted version initially dropped, so a failed ARQ enqueue never leaves an orphaned pipeline behind in the database. - Add unit and integration tests for `enqueue_pipeline_for_score_set`, covering correlation-id precedence, param merging, ARQ dedup handling, and pipeline cleanup on enqueue failure. --- src/mavedb/lib/workflow/kickoff.py | 96 +++++++++++ src/mavedb/routers/score_sets.py | 61 ++----- tests/lib/workflow/test_kickoff.py | 258 +++++++++++++++++++++++++++++ tests/routers/test_score_set.py | 52 ++++-- 4 files changed, 406 insertions(+), 61 deletions(-) create mode 100644 src/mavedb/lib/workflow/kickoff.py create mode 100644 tests/lib/workflow/test_kickoff.py diff --git a/src/mavedb/lib/workflow/kickoff.py b/src/mavedb/lib/workflow/kickoff.py new file mode 100644 index 000000000..94feec0bf --- /dev/null +++ b/src/mavedb/lib/workflow/kickoff.py @@ -0,0 +1,96 @@ +"""Create and enqueue a named pipeline for a single score set. + +The one shared kickoff seam: build the ``Pipeline`` / ``JobRun`` / ``JobDependency`` records via +:class:`PipelineFactory`, then enqueue the ``start_pipeline`` entrypoint in ARQ. Both the ad-hoc +``run_pipeline`` script and the batch ``enrich_backfilled_score_sets`` driver go through here so the +enqueue contract (entrypoint id + :func:`arq_job_id`) lives in exactly one place. +""" + +from __future__ import annotations + +import datetime +import logging +from typing import Any, Optional + +from arq.connections import ArqRedis +from arq.jobs import Job +from sqlalchemy.orm import Session + +from mavedb.lib.logging.context import correlation_id_for_context, logging_context, save_to_logging_context +from mavedb.lib.workflow.pipeline_factory import PipelineFactory +from mavedb.models.pipeline import Pipeline +from mavedb.models.score_set import ScoreSet +from mavedb.models.user import User +from mavedb.worker.lib.managers.utils import arq_job_id + +logger = logging.getLogger(__name__) + + +async def enqueue_pipeline_for_score_set( + db: Session, + redis: ArqRedis, + *, + pipeline_name: str, + score_set: ScoreSet, + user: User, + extra_params: Optional[dict[str, Any]] = None, +) -> tuple[Pipeline, Optional[Job]]: + """Create the pipeline records for ``score_set`` and enqueue its ``start_pipeline`` entrypoint. + + Returns the created :class:`Pipeline` and the enqueued ARQ :class:`Job` (``None`` when ARQ + deduplicated an already-queued job with the same id). Raises ``KeyError`` for an unknown + ``pipeline_name`` and ``ValueError`` for an invalid pipeline definition, surfaced from + :meth:`PipelineFactory.create_pipeline`. If the ARQ enqueue itself raises, the just-created + pipeline records are discarded (via :meth:`PipelineFactory.discard_pipeline`) before the + exception is re-raised, so a failed enqueue never leaves an orphaned pipeline behind. + """ + correlation_id = ( + correlation_id_for_context() + or f"{pipeline_name}_{score_set.urn}_{user.id}_{datetime.datetime.now().isoformat()}" + ) + params: dict[str, Any] = { + "correlation_id": correlation_id, + "score_set_id": score_set.id, + "updater_id": user.id, + } + if extra_params: + params.update(extra_params) + + factory = PipelineFactory(session=db) + pipeline, entrypoint = factory.create_pipeline( + pipeline_name=pipeline_name, creating_user=user, pipeline_params=params + ) + job_id = arq_job_id(entrypoint) + + try: + job = await redis.enqueue_job(entrypoint.job_function, entrypoint.id, _job_id=job_id) + except Exception: + logger.error( + "Failed to enqueue pipeline %s for score set %s; discarding pipeline %s", + pipeline_name, + score_set.urn, + pipeline.id, + extra=logging_context(), + ) + factory.discard_pipeline(pipeline) + raise + + save_to_logging_context({"pipeline_id": pipeline.id, "job_id": job_id}) + + if not job: + logger.info( + "Pipeline %s for score set %s already enqueued (job id: %s)", + pipeline_name, + score_set.urn, + job_id, + extra=logging_context(), + ) + else: + logger.info( + "Enqueued pipeline %s for score set %s (job id: %s)", + pipeline_name, + score_set.urn, + job_id, + extra=logging_context(), + ) + return pipeline, job diff --git a/src/mavedb/routers/score_sets.py b/src/mavedb/routers/score_sets.py index 506ed0b5b..7baeb0893 100644 --- a/src/mavedb/routers/score_sets.py +++ b/src/mavedb/routers/score_sets.py @@ -44,11 +44,7 @@ find_or_create_publication_identifier, ) from mavedb.lib.logging import LoggedRoute -from mavedb.lib.logging.context import ( - correlation_id_for_context, - logging_context, - save_to_logging_context, -) +from mavedb.lib.logging.context import logging_context, save_to_logging_context from mavedb.lib.permissions import Action, assert_permission, has_permission from mavedb.lib.score_calibrations import create_score_calibration from mavedb.lib.score_set_variants import get_lean_score_set_variants @@ -75,14 +71,13 @@ generate_experiment_urn, generate_score_set_urn, ) -from mavedb.lib.workflow.pipeline_factory import PipelineFactory +from mavedb.lib.workflow.kickoff import enqueue_pipeline_for_score_set from mavedb.models.contributor import Contributor from mavedb.models.enums.processing_state import ProcessingState from mavedb.models.experiment import Experiment from mavedb.models.gnomad_variant import GnomADVariant from mavedb.models.license import License from mavedb.models.mapped_variant import MappedVariant -from mavedb.models.pipeline import Pipeline from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.score_set import ScoreSet from mavedb.models.target_accession import TargetAccession @@ -105,7 +100,6 @@ from mavedb.view_models.score_set_dataset_columns import DatasetColumnMetadata from mavedb.view_models.search import ScoreSetsSearch, ScoreSetsSearchFilterOptionsResponse, ScoreSetsSearchResponse from mavedb.view_models.target_gene import TargetGeneCreate -from mavedb.worker.lib.managers.utils import arq_job_id TAG_NAME = "Score Sets" logger = logging.getLogger(__name__) @@ -114,37 +108,6 @@ SCORE_SET_SEARCH_MAX_PUBLICATION_IDENTIFIERS = 40 -async def enqueue_pipeline_entrypoint( - *, - db: Session, - worker: ArqRedis, - pipeline_name: str, - creating_user: Any, - pipeline_params: dict[str, Any], -) -> None: - pipeline_factory = PipelineFactory(session=db) - pipeline: Optional[Pipeline] = None - - try: - pipeline, pipeline_entrypoint = pipeline_factory.create_pipeline( - pipeline_name=pipeline_name, - creating_user=creating_user, - pipeline_params=pipeline_params, - ) - job = await worker.enqueue_job( - pipeline_entrypoint.job_function, pipeline_entrypoint.id, _job_id=arq_job_id(pipeline_entrypoint) - ) - except Exception: - pipeline_factory.discard_pipeline(pipeline) - raise - - if job is None: - logger.info(msg=f"Pipeline entrypoint for {pipeline_name} has already been enqueued.", extra=logging_context()) - else: - save_to_logging_context({"worker_job_id": job.job_id}) - logger.info(msg=f"Enqueued pipeline entrypoint for {pipeline_name}.", extra=logging_context()) - - async def enqueue_variant_creation( *, item: ScoreSet, @@ -212,15 +175,13 @@ async def enqueue_variant_creation( ) try: - await enqueue_pipeline_entrypoint( + await enqueue_pipeline_for_score_set( db=db, - worker=worker, + redis=worker, pipeline_name="validate_map_annotate_score_set", - creating_user=user_data.user, - pipeline_params={ - "correlation_id": correlation_id_for_context(), - "score_set_id": item.id, - "updater_id": user_data.user.id, + score_set=item, + user=user_data.user, + extra_params={ "scores_file_key": scores_file_key, "counts_file_key": counts_file_key, "score_columns_metadata": item.dataset_columns.get("score_columns_metadata") @@ -2461,12 +2422,12 @@ async def publish_score_set( db.refresh(item) try: - await enqueue_pipeline_entrypoint( + await enqueue_pipeline_for_score_set( db=db, - worker=worker, + redis=worker, pipeline_name="publish_score_set", - creating_user=user_data.user, - pipeline_params={"correlation_id": correlation_id_for_context(), "score_set_id": item.id}, + score_set=item, + user=user_data.user, ) except Exception as exc: logger.warning( diff --git a/tests/lib/workflow/test_kickoff.py b/tests/lib/workflow/test_kickoff.py new file mode 100644 index 000000000..9b98eaf45 --- /dev/null +++ b/tests/lib/workflow/test_kickoff.py @@ -0,0 +1,258 @@ +# ruff: noqa: E402 +import pytest + +pytest.importorskip("arq") + +from unittest.mock import AsyncMock, Mock, patch + +from arq.jobs import Job as ArqJob +from sqlalchemy import select + +from mavedb.lib.workflow.kickoff import enqueue_pipeline_for_score_set +from mavedb.models.job_run import JobRun +from mavedb.models.pipeline import Pipeline +from mavedb.models.user import User +from tests.helpers.constants import TEST_USER + + +def _mock_entrypoint(*, id=42, job_function="start_pipeline", urn="urn:mavedb:job-fixed", retry_count=0): + return Mock(id=id, job_function=job_function, urn=urn, retry_count=retry_count) + + +@pytest.mark.unit +@pytest.mark.asyncio +class TestEnqueuePipelineForScoreSetUnit: + """Unit tests with PipelineFactory and ARQ mocked out.""" + + @pytest.fixture(autouse=True) + def setup(self): + self.db = Mock() + self.redis = AsyncMock() + self.score_set = Mock(id=99, urn="urn:mavedb:00000001-a-1") + self.user = Mock(id=5) + self.pipeline = Mock(spec=Pipeline) + self.entrypoint = _mock_entrypoint() + + factory_patch = patch("mavedb.lib.workflow.kickoff.PipelineFactory") + self.factory_cls = factory_patch.start() + self.factory = self.factory_cls.return_value + self.factory.create_pipeline.return_value = (self.pipeline, self.entrypoint) + yield + factory_patch.stop() + + async def _call(self, **kwargs): + return await enqueue_pipeline_for_score_set( + self.db, self.redis, pipeline_name="test_pipeline", score_set=self.score_set, user=self.user, **kwargs + ) + + async def test_returns_pipeline_and_job_when_enqueued(self): + job = Mock(spec=ArqJob) + self.redis.enqueue_job.return_value = job + + result = await self._call() + + assert result == (self.pipeline, job) + self.redis.enqueue_job.assert_awaited_once_with( + self.entrypoint.job_function, + self.entrypoint.id, + _job_id=f"{self.entrypoint.urn}#{self.entrypoint.retry_count}", + ) + + async def test_returns_none_job_when_arq_deduplicates(self): + self.redis.enqueue_job.return_value = None + + result = await self._call() + + assert result == (self.pipeline, None) + + async def test_propagates_key_error_for_unknown_pipeline_name(self): + self.factory.create_pipeline.side_effect = KeyError("unknown_pipeline") + + with pytest.raises(KeyError): + await self._call() + + async def test_discards_pipeline_and_reraises_when_enqueue_raises(self): + self.redis.enqueue_job.side_effect = ConnectionError("redis is unreachable") + + with pytest.raises(ConnectionError): + await self._call() + + self.factory.discard_pipeline.assert_called_once_with(self.pipeline) + + async def test_does_not_discard_pipeline_when_enqueue_succeeds(self): + self.redis.enqueue_job.return_value = Mock(spec=ArqJob) + + await self._call() + + self.factory.discard_pipeline.assert_not_called() + + async def test_does_not_discard_pipeline_when_arq_deduplicates(self): + self.redis.enqueue_job.return_value = None + + await self._call() + + self.factory.discard_pipeline.assert_not_called() + + async def test_default_correlation_id_built_from_pipeline_name_score_set_and_user(self): + with patch("mavedb.lib.workflow.kickoff.correlation_id_for_context", return_value=None): + await self._call() + + params = self.factory.create_pipeline.call_args.kwargs["pipeline_params"] + assert params["correlation_id"].startswith(f"test_pipeline_{self.score_set.urn}_{self.user.id}_") + + async def test_correlation_id_from_context_used_when_present(self): + with patch("mavedb.lib.workflow.kickoff.correlation_id_for_context", return_value="context-correlation-id"): + await self._call() + + params = self.factory.create_pipeline.call_args.kwargs["pipeline_params"] + assert params["correlation_id"] == "context-correlation-id" + + async def test_extra_params_correlation_id_overrides_context(self): + with patch("mavedb.lib.workflow.kickoff.correlation_id_for_context", return_value="context-correlation-id"): + await self._call(extra_params={"correlation_id": "explicit-correlation-id"}) + + params = self.factory.create_pipeline.call_args.kwargs["pipeline_params"] + assert params["correlation_id"] == "explicit-correlation-id" + + async def test_extra_params_merged_alongside_base_params(self): + await self._call(extra_params={"some_extra_param": "some_value"}) + + params = self.factory.create_pipeline.call_args.kwargs["pipeline_params"] + assert params["some_extra_param"] == "some_value" + assert params["score_set_id"] == self.score_set.id + assert params["updater_id"] == self.user.id + + async def test_create_pipeline_called_with_pipeline_name_and_user(self): + await self._call() + + call_kwargs = self.factory.create_pipeline.call_args.kwargs + assert call_kwargs["pipeline_name"] == "test_pipeline" + assert call_kwargs["creating_user"] is self.user + + +@pytest.mark.integration +@pytest.mark.asyncio +class TestEnqueuePipelineForScoreSetIntegration: + """Integration tests exercising the real PipelineFactory and a fakeredis-backed ARQ instance.""" + + @pytest.fixture(autouse=True) + def setup(self, session, setup_lib_db_with_score_set): + self.score_set = setup_lib_db_with_score_set + self.user = session.query(User).filter(User.username == TEST_USER["username"]).first() + + async def test_creates_pipeline_and_job_run_records_and_enqueues_job( + self, session, arq_redis, with_test_pipeline_definition_ctx, sample_independent_pipeline_definition + ): + pipeline, job = await enqueue_pipeline_for_score_set( + session, + arq_redis, + pipeline_name=sample_independent_pipeline_definition["name"], + score_set=self.score_set, + user=self.user, + extra_params={"required_param": "some_value"}, + ) + + assert job is not None + assert session.execute(select(Pipeline).where(Pipeline.id == pipeline.id)).scalar_one() is pipeline + + job_run = session.execute( + select(JobRun).where(JobRun.pipeline_id == pipeline.id, JobRun.job_function == "process_data") + ).scalar_one() + assert job_run.job_params["required_param"] == "some_value" + + async def test_correlation_id_defaults_to_generated_value_without_request_context( + self, session, arq_redis, with_test_pipeline_definition_ctx, sample_independent_pipeline_definition + ): + pipeline, _ = await enqueue_pipeline_for_score_set( + session, + arq_redis, + pipeline_name=sample_independent_pipeline_definition["name"], + score_set=self.score_set, + user=self.user, + extra_params={"required_param": "some_value"}, + ) + + assert pipeline.correlation_id.startswith( + f"{sample_independent_pipeline_definition['name']}_{self.score_set.urn}_{self.user.id}_" + ) + + async def test_independent_calls_create_independent_pipelines_and_jobs( + self, session, arq_redis, with_test_pipeline_definition_ctx, sample_independent_pipeline_definition + ): + pipeline_one, job_one = await enqueue_pipeline_for_score_set( + session, + arq_redis, + pipeline_name=sample_independent_pipeline_definition["name"], + score_set=self.score_set, + user=self.user, + extra_params={"required_param": "some_value"}, + ) + pipeline_two, job_two = await enqueue_pipeline_for_score_set( + session, + arq_redis, + pipeline_name=sample_independent_pipeline_definition["name"], + score_set=self.score_set, + user=self.user, + extra_params={"required_param": "some_value"}, + ) + + assert pipeline_one.id != pipeline_two.id + assert job_one is not None + assert job_two is not None + + async def test_returns_none_job_when_arq_job_id_collides( + self, session, arq_redis, with_test_pipeline_definition_ctx, sample_independent_pipeline_definition + ): + with patch("mavedb.lib.workflow.kickoff.arq_job_id", return_value="fixed-dedupe-id"): + pipeline_one, job_one = await enqueue_pipeline_for_score_set( + session, + arq_redis, + pipeline_name=sample_independent_pipeline_definition["name"], + score_set=self.score_set, + user=self.user, + extra_params={"required_param": "some_value"}, + ) + pipeline_two, job_two = await enqueue_pipeline_for_score_set( + session, + arq_redis, + pipeline_name=sample_independent_pipeline_definition["name"], + score_set=self.score_set, + user=self.user, + extra_params={"required_param": "some_value"}, + ) + + # ARQ deduplicates the second enqueue on the collided job id, but pipeline records are + # still created for both calls -- kickoff does not roll back on dedup. + assert job_one is not None + assert job_two is None + assert pipeline_one.id != pipeline_two.id + + async def test_unknown_pipeline_name_raises_key_error(self, session, arq_redis): + with pytest.raises(KeyError): + await enqueue_pipeline_for_score_set( + session, + arq_redis, + pipeline_name="does_not_exist_pipeline", + score_set=self.score_set, + user=self.user, + ) + + async def test_discards_pipeline_records_when_enqueue_raises( + self, session, arq_redis, with_test_pipeline_definition_ctx, sample_independent_pipeline_definition + ): + with ( + patch.object(arq_redis, "enqueue_job", AsyncMock(side_effect=ConnectionError("redis is unreachable"))), + pytest.raises(ConnectionError), + ): + await enqueue_pipeline_for_score_set( + session, + arq_redis, + pipeline_name=sample_independent_pipeline_definition["name"], + score_set=self.score_set, + user=self.user, + extra_params={"required_param": "some_value"}, + ) + + # The pipeline (and its job runs) created before the failed enqueue must not be left behind. + assert session.execute(select(Pipeline)).scalars().all() == [] + assert session.execute(select(JobRun)).scalars().all() == [] diff --git a/tests/routers/test_score_set.py b/tests/routers/test_score_set.py index 11a07bf51..77150ba4c 100644 --- a/tests/routers/test_score_set.py +++ b/tests/routers/test_score_set.py @@ -19,12 +19,12 @@ from mavedb.lib.exceptions import NonexistentOrcidUserError from mavedb.lib.validation.urn_re import MAVEDB_EXPERIMENT_URN_RE, MAVEDB_SCORE_SET_URN_RE, MAVEDB_TMP_URN_RE +from mavedb.models.clinvar_allele_link import ClinvarAlleleLink from mavedb.models.enums.processing_state import ProcessingState from mavedb.models.enums.target_category import TargetCategory from mavedb.models.experiment import Experiment as ExperimentDbModel from mavedb.models.job_run import JobRun from mavedb.models.pipeline import Pipeline -from mavedb.models.clinvar_allele_link import ClinvarAlleleLink from mavedb.models.score_set import ScoreSet as ScoreSetDbModel from mavedb.models.variant import Variant as VariantDbModel from mavedb.view_models.lean_variant import LeanVariant @@ -64,6 +64,7 @@ VALID_CLINGEN_CA_ID, ) from tests.helpers.dependency_overrider import DependencyOverrider +from tests.helpers.util.annotation import AlleleSpec, seed_mapping_record from tests.helpers.util.common import ( deepcamelize, parse_ndjson_response, @@ -76,7 +77,6 @@ create_publish_and_promote_score_calibration, create_test_score_calibration_in_score_set_via_client, ) -from tests.helpers.util.annotation import AlleleSpec, seed_mapping_record from tests.helpers.util.score_set import ( create_seq_score_set, create_seq_score_set_with_mapped_variants, @@ -1537,7 +1537,7 @@ def test_upload_score_set_variant_data_deletes_s3_files_when_pipeline_creation_f with ( open(scores_csv_path, "rb") as scores_file, - patch("mavedb.routers.score_sets.PipelineFactory.create_pipeline", side_effect=Exception("pipeline failure")), + patch("mavedb.lib.workflow.kickoff.PipelineFactory.create_pipeline", side_effect=Exception("pipeline failure")), patch.object(mock_s3_client, "upload_fileobj", return_value=None), ): response = client.post( @@ -2004,7 +2004,10 @@ def test_multiple_score_set_meta_analysis_single_experiment( published_score_set_1_refresh = (client.get(f"/api/v1/score-sets/{published_score_set_1['urn']}")).json() assert meta_score_set["metaAnalyzesScoreSetUrns"] == sorted( - [published_score_set_1["urn"], published_score_set_2["urn"]] + [ + published_score_set_1["urn"], + published_score_set_2["urn"], + ] ) assert published_score_set_1_refresh["metaAnalyzedByScoreSetUrns"] == [meta_score_set["urn"]] @@ -2044,7 +2047,10 @@ def test_multiple_score_set_meta_analysis_multiple_experiment_sets( ) published_score_set_1_refresh = (client.get(f"/api/v1/score-sets/{published_score_set_1['urn']}")).json() assert meta_score_set["metaAnalyzesScoreSetUrns"] == sorted( - [published_score_set_1["urn"], published_score_set_2["urn"]] + [ + published_score_set_1["urn"], + published_score_set_2["urn"], + ] ) assert published_score_set_1_refresh["metaAnalyzedByScoreSetUrns"] == [meta_score_set["urn"]] @@ -2086,7 +2092,10 @@ def test_multiple_score_set_meta_analysis_multiple_experiments( ) published_score_set_1_refresh = (client.get(f"/api/v1/score-sets/{published_score_set_1['urn']}")).json() assert meta_score_set["metaAnalyzesScoreSetUrns"] == sorted( - [published_score_set_1["urn"], published_score_set_2["urn"]] + [ + published_score_set_1["urn"], + published_score_set_2["urn"], + ] ) assert published_score_set_1_refresh["metaAnalyzedByScoreSetUrns"] == [meta_score_set["urn"]] @@ -2142,7 +2151,10 @@ def test_multiple_score_set_meta_analysis_multiple_experiment_sets_different_sco published_score_set_1_1_refresh = (client.get(f"/api/v1/score-sets/{published_score_set_1_1['urn']}")).json() assert meta_score_set_1["metaAnalyzesScoreSetUrns"] == sorted( - [published_score_set_1_1["urn"], published_score_set_1_2["urn"]] + [ + published_score_set_1_1["urn"], + published_score_set_1_2["urn"], + ] ) assert published_score_set_1_1_refresh["metaAnalyzedByScoreSetUrns"] == [meta_score_set_1["urn"]] @@ -2159,7 +2171,10 @@ def test_multiple_score_set_meta_analysis_multiple_experiment_sets_different_sco ) published_score_set_2_1_refresh = (client.get(f"/api/v1/score-sets/{published_score_set_2_1['urn']}")).json() assert meta_score_set_2["metaAnalyzesScoreSetUrns"] == sorted( - [published_score_set_2_1["urn"], published_score_set_2_2["urn"]] + [ + published_score_set_2_1["urn"], + published_score_set_2_2["urn"], + ] ) assert published_score_set_2_1_refresh["metaAnalyzedByScoreSetUrns"] == [meta_score_set_2["urn"]] @@ -2281,7 +2296,10 @@ def test_multiple_score_set_meta_analysis_single_experiment_with_different_creat published_score_set_1_refresh = (client.get(f"/api/v1/score-sets/{published_score_set_1['urn']}")).json() assert meta_score_set["metaAnalyzesScoreSetUrns"] == sorted( - [published_score_set_1["urn"], published_score_set_2["urn"]] + [ + published_score_set_1["urn"], + published_score_set_2["urn"], + ] ) assert published_score_set_1_refresh["metaAnalyzedByScoreSetUrns"] == [meta_score_set["urn"]] @@ -2323,7 +2341,10 @@ def test_multiple_score_set_meta_analysis_multiple_experiment_sets_with_differen published_score_set_1_refresh = (client.get(f"/api/v1/score-sets/{published_score_set_1['urn']}")).json() assert meta_score_set["metaAnalyzesScoreSetUrns"] == sorted( - [published_score_set_1["urn"], published_score_set_2["urn"]] + [ + published_score_set_1["urn"], + published_score_set_2["urn"], + ] ) assert published_score_set_1_refresh["metaAnalyzedByScoreSetUrns"] == [meta_score_set["urn"]] @@ -3532,7 +3553,16 @@ def test_download_scores_and_counts_file(session, data_provider, client, setup_r download_scores_and_counts_csv = download_scores_and_counts_csv_response.text reader = csv.DictReader(StringIO(download_scores_and_counts_csv)) assert sorted(reader.fieldnames) == sorted( - ["accession", "hgvs_nt", "hgvs_pro", "scores.score", "scores.s_0", "scores.s_1", "counts.c_0", "counts.c_1"] + [ + "accession", + "hgvs_nt", + "hgvs_pro", + "scores.score", + "scores.s_0", + "scores.s_1", + "counts.c_0", + "counts.c_1", + ] ) From a71226241ec87c7f2b0d3d2d0cbbfc9506178106 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Mon, 20 Jul 2026 10:44:50 -0700 Subject: [PATCH 094/252] fixup ~2 --- src/mavedb/lib/allele_detail.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mavedb/lib/allele_detail.py b/src/mavedb/lib/allele_detail.py index 14603f509..b81b0db91 100644 --- a/src/mavedb/lib/allele_detail.py +++ b/src/mavedb/lib/allele_detail.py @@ -1,4 +1,4 @@ -"""The allele-detail view backing ``GET /alleles/{digest:CAID}``. +"""The allele-detail view backing ``GET /alleles/{digest|CAID}``. The allele-grain sibling of :mod:`lib.variant_detail`. Where the variant view anchors on a *measurement*, this anchors on an *allele* (a VRS digest, or a CAID naming the nt-canonical change) From c87b72fa473d28445da018031e7b6a99344c5e9e Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Mon, 20 Jul 2026 11:48:30 -0700 Subject: [PATCH 095/252] feat(scripts): add batch driver for score-set mapping/enrichment pipelines - Add run_score_set_pipelines.py, which selects unmapped or unenriched score sets and enqueues the appropriate pipeline for each, ordered by gene to maximize ClinGen CAR cache reuse - Refactor run_pipeline.py to share pipeline creation/enqueueing logic via the new enqueue_pipeline_for_score_set helper instead of duplicating PipelineFactory/redis wiring inline --- src/mavedb/scripts/run_pipeline.py | 41 +-- src/mavedb/scripts/run_score_set_pipelines.py | 276 ++++++++++++++++++ 2 files changed, 288 insertions(+), 29 deletions(-) create mode 100644 src/mavedb/scripts/run_score_set_pipelines.py diff --git a/src/mavedb/scripts/run_pipeline.py b/src/mavedb/scripts/run_pipeline.py index c947fe122..658037fbb 100644 --- a/src/mavedb/scripts/run_pipeline.py +++ b/src/mavedb/scripts/run_pipeline.py @@ -11,7 +11,6 @@ poetry run python -m mavedb.scripts.run_pipeline --list """ -import datetime import logging import sys @@ -21,10 +20,9 @@ from mavedb.db.session import SessionLocal from mavedb.lib.workflow.definitions import PIPELINE_DEFINITIONS -from mavedb.lib.workflow.pipeline_factory import PipelineFactory +from mavedb.lib.workflow.kickoff import enqueue_pipeline_for_score_set from mavedb.models.score_set import ScoreSet from mavedb.models.user import User -from mavedb.worker.lib.managers.utils import arq_job_id from mavedb.worker.settings import RedisWorkerSettings logger = logging.getLogger(__name__) @@ -108,40 +106,25 @@ async def main( click.echo(f"User not found: {resolved_updater_id}", err=True) sys.exit(1) - correlation_id = f"{pipeline_name}_{score_set.urn}_{user.id}_{datetime.datetime.now().isoformat()}" - pipeline_params: dict = { - "correlation_id": correlation_id, - "score_set_id": score_set.id, - "updater_id": user.id, - } - for key, value in extra_params: - pipeline_params[key] = value - - try: - pipeline_factory = PipelineFactory(session=db) - pipeline, pipeline_entrypoint = pipeline_factory.create_pipeline( - pipeline_name=pipeline_name, - creating_user=user, - pipeline_params=pipeline_params, - ) - except (KeyError, ValueError) as e: - click.echo(f"Failed to create pipeline: {e}", err=True) - sys.exit(1) - - click.echo(f"Created pipeline '{pipeline_name}' (id={pipeline.id}, correlation_id={correlation_id})") - # Connect to Redis and enqueue redis = await create_pool(RedisWorkerSettings) try: - job = await redis.enqueue_job( - pipeline_entrypoint.job_function, - pipeline_entrypoint.id, - _job_id=arq_job_id(pipeline_entrypoint), + pipeline, job = await enqueue_pipeline_for_score_set( + db, + redis, + pipeline_name=pipeline_name, + score_set=score_set, + user=user, + extra_params={key: value for key, value in extra_params}, ) + click.echo(f"Created pipeline '{pipeline_name}' (id={pipeline.id}, correlation_id={pipeline.correlation_id})") if job: click.echo(f"Enqueued start_pipeline job: {job.job_id}. Pipeline will execute asynchronously.") else: click.echo("Job was already enqueued (duplicate).", err=True) + except (KeyError, ValueError) as e: + click.echo(f"Failed to create pipeline: {e}", err=True) + sys.exit(1) finally: await redis.aclose() db.close() diff --git a/src/mavedb/scripts/run_score_set_pipelines.py b/src/mavedb/scripts/run_score_set_pipelines.py new file mode 100644 index 000000000..169f17031 --- /dev/null +++ b/src/mavedb/scripts/run_score_set_pipelines.py @@ -0,0 +1,276 @@ +"""Drive a score-set pipeline (mapping or annotation enrichment) over many score sets at once. + +Two selection modes: + +* ``--select unmapped`` (default) → runs ``map_annotate_score_set`` over score sets that still need + mapping. "Needs mapping" = a variant lacking a **live** ``MappingRecord`` dated on/after + ``--remap-before`` (default 2026-05-08, significant mapping-capability upgrade). This catches + never-mapped, partially-mapped, and *stale-but-complete* score sets — including reshape-backfilled + ones, whose records carry the old ``mapped_date`` and so need a real re-map under the upgraded + pipeline. +* ``--select unenriched`` → runs ``annotate_score_set`` over mapped score sets that have not yet had + the reverse-translation fan-out (no derived, non-authoritative live allele link). This is the + post-backfill enrichment half. + +**Throughput — gene-grouped ordering.** Score sets whose variants resolve to the same CAIDs/PAIDs +cache the expensive ClinGen CAR resolution (24h TTL) and reuse already-deduplicated alleles. So by +default the run is ordered by a normalized gene key — ``coalesce(mapped_hgnc_name, +target_accession.gene, target_gene.name)``, first whitespace token uppercased (``JAG1 Exon 1-7`` → +``JAG1``) — clustering same-gene score sets consecutively so the second-onward sets in a gene group +hit a warm cache and existing alleles. (The resolved ``mapped_hgnc_name`` is null pre-mapping, so the +unmapped mode falls back to the accession gene / submitter name.) ``--no-group-by-gene`` sorts by id. + +Enrichment is non-blocking; both pipelines hit external services, so ``--delay-seconds`` paces starts +and ``--limit`` bounds a batch. Idempotent: re-runs skip score sets already in the target end state +(``--force`` overrides). + +Usage:: + + poetry run python -m mavedb.scripts.run_score_set_pipelines --dry-run + poetry run python -m mavedb.scripts.run_score_set_pipelines --limit 50 --delay-seconds 3 + poetry run python -m mavedb.scripts.run_score_set_pipelines --select unenriched --limit 50 + poetry run python -m mavedb.scripts.run_score_set_pipelines --remap-before 2026-05-08 +""" + +from __future__ import annotations + +import asyncio +import datetime +import logging +from typing import Optional + +import asyncclick as click +from arq import create_pool +from sqlalchemy import and_, not_, select +from sqlalchemy.orm import Session, selectinload + +from mavedb.db.session import SessionLocal +from mavedb.lib.workflow.kickoff import enqueue_pipeline_for_score_set +from mavedb.models.enums.processing_state import ProcessingState +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.score_set import ScoreSet +from mavedb.models.target_gene import TargetGene +from mavedb.models.user import User +from mavedb.models.variant import Variant +from mavedb.worker.settings import RedisWorkerSettings + +logger = logging.getLogger(__name__) + +# The mapping-capability upgrade: score sets last mapped before this need a re-map, even if their +# mapping_state is "complete". +DEFAULT_REMAP_BEFORE = datetime.date(2026, 5, 8) + +# Default pipeline per selection mode. +PIPELINE_FOR_SELECT = { + "unmapped": "map_annotate_score_set", + "unenriched": "annotate_score_set", +} + + +def _normalize_gene(raw: str) -> str: + """Gene key for cache-locality ordering: first whitespace token, uppercased. + + ``"JAG1 Exon 1-7"`` → ``"JAG1"``, ``" brca1 "`` → ``"BRCA1"``. Gene symbols are one token, so the + first word is the gene; upper-casing folds submitter case differences together. + """ + tokens = raw.strip().split() + return tokens[0].upper().strip(".,;:") if tokens else "" + + +def _grouping_key(score_set: ScoreSet) -> str: + """Normalized gene key for a score set — the min across its target genes for determinism. + + Prefers the resolved gene (``mapped_hgnc_name``, populated only post-mapping), then the accession's + gene, then the submitter target name. + """ + keys = [] + for target_gene in score_set.target_genes: + accession_gene = target_gene.target_accession.gene if target_gene.target_accession else None + raw = target_gene.mapped_hgnc_name or accession_gene or target_gene.name + if raw: + keys.append(_normalize_gene(raw)) + return min(keys) if keys else "" + + +def _needs_mapping(remap_before: datetime.date): + """Predicate: the score set has a variant lacking a live MappingRecord dated on/after the cutoff.""" + fresh_record = select(MappingRecord.id).where( + MappingRecord.variant_id == Variant.id, + MappingRecord.valid_to.is_(None), + MappingRecord.mapped_date >= remap_before, + ) + variant_needing_mapping = select(Variant.id).where(Variant.score_set_id == ScoreSet.id, ~fresh_record.exists()) + return variant_needing_mapping.exists() + + +def _needs_enrichment(): + """Predicate: the score set is mapped (a live MappingRecord) but has no reverse-translation-derived + (non-authoritative) live allele link yet.""" + has_live_record = ( + select(MappingRecord.id) + .join(Variant, Variant.id == MappingRecord.variant_id) + .where(Variant.score_set_id == ScoreSet.id, MappingRecord.valid_to.is_(None)) + ) + has_derived_link = ( + select(MappingRecordAllele.id) + .join(MappingRecord, MappingRecord.id == MappingRecordAllele.mapping_record_id) + .join(Variant, Variant.id == MappingRecord.variant_id) + .where( + Variant.score_set_id == ScoreSet.id, + MappingRecord.valid_to.is_(None), + MappingRecordAllele.valid_to.is_(None), + MappingRecordAllele.is_authoritative.is_(False), + ) + ) + return and_(has_live_record.exists(), not_(has_derived_link.exists())) + + +def select_score_sets( + db: Session, + *, + phase: str = "unmapped", + remap_before: datetime.date = DEFAULT_REMAP_BEFORE, + only_published: bool = True, + force: bool = False, + score_set_urn: Optional[str] = None, + group_by_gene: bool = True, + limit: Optional[int] = None, +) -> list[ScoreSet]: + """Return score sets to run, gene-grouped by default. + + ``phase`` picks the eligibility predicate (``"unmapped"`` / ``"unenriched"``); ``force`` bypasses + it (every processed score set qualifies). Only variant-processed score sets are considered (the + pipelines assume variants exist), published unless ``only_published=False``. Ordering: by the + normalized gene key so same-gene sets cluster (``group_by_gene``), else by id; ``limit`` applies + after ordering so whole gene groups are taken first. + """ + query = ( + select(ScoreSet) + .options(selectinload(ScoreSet.target_genes).selectinload(TargetGene.target_accession)) + .where(ScoreSet.processing_state == ProcessingState.success) + ) + if only_published: + query = query.where(ScoreSet.private.is_(False), ScoreSet.published_date.isnot(None)) + if score_set_urn is not None: + query = query.where(ScoreSet.urn == score_set_urn) + if not force: + query = query.where(_needs_mapping(remap_before) if phase == "unmapped" else _needs_enrichment()) + + score_sets = list(db.scalars(query).unique().all()) + score_sets.sort(key=(lambda ss: (_grouping_key(ss), ss.id)) if group_by_gene else (lambda ss: ss.id)) + return score_sets[:limit] if limit is not None else score_sets + + +def _resolve_user(db: Session, score_set: ScoreSet, updater_id: Optional[int]) -> Optional[User]: + """The user to attribute the pipeline to: explicit override, else the score set's modifier/creator.""" + resolved_id = updater_id or score_set.modified_by_id or score_set.created_by_id + if resolved_id is None: + return None + return db.scalars(select(User).where(User.id == resolved_id)).one_or_none() + + +@click.command() +@click.option( + "--select", + "phase", + type=click.Choice(["unmapped", "unenriched"]), + default="unmapped", + show_default=True, + help="Which score sets to run: those needing mapping, or those mapped-but-not-enriched.", +) +@click.option("--pipeline", "pipeline_name", default=None, help="Override the per-mode default pipeline.") +@click.option( + "--remap-before", + "remap_before", + type=click.DateTime(formats=["%Y-%m-%d"]), + default=DEFAULT_REMAP_BEFORE.isoformat(), + show_default=True, + help="Re-map score sets last mapped before this date (unmapped mode only).", +) +@click.option("--score-set-urn", "score_set_urn", default=None, help="Run only this score set.") +@click.option("--updater-id", "updater_id", type=int, default=None, help="User to attribute pipelines to.") +@click.option("--limit", type=int, default=None, help="Max score sets to enqueue this run.") +@click.option( + "--delay-seconds", "delay_seconds", type=float, default=0.0, show_default=True, help="Pause between starts." +) +@click.option("--no-group-by-gene", "group_by_gene", flag_value=False, default=True, help="Sort by id instead of gene.") +@click.option("--force", is_flag=True, help="Ignore the eligibility check (include already-done score sets).") +@click.option("--include-unpublished", is_flag=True, help="Also include private/unpublished score sets.") +@click.option("--dry-run", is_flag=True, help="List the selected score sets without enqueuing anything.") +async def main( + phase: str, + pipeline_name: Optional[str], + remap_before: datetime.datetime, + score_set_urn: Optional[str], + updater_id: Optional[int], + limit: Optional[int], + delay_seconds: float, + group_by_gene: bool, + force: bool, + include_unpublished: bool, + dry_run: bool, +) -> None: + """Run a score-set pipeline over many score sets, gene-grouped for cache reuse.""" + pipeline = pipeline_name or PIPELINE_FOR_SELECT[phase] + db = SessionLocal() + try: + score_sets = select_score_sets( + db, + phase=phase, + remap_before=remap_before.date(), + only_published=not include_unpublished, + force=force, + score_set_urn=score_set_urn, + group_by_gene=group_by_gene, + limit=limit, + ) + click.echo(f"{len(score_sets)} score set(s) selected ({phase}) for pipeline '{pipeline}'.") + + if dry_run: + for score_set in score_sets: + click.echo(f" {_grouping_key(score_set):<12} {score_set.urn}") + click.echo("Dry run — nothing enqueued.") + return + + if not score_sets: + return + + redis = await create_pool(RedisWorkerSettings) + enqueued = 0 + try: + for index, score_set in enumerate(score_sets): + user = _resolve_user(db, score_set, updater_id) + if user is None: + click.echo(f" SKIP {score_set.urn}: no updater found (pass --updater-id).", err=True) + continue + + try: + pipeline_run, job = await enqueue_pipeline_for_score_set( + db, redis, pipeline_name=pipeline, score_set=score_set, user=user + ) + except (KeyError, ValueError) as exc: + click.echo(f" FAIL {score_set.urn}: {exc}", err=True) + continue + + enqueued += 1 + disposition = "queued" if job else "duplicate" + click.echo( + f" [{enqueued}/{len(score_sets)}] {score_set.urn} → pipeline {pipeline_run.id} ({disposition})" + ) + + if delay_seconds and index < len(score_sets) - 1: + await asyncio.sleep(delay_seconds) + + finally: + await redis.aclose() + + click.echo(f"Enqueued {enqueued} pipeline(s).") + + finally: + db.close() + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + main() From 4b640e12a91948868f965b4c53f59be7023ca204 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Mon, 20 Jul 2026 14:10:29 -0700 Subject: [PATCH 096/252] fix(variants): accept VariationDescriptor when extracting hgvs from post-mapped vrs get_hgvs_from_post_mapped only matched a bare "Allele" type, so post-mapped payloads wrapped in a VariationDescriptor fell through to the unhandled branch and returned None instead of their hgvs. --- src/mavedb/lib/variants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mavedb/lib/variants.py b/src/mavedb/lib/variants.py index 03203e3e7..677eb5de5 100644 --- a/src/mavedb/lib/variants.py +++ b/src/mavedb/lib/variants.py @@ -77,7 +77,7 @@ def get_hgvs_from_post_mapped(post_mapped_vrs: Optional[Any], *, combine_cis: bo if post_mapped_vrs["type"] in ("Haplotype", "CisPhasedBlock"): # type: ignore members = post_mapped_vrs["members"] - elif post_mapped_vrs["type"] == "Allele": # type: ignore + elif post_mapped_vrs["type"] in ("Allele", "VariationDescriptor"): # type: ignore members = [post_mapped_vrs] else: return None From 575a2e26894e7e959a90b86ce8d2ed4263e8fb28 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Mon, 20 Jul 2026 17:09:16 -0700 Subject: [PATCH 097/252] feat(migration): add manual backfill of allele substrate from mapped_variants Historical score sets have no mapping_records/alleles data yet, since that substrate is populated only by the live mapping job going forward. This reconstructs it deterministically from existing mapped_variants rows so the new serving/read layer (v_variant_annotations, published_variants MV) resolves for old data too, without redoing the reverse-translation fan-out (deferred to a decoupled enrichment pass). - add migrate_mapped_variants_to_allele_substrate manual migration, with verify/rollback/rebuild-annotations subcommands and idempotent re-run support - freeze the legacy MV-index migration's column signature as a literal instead of importing it from the model, since a later migration rewrites that model - add EventReason.MIGRATED for backfill-reconstructed annotation events, since a migration can't know the original creation history, only present/absent/failed --- ...ate_mapped_variants_to_allele_substrate.py | 1326 +++++++++++++++++ src/mavedb/models/enums/event_reason.py | 9 + 2 files changed, 1335 insertions(+) create mode 100644 alembic/manual_migrations/migrate_mapped_variants_to_allele_substrate.py diff --git a/alembic/manual_migrations/migrate_mapped_variants_to_allele_substrate.py b/alembic/manual_migrations/migrate_mapped_variants_to_allele_substrate.py new file mode 100644 index 000000000..32c752a6b --- /dev/null +++ b/alembic/manual_migrations/migrate_mapped_variants_to_allele_substrate.py @@ -0,0 +1,1326 @@ +"""Backfill the Allele substrate (``mapping_records`` / ``alleles`` / +``mapping_record_alleles`` + the annotation link tables) from existing +``mapped_variants`` rows. + +Context +------- +The annotation/serving layer is migrating off ``MappedVariant`` onto a +parallel-tables model:: + + Variant ─< MappingRecord (1 live/variant, ValidTime) ─< MappingRecordAllele (ValidTime) >─ Allele (dedup by vrs_digest) + +This script populates the new tables from the *existing* ``mapped_variants`` data so +the read-cutover (the ``v_variant_annotations`` view / ``published_variants`` MV +rewrites and the serving readers) resolves for historical score sets, which are +otherwise empty in the new substrate. + +This is the **reshape half of the hybrid migration**: it reconstructs the +authoritative spine deterministically from old data with no external-service +calls. It deliberately does **not** reproduce the reverse-translation +equivalence fan-out (the per-level coding/genomic/protein alleles and their +``projection_group`` pairing) — that data was never computed for old mappings +(``MappedVariant`` holds a single ``post_mapped`` VRS plus cross-level HGVS +*strings*, and an Allele needs a real ``vrs_digest`` that a string cannot +supply). Restoring that richness is the decoupled *enrichment* half: queue the +reverse-translation + annotation-refresh jobs per score set on the worker after +this backfill lands. Serving is correct at the measured level as soon as this +runs; enrichment only adds cross-level breadth and blocks nothing. + +What is reconstructed +--------------------- +* **MappingRecord history** — every ``MappedVariant`` version of a variant with a resolvable sequence + level becomes a ``MappingRecord`` with a gap-free ``[valid_from, valid_to)`` window derived from the + versions' ``mapped_date`` ordering. The ``current=True`` version is the live tail (``valid_to IS + NULL``). History dates are inherently approximate: ``MappedVariant`` records versioning with a + ``current`` boolean, not transaction time, so a window boundary is the successor's ``mapped_date``, + not the exact instant the swap happened. A version whose sequence level can't be attributed is + skipped **individually** (its siblings still get their records — see Idempotency below); if that + version happens to be the live one, the variant has no live record and stays eligible for retry. + Live dates that precede a historical version's are a logged anomaly (a likely re-map candidate), not + a fatal condition — see :func:`_reconstruct_windows`. +* **Authoritative Allele** — from each version's ``post_mapped`` VRS, deduplicated by ``vrs_digest`` + against an in-memory, per-chunk cache (:func:`_authoritative_allele` — not the live job's + ``get_or_create_allele``, which queries per call; see :func:`do_migration`'s docstring for why). + Carries the CAID and the assay-level HGVS + across so the allele-keyed serving endpoints resolve. The assay-level HGVS itself is read through a + three-tier fallback (:func:`_hgvs_assay_level_with_source`): ``mv.hgvs_assay_level`` directly, else + the per-level column matching the resolved level (``hgvs_g``/``hgvs_c``/``hgvs_p`` — backfilled later + and never caught up on a large share of old data), else the ``post_mapped`` VRS blob's own + ``expressions`` (the mapper sometimes stamps or reconstructs an HGVS expression directly on the VRS + Allele even when neither dedicated column got populated). Even with all three tiers, some legacy rows + carry no recoverable HGVS at all — that gap is real and needs the mapper re-run to close, not another + migration-time trick. Reading only ``hgvs_assay_level`` (as this script originally did) left most + reconstructed Alleles (and ``MappingRecord.hgvs_assay_level``, and therefore its derived + ``transcript``) NULL even when a real string was recoverable — silently starving any downstream pass + that resolves a transcript from it. +* **MappingRecordAllele** — one ``is_authoritative=True`` link per record on the + record's validity window. ``projection_group`` stays NULL (no fan-out). +* **Annotation timelines** — reconstructed across *every* mapping version, not just the live one: + ``gnomad_variants`` → ``gnomad_allele_links``, ``clinical_controls`` → ``clinvar_allele_links``, + VEP columns → ``vep_allele_consequences``. Each mapping version's bundled annotations are captured + against that version's window and, per allele, merged per source key (gnomAD variant / ClinVar + control / consequence value) into maximal intervals — so a continuously-held annotation is one link + dated from its **earliest** mapping version, open-ended only if it reaches the live record. A + release change that coincided with a re-map surfaces as a superseding closed+live pair. This is + faithful (the old jobs wrote annotations on the current mapping row, so their changes aligned to + re-map boundaries), just **coarse** — sub-mapping-version churn collapses to the latest value in + that window, and boundary timestamps are day-grained. The valid-time axis is uniformly the mapping + date across sources (VEP's real ``access_date`` is kept in its own audit column, not as + ``valid_from``, so ``as_of`` stays coherent). VEP has no Ensembl release string in old data, but + ``access_date`` is enough to derive one: ``source_version`` is looked up from a snapshotted table of + Ensembl release dates (:data:`_ENSEMBL_VEP_RELEASE_DATES`), not stamped as an opaque sentinel — see + :func:`_resolve_vep_source_version`. Only a missing ``access_date``, or one older than the table's + earliest known release, falls back to the ``"legacy"`` sentinel. + + **Exception:** a gnomAD or ClinVar window that would otherwise land live on a **protein-level** allele + is force-closed at the migration's own run time instead (VEP is unaffected — see below). The old + model could proxy both onto protein-level rows because there was nowhere else to attach them; the new + model resolves them correctly through the CA/PA link graph onto nucleotide siblings instead + (``lib/allele_measurements.py``), so carrying the proxy forward live would just create a second, + competing provenance story for the same value. Applied uniformly even though gnomAD is not known to + have actually been proxied this way in practice (ClinVar was) — it is the same underlying anti-pattern + (a nucleotide-level concept attached to a protein-level allele), so the guard costs nothing to keep + general. The historical fact is kept (closed, not deleted) for point-in-time audit; live annotations + for these alleles return once cross-level annotation fan-out re-derives them properly. + +Idempotency / no double-write +----------------------------- +A variant is either old-substrate or new (the parallel-tables invariant); native new-model variants +have no ``mapped_variants``. The migration therefore processes only variants that have +``mapped_variants`` and **no live** ``MappingRecord`` yet — live, not "any", because a variant whose +live mapped_variant version has no resolvable sequence level is left with only historical records (see +above) and must stay selectable so a later re-map or QC fix can complete it. Within a selected variant, +each window is additionally guarded by a ``(variant_id, mapped_date)`` existence check, so a re-run +that revisits a partially-migrated variant only fills in the windows still missing rather than +duplicating the ones a prior pass already wrote. Annotation links are existence-checked before +insertion, so a partial run resumes cleanly; run ``rebuild-annotations`` after resuming a +partially-migrated variant to pick up any timelines the skipped windows didn't get to build. + +Standalone convention +--------------------- +Follows the ``migrate_jsonb_ranges_to_table_rows.py`` pattern: invoked outside +``alembic upgrade`` (it can take a long time on production data), commits in +variant batches, and offers ``verify`` / ``rollback`` commands. + +Usage +----- +:: + + python -m alembic.manual_migrations.migrate_mapped_variants_to_allele_substrate # run + python -m alembic.manual_migrations.migrate_mapped_variants_to_allele_substrate verify # inspect without writing + python -m alembic.manual_migrations.migrate_mapped_variants_to_allele_substrate rollback # remove backfilled records (destructive) + python -m alembic.manual_migrations.migrate_mapped_variants_to_allele_substrate rebuild-annotations # (re)build annotation links for already-migrated variants + python -m alembic.manual_migrations.migrate_mapped_variants_to_allele_substrate migrate --score-set urn:mavedb:00000001-a-1 + python -m alembic.manual_migrations.migrate_mapped_variants_to_allele_substrate migrate --batch-size 1000 --dry-run +""" + +from __future__ import annotations + +import argparse +import logging +from collections import Counter, defaultdict +from dataclasses import dataclass +from datetime import date, datetime, time, timezone +from time import perf_counter # `time` above is datetime.time; import the timer directly +from typing import Optional + +import sqlalchemy as sa +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Session, configure_mappers + +from mavedb.models import * # noqa: F401,F403 pylint: disable=wildcard-import — register all mappers for configure_mappers() +from mavedb.db.session import SessionLocal +from mavedb.lib.annotation_status_manager import AnnotationStatusManager +from mavedb.lib.variants import get_hgvs_from_post_mapped +from mavedb.models.allele import Allele +from mavedb.models.annotation_event import AnnotationEvent +from mavedb.models.clinvar_allele_link import ClinvarAlleleLink +from mavedb.models.enums.annotation_type import AnnotationType +from mavedb.models.enums.disposition import Disposition +from mavedb.models.enums.event_reason import EventReason +from mavedb.models.enums.sequence_level import SequenceLevel +from mavedb.models.gnomad_allele_link import GnomadAlleleLink +from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.score_set import ScoreSet +from mavedb.models.variant import Variant +from mavedb.models.vep_allele_consequence import VepAlleleConsequence + +configure_mappers() + +logger = logging.getLogger(__name__) + +# Old VEP data carries no Ensembl release string, only an access_date — but source_version is derivable: +# Ensembl releases (and VEP's version with them) are coordinated, dated, and public +# (https://github.com/Ensembl/ensembl-vep/releases), so the release active on a given access_date can +# be looked up rather than stamped as an opaque sentinel. Each entry is a release's *earliest* GitHub +# tag date (patch tags like 92.1-92.6 are hotfixes within release 92, not new majors); an access_date +# is attributed to the newest release whose date is <= it. Snapshotted from the GitHub API on +# 2026-07-20 (covers releases 92 through 116); this table does not self-update, so re-run the same +# query and extend it before backfilling data annotated under a release newer than the last entry here. +_ENSEMBL_VEP_RELEASE_DATES: list[tuple[date, str]] = [ + (date(2018, 4, 12), "92"), + (date(2018, 7, 17), "93"), + (date(2018, 10, 2), "94"), + (date(2019, 1, 10), "95"), + (date(2019, 4, 8), "96"), + (date(2019, 7, 3), "97"), + (date(2019, 9, 27), "98"), + (date(2020, 1, 16), "99"), + (date(2020, 4, 29), "100"), + (date(2020, 8, 20), "101"), + (date(2020, 11, 30), "102"), + (date(2021, 2, 16), "103"), + (date(2021, 5, 5), "104"), + (date(2021, 12, 9), "105"), + (date(2022, 4, 14), "106"), + (date(2022, 7, 13), "107"), + (date(2022, 10, 20), "108"), + (date(2023, 2, 8), "109"), + (date(2023, 7, 18), "110"), + (date(2024, 1, 11), "111"), + (date(2024, 5, 16), "112"), + (date(2024, 10, 18), "113"), + (date(2025, 5, 8), "114"), + (date(2025, 9, 3), "115"), + (date(2026, 6, 10), "116"), +] + +# Fallback sentinel for the cases the release-date table can't resolve: no access_date at all, or an +# access_date older than the table's earliest known release (GitHub has no tags before release/92, so +# anything before 2018-04-12 is out of range). VepAlleleConsequence.source_version is NOT NULL, and a +# refresh run should treat this value as unversioned and re-confirm it against the current release +# rather than trusting it as current — the same behavior the sentinel always had. +LEGACY_VEP_SOURCE_VERSION = "legacy" + + +def _resolve_vep_source_version(access_date: Optional[date]) -> str: + """Map a legacy VEP ``access_date`` to the Ensembl release active on that day. + + Returns :data:`LEGACY_VEP_SOURCE_VERSION` when ``access_date`` is missing or predates the earliest + release the table knows about — both cases where the release genuinely can't be determined, as + opposed to just not being in the table's covered range for lack of maintenance. + """ + if access_date is None or access_date < _ENSEMBL_VEP_RELEASE_DATES[0][0]: + return LEGACY_VEP_SOURCE_VERSION + resolved = _ENSEMBL_VEP_RELEASE_DATES[0][1] + for release_date, version in _ENSEMBL_VEP_RELEASE_DATES: + if release_date > access_date: + break + resolved = version + return resolved + + +def _as_dt(d: date) -> datetime: + """Lift a ``date`` to a timezone-aware ``datetime`` for a ValidTime boundary. + + ValidTime windows are ``DateTime(timezone=True)``; ``MappedVariant`` versioning + is day-grained (``mapped_date``), so midnight UTC is the reconstructed boundary. + """ + return datetime.combine(d, time.min, tzinfo=timezone.utc) + + +def _resolve_level(mv: MappedVariant) -> Optional[SequenceLevel]: + """The sequence level a ``MappedVariant`` was assayed/aligned at. + + ``alignment_level`` is populated on every row by the target-gene-mapping QC + backfill and is the direct source (the mapping job sets a record's + ``assay_level`` and ``alignment_level`` to the same value). For the rare + legacy row the QC backfill could not attribute, fall back to whichever mapped + HGVS column is populated. Returns ``None`` when neither signal is present — + the record cannot be created (``assay_level`` is NOT NULL) and the caller + skips the variant with a logged anomaly. + """ + if mv.alignment_level is not None: + return mv.alignment_level + if mv.hgvs_g: + return SequenceLevel.genomic + if mv.hgvs_c: + return SequenceLevel.cdna + if mv.hgvs_p: + return SequenceLevel.protein + return None + + +_LEVEL_HGVS_COLUMN = { + SequenceLevel.genomic: "hgvs_g", + SequenceLevel.cdna: "hgvs_c", + SequenceLevel.protein: "hgvs_p", +} + + +def _hgvs_assay_level_with_source(mv: MappedVariant, level: SequenceLevel) -> tuple[Optional[str], str]: + """Resolve the assay-level HGVS string, reporting which of three tiers supplied it. + + ``mv.hgvs_assay_level`` is frequently ``NULL`` on legacy rows — it was backfilled later and never + caught up on a large share of old data — and reading only it (as this script originally did) meant + most reconstructed Alleles got a NULL ``hgvs_g``/``hgvs_c``/``hgvs_p`` and + ``MappingRecord.hgvs_assay_level`` even when the real string was recoverable. Since + ``MappingRecord.transcript`` derives from ``hgvs_assay_level``, that silently starved any downstream + pass that resolves a transcript from it (e.g. cross-level annotation fan-out). + + Three tiers, in order: + + 1. ``"direct"`` — ``mv.hgvs_assay_level`` itself. + 2. ``"level_column"`` — the per-level column matching ``level`` (``hgvs_g``/``hgvs_c``/``hgvs_p``), + mirroring :func:`_resolve_level`'s own fallback (same column), just for the *value* rather than + the level. + 3. ``"vrs_expression"`` — the ``post_mapped`` VRS blob's own ``expressions``. Some legacy rows have + *neither* dedicated column populated, but the mapper stamped an HGVS expression directly onto + the VRS Allele it produced (``dcd_mapping`` sets this unconditionally for coding-layer alleles, + and reconstructs it for genomic/protein when accession resolution succeeds — so it is not + universal either, just an independent, additional chance). Reuses + :func:`mavedb.lib.variants.get_hgvs_from_post_mapped`, the same extraction serving already relies + on, rather than re-parsing VRS JSON here. A malformed or VRS-1.x blob raises there; caught here + and treated as "no expression" so one bad blob doesn't abort the whole variant. + + Returns ``(None, "none")`` when no tier resolves anything — some legacy data genuinely carries no + recoverable HGVS at all, and that gap cannot be closed without re-running the mapper. + """ + if mv.hgvs_assay_level: + return mv.hgvs_assay_level, "direct" + + level_value = getattr(mv, _LEVEL_HGVS_COLUMN[level]) + if level_value: + return level_value, "level_column" + + try: + vrs_value = get_hgvs_from_post_mapped(mv.post_mapped) + except (KeyError, ValueError): + vrs_value = None + if vrs_value: + return vrs_value, "vrs_expression" + + return None, "none" + + +def _resolve_hgvs_assay_level(mv: MappedVariant, level: SequenceLevel) -> Optional[str]: + """The assay-level HGVS string for a version — see :func:`_hgvs_assay_level_with_source`.""" + return _hgvs_assay_level_with_source(mv, level)[0] + + +@dataclass +class _WindowResult: + """``_reconstruct_windows``'s output: the ordered windows plus an anomaly flag. + + Kept as one return value (rather than ``None``-means-skip on windows, anomaly bundled in) so a + caller can't forget to check the flag the way a bare tuple invites. + """ + + windows: list[tuple[MappedVariant, datetime, Optional[datetime]]] + live_precedes_history: bool + + +def _reconstruct_windows(mvs: list[MappedVariant]) -> Optional[_WindowResult]: + """Order a variant's ``MappedVariant`` versions and assign ValidTime windows. + + Returns ``(mv, valid_from, valid_to)`` per version, gap-free: each version's + ``valid_to`` is its successor's ``valid_from``, and the live tail's is + ``None``. The ``current=True`` version is the live tail regardless of its + ``mapped_date``. Returns ``None`` (skip) when no version is current — a + variant with mappings but no live one is a source anomaly we do not guess a + live state for. + + ``live_precedes_history=True`` flags the case where the live version's ``mapped_date`` is earlier + than some historical version's — a data-quality anomaly (``current`` should always be the newest). + When it happens, the pairwise clamp below still prevents inverted windows, but at the cost of + collapsing the affected historical window to zero-width (invisible to ``as_of``) rather than + losing data outright: the caller surfaces the flag as a loud, variant-attributed warning instead + of silently reordering, since it likely means the source variant wants a re-map. + """ + current = [m for m in mvs if m.current] + history = [m for m in mvs if not m.current] + + if not current: + return None + if len(current) == 1: + live = current[0] + else: + # Multiple live rows is a source anomaly; keep the newest live and demote + # the rest into history so no data is dropped. + current.sort(key=lambda m: (m.mapped_date, m.id)) + live = current[-1] + history.extend(current[:-1]) + logger.warning("Variant %s has %d current mapped_variants; keeping newest live.", live.variant_id, len(current)) + + history.sort(key=lambda m: (m.mapped_date, m.id)) + ordered = history + [live] + live_precedes_history = bool(history) and live.mapped_date < history[-1].mapped_date + + windows: list[tuple[MappedVariant, datetime, Optional[datetime]]] = [] + for i, m in enumerate(ordered): + valid_from = _as_dt(m.mapped_date) + valid_to: Optional[datetime] = None + if i + 1 < len(ordered): + successor = _as_dt(ordered[i + 1].mapped_date) + # Same-day or out-of-order successor: clamp so the window is never + # inverted (a zero-width [t, t) window is simply invisible to as_of). + valid_to = max(successor, valid_from) + windows.append((m, valid_from, valid_to)) + return _WindowResult(windows=windows, live_precedes_history=live_precedes_history) + + +def _fill_missing_allele_fields(allele: Allele, mv: MappedVariant, level: SequenceLevel) -> None: + """Backfill identity fields onto a (possibly pre-existing, deduped) allele. + + Dedup means the first variant to mint an allele sets its fields; a later + variant sharing the ``vrs_digest`` may carry the CAID or level HGVS the first + lacked. Those are content-derived (deterministic from the allele), so + fill-if-missing is safe and never overwrites a populated value. + """ + if allele.clingen_allele_id is None and mv.clingen_allele_id: + allele.clingen_allele_id = mv.clingen_allele_id + + hgvs_col = _LEVEL_HGVS_COLUMN[level] + hgvs_value = _resolve_hgvs_assay_level(mv, level) + if getattr(allele, hgvs_col) is None and hgvs_value: + setattr(allele, hgvs_col, hgvs_value) + + +def _authoritative_allele( + db: Session, mv: MappedVariant, level: SequenceLevel, allele_cache: dict[str, Allele] +) -> Optional[Allele]: + """Get-or-create the authoritative allele from ``mv.post_mapped``, via an in-memory dedup cache. + + Returns ``None`` for versions with no post-mapped representation (failed or benign-absent variants + get a record but no linked allele, matching the mapping job). + + Unlike :func:`mavedb.lib.variant_translations.get_or_create_allele` (used by the live mapping job), + this does **not** query the database or flush per call. ``allele_cache`` is pre-seeded by the + caller with one bulk ``vrs_digest IN (...)`` query per chunk (see :func:`do_migration`), and this + function keeps it updated in-memory as new alleles are drafted — so a digest shared by two windows + in the *same* chunk (a common dedup case) resolves from the dict, not a second round trip. This is + safe only because the session runs with ``autoflush=False`` (a query wouldn't see an unflushed + insert anyway) and because the caller flushes once per variant, not per window: an actual + ``uq_alleles_vrs_digest`` collision (e.g. a concurrent live mapping run) still surfaces at that + flush, inside the same per-variant SAVEPOINT it always has — this only removes the redundant reads, + not the write-time integrity check. + """ + post_mapped = mv.post_mapped or {} + digest = post_mapped.get("id") + if not digest: + return None + + existing = allele_cache.get(digest) + if existing is not None: + _fill_missing_allele_fields(existing, mv, level) + return existing + + hgvs_value = _resolve_hgvs_assay_level(mv, level) + allele = Allele( + vrs_digest=digest, + level=level, + hgvs_g=hgvs_value if level == SequenceLevel.genomic else None, + hgvs_c=hgvs_value if level == SequenceLevel.cdna else None, + hgvs_p=hgvs_value if level == SequenceLevel.protein else None, + clingen_allele_id=mv.clingen_allele_id, + post_mapped=post_mapped, + ) + db.add(allele) + allele_cache[digest] = allele + return allele + + +@dataclass +class _AnnotationObservation: + """One mapping version's annotation payload for an authoritative allele, tagged with that + version's validity window. + + Built fresh from the DB per allele-batch (:func:`_observations_for_alleles`) rather than collected + inline during record creation — so a deduped allele shared by several variants, or spanning a + variant's re-map history, still yields one merged timeline rather than per-version fragments (see + :func:`_write_annotation_timelines`), without needing every observation for the whole run held in + memory at once. + """ + + valid_from: datetime + valid_to: Optional[datetime] + gnomad_variant_ids: tuple[int, ...] + clinvar_control_ids: tuple[int, ...] + vep_consequence: Optional[str] + vep_access_date: Optional[date] + + +def _observation_for( + mv: MappedVariant, valid_from: datetime, valid_to: Optional[datetime] +) -> Optional[_AnnotationObservation]: + """Capture a mapping version's annotations for its window, or ``None`` if it carries none.""" + gnomad_ids = tuple(g.id for g in mv.gnomad_variants) + clinvar_ids = tuple(c.id for c in mv.clinical_controls) + if not gnomad_ids and not clinvar_ids and not mv.vep_functional_consequence: + return None + return _AnnotationObservation( + valid_from=valid_from, + valid_to=valid_to, + gnomad_variant_ids=gnomad_ids, + clinvar_control_ids=clinvar_ids, + vep_consequence=mv.vep_functional_consequence, + vep_access_date=mv.vep_access_date, + ) + + +def _merge_intervals( + windows: list[tuple[datetime, Optional[datetime]]], +) -> list[tuple[datetime, Optional[datetime]]]: + """Merge overlapping/adjacent ``[valid_from, valid_to)`` windows; ``valid_to=None`` is open-ended. + + An annotation held across a run of mapping versions produces contiguous windows (and overlapping + ones when a deduped allele is shared by several variants). Merging collapses each continuous run + into one link — ``valid_from`` = its earliest mapping date, ``valid_to`` open only if the run + reaches the live record — and closes a link only where the annotation actually lapses, rather than + fabricating a boundary at every re-map. + """ + ordered = sorted(windows, key=lambda w: w[0]) + merged: list[tuple[datetime, Optional[datetime]]] = [] + cur_from, cur_to = ordered[0] + for vf, vt in ordered[1:]: + if cur_to is None or vf <= cur_to: # overlap/adjacency; an open-ended window absorbs the rest + cur_to = None if (cur_to is None or vt is None) else max(cur_to, vt) + else: + merged.append((cur_from, cur_to)) + cur_from, cur_to = vf, vt + merged.append((cur_from, cur_to)) + return merged + + +def _allele_has_live_link(db: Session, model: type, allele_id: int) -> bool: + """Whether ``allele_id`` already has a live (``valid_to IS NULL``) link in ``model``. + + Sees **committed** rows only — the session runs ``autoflush=False``, so this does not observe + not-yet-flushed adds from the current pass. It therefore catches prior-run and concurrently- + committed live links (e.g. from the mapping pipeline running alongside this backfill); single-live + within one pass is enforced separately, in memory, by :func:`_write_allele_timeline`. + """ + return db.scalar(sa.select(model.id).where(model.allele_id == allele_id, model.valid_to.is_(None))) is not None + + +def _emit_gnomad_link( + db: Session, + allele_id: int, + gnomad_variant_id: int, + valid_from: datetime, + valid_to: Optional[datetime], + stats: Counter, +) -> None: + already = db.scalar( + sa.select(GnomadAlleleLink.id).where( + GnomadAlleleLink.allele_id == allele_id, + GnomadAlleleLink.gnomad_variant_id == gnomad_variant_id, + GnomadAlleleLink.valid_from == valid_from, + ) + ) + if already is not None: # idempotent re-run + return + db.add( + GnomadAlleleLink( + allele_id=allele_id, gnomad_variant_id=gnomad_variant_id, valid_from=valid_from, valid_to=valid_to + ) + ) + stats["gnomad_links"] += 1 + + +def _emit_clinvar_link( + db: Session, + allele_id: int, + clinvar_control_id: int, + valid_from: datetime, + valid_to: Optional[datetime], + stats: Counter, +) -> None: + already = db.scalar( + sa.select(ClinvarAlleleLink.id).where( + ClinvarAlleleLink.allele_id == allele_id, + ClinvarAlleleLink.clinvar_control_id == clinvar_control_id, + ClinvarAlleleLink.valid_from == valid_from, + ) + ) + if already is not None: # idempotent re-run; ClinVar is multi-live per (allele, control) + return + db.add( + ClinvarAlleleLink( + allele_id=allele_id, clinvar_control_id=clinvar_control_id, valid_from=valid_from, valid_to=valid_to + ) + ) + stats["clinvar_links"] += 1 + + +def _emit_vep_consequence( + db: Session, + allele_id: int, + consequence: str, + access_date: date, + valid_from: datetime, + valid_to: Optional[datetime], + stats: Counter, +) -> None: + already = db.scalar( + sa.select(VepAlleleConsequence.id).where( + VepAlleleConsequence.allele_id == allele_id, + VepAlleleConsequence.functional_consequence == consequence, + VepAlleleConsequence.valid_from == valid_from, + ) + ) + if already is not None: # idempotent re-run + return + source_version = _resolve_vep_source_version(access_date) + stats["vep_source_version_legacy_fallback" if source_version == LEGACY_VEP_SOURCE_VERSION else "vep_source_version_resolved"] += 1 + db.add( + VepAlleleConsequence( + allele_id=allele_id, + functional_consequence=consequence, + source_version=source_version, + access_date=access_date, + valid_from=valid_from, + valid_to=valid_to, + ) + ) + stats["vep_consequences"] += 1 + + +def _write_allele_timeline( + db: Session, allele_id: int, obs_list: list[_AnnotationObservation], stats: Counter, run_at: datetime +) -> None: + """Build and emit one allele's merged annotation link timelines (no flush/commit here). + + Keyed per source — gnomAD by ``gnomad_variant_id``, ClinVar by ``clinvar_control_id``, VEP by + consequence value — each key's windows are merged so a continuously-held annotation is a single + link dated from its earliest mapping version, open-ended only if it reaches the live record. + + **Single-live-per-allele** for gnomAD/VEP is enforced with an in-memory flag seeded from committed + state (:func:`_allele_has_live_link`). This is the fix for the ``autoflush=False`` session: a + deduped allele linked to two *different* gnomAD variants (from two variants sharing the allele) + would otherwise emit two live links and violate ``uq_gnomad_allele_links_live`` at flush, since the + per-add DB check can't see the earlier unflushed add. The flag also folds in a link already live + from a prior run or the concurrent mapping pipeline. ClinVar is multi-live per ``(allele, control)``. + + **Protein-level gnomAD/ClinVar are a legacy proxy the new model doesn't carry forward live.** The + old model could attach ClinVar significance and gnomAD frequency to protein-level ``MappedVariant`` + rows as a proxy — there was nowhere else to put them. Both are intrinsically nucleotide-level + concepts (a ClinVar submission and a gnomAD allele frequency are both keyed to a genomic variant, + not a protein change), and the new model resolves them correctly for a protein allele via the CA/PA + link graph onto its nucleotide siblings (see ``lib/allele_measurements.py``), not by direct + attachment. A live direct link here would just be a second, competing provenance story for the same + value. So a protein allele's gnomAD/ClinVar window is force-closed at ``run_at`` (the migration's + own run time) instead of staying open-ended: the historical fact that this had been proxy-annotated + is kept for point-in-time audit, but it stops being *live* data at the moment of reshape, to be + replaced once cross-level annotation fan-out re-derives it correctly through the link graph. VEP is + exempt — a functional consequence is computed relative to the assayed representation itself, so it + is not a proxied nucleotide-level value the way gnomAD/ClinVar are. + """ + gnomad_windows: dict[int, list[tuple[datetime, Optional[datetime]]]] = defaultdict(list) + clinvar_windows: dict[int, list[tuple[datetime, Optional[datetime]]]] = defaultdict(list) + vep_windows: dict[str, list[tuple[datetime, Optional[datetime]]]] = defaultdict(list) + vep_access: dict[str, list[date]] = defaultdict(list) + + for obs in obs_list: + for gnomad_variant_id in obs.gnomad_variant_ids: + gnomad_windows[gnomad_variant_id].append((obs.valid_from, obs.valid_to)) + for clinvar_control_id in obs.clinvar_control_ids: + clinvar_windows[clinvar_control_id].append((obs.valid_from, obs.valid_to)) + if obs.vep_consequence: + vep_windows[obs.vep_consequence].append((obs.valid_from, obs.valid_to)) + if obs.vep_access_date is not None: + vep_access[obs.vep_consequence].append(obs.vep_access_date) + + is_protein = False + if gnomad_windows or clinvar_windows: + allele_level = db.scalar(sa.select(Allele.level).where(Allele.id == allele_id)) + is_protein = allele_level == SequenceLevel.protein.value + + gnomad_live_taken = _allele_has_live_link(db, GnomadAlleleLink, allele_id) + for gnomad_variant_id, windows in gnomad_windows.items(): + for valid_from, valid_to in _merge_intervals(windows): + if is_protein and valid_to is None: + # Force-closed, not left live: not subject to the single-live guard below, since it + # never becomes a live row that could collide with uq_gnomad_allele_links_live. + valid_to = run_at + stats["protein_gnomad_links_closed"] += 1 + elif valid_to is None: + if gnomad_live_taken: + logger.warning( + "Allele %s already has a live gnomAD link; skipping gnomad_variant %s.", + allele_id, + gnomad_variant_id, + ) + continue + gnomad_live_taken = True + _emit_gnomad_link(db, allele_id, gnomad_variant_id, valid_from, valid_to, stats) + + for clinvar_control_id, windows in clinvar_windows.items(): + for valid_from, valid_to in _merge_intervals(windows): + if is_protein and valid_to is None: + valid_to = run_at + stats["protein_clinvar_links_closed"] += 1 + _emit_clinvar_link(db, allele_id, clinvar_control_id, valid_from, valid_to, stats) + + vep_live_taken = _allele_has_live_link(db, VepAlleleConsequence, allele_id) + for consequence, windows in vep_windows.items(): + observed_dates = vep_access[consequence] + for valid_from, valid_to in _merge_intervals(windows): + if valid_to is None: + if vep_live_taken: + logger.warning("Allele %s already has a live VEP consequence; skipping %r.", allele_id, consequence) + continue + vep_live_taken = True + access_date = min(observed_dates) if observed_dates else valid_from.date() + _emit_vep_consequence(db, allele_id, consequence, access_date, valid_from, valid_to, stats) + + +def _write_annotation_timelines( + db: Session, observations: dict[int, list[_AnnotationObservation]], stats: Counter, run_at: datetime +) -> None: + """Write each allele's annotation timelines, isolating conflicts per allele. + + Each allele is built + flushed inside its own SAVEPOINT: if a live-uniqueness collision still + slips through the in-memory guard — e.g. the concurrently-running mapping pipeline commits a live + link for a shared allele between the guard's committed-state read and this flush — only that + allele's links roll back (counted as ``annotation_conflicts``), and the rest of the pass proceeds + instead of the whole final commit aborting. + + ``run_at`` is a single timestamp for the whole pass (not re-read per allele), so every protein-level + ClinVar link force-closed by :func:`_write_allele_timeline` in this run shares the same cutover + instant rather than drifting with wall-clock time across a long batch. + """ + for allele_id, obs_list in observations.items(): + try: + with db.begin_nested(): + _write_allele_timeline(db, allele_id, obs_list, stats, run_at) + db.flush() # surface any live-uniqueness violation inside this allele's savepoint + except IntegrityError: + stats["annotation_conflicts"] += 1 + logger.warning("Annotation links for allele %s conflicted (live elsewhere); skipped.", allele_id) + + +def _observations_for_alleles(db: Session, allele_ids: list[int]) -> dict[int, list[_AnnotationObservation]]: + """Rebuild annotation observations for one batch of alleles, from fresh DB state. + + Scoped by allele id, not by variant/chunk: the merge :func:`_write_allele_timeline` performs is only + ever correct once *every* mapping version that ever authoritatively linked to a given allele has been + observed, and alleles are shared arbitrarily across variants — a variant-id chunk boundary says + nothing about which alleles are "done" (an allele's contributing variants can land in different + chunks). Batching by allele id instead means every batch's observations are complete for the alleles + in it, however many separate variant-chunks originally fed those alleles. + + Pairs each authoritative ``MappingRecord`` to its source ``MappedVariant`` version by + ``(variant_id, mapped_date)`` — the record was created with that version's ``mapped_date`` — mirroring + the pairing :func:`_migrate_variant` used when the record was first built. + """ + links = db.execute( + sa.select( + MappingRecordAllele.allele_id, + MappingRecord.variant_id, + MappingRecord.mapped_date, + MappingRecord.valid_from, + MappingRecord.valid_to, + ) + .join(MappingRecord, MappingRecordAllele.mapping_record_id == MappingRecord.id) + .where(MappingRecordAllele.allele_id.in_(allele_ids), MappingRecordAllele.is_authoritative) + ).all() + if not links: + return {} + + variant_ids = {variant_id for _, variant_id, _, _, _ in links} + mv_by_key: dict[tuple[int, date], MappedVariant] = { + (mv.variant_id, mv.mapped_date): mv + for mv in db.scalars(sa.select(MappedVariant).where(MappedVariant.variant_id.in_(variant_ids))).all() + } + + observations: dict[int, list[_AnnotationObservation]] = defaultdict(list) + for allele_id, variant_id, mapped_date, valid_from, valid_to in links: + mv = mv_by_key.get((variant_id, mapped_date)) + if mv is None: + continue # authoritative link survives independently of its source mapped_variant row + observation = _observation_for(mv, valid_from, valid_to) + if observation is not None: + observations[allele_id].append(observation) + return observations + + +def _backfill_annotation_timelines( + db: Session, allele_ids: list[int], *, batch_size: int, run_at: datetime, stats: Counter +) -> None: + """Build and commit annotation link timelines for ``allele_ids``, batch by batch. + + Each batch re-reads its own observations fresh from the DB (:func:`_observations_for_alleles`) and + commits before moving to the next, so memory is bounded by ``batch_size`` alleles at a time + regardless of how many millions of variants fed into ``allele_ids``. This replaces the previous + design, which accumulated every ``_AnnotationObservation`` for the *whole run* in memory before + writing anything at the very end — correct, but the accumulator's unbounded growth over a + multi-million-variant run put steadily increasing pressure on the GC, degrading throughput well + before the run completed (not just a flat per-call cost, but one that compounded with how much had + already been processed). + """ + total = len(allele_ids) + pass_start = perf_counter() + for offset in range(0, total, batch_size): + chunk = allele_ids[offset : offset + batch_size] + observations = _observations_for_alleles(db, chunk) + if observations: + _write_annotation_timelines(db, observations, stats, run_at) + db.commit() + processed = min(offset + batch_size, total) + elapsed = perf_counter() - pass_start + rate = processed / elapsed if elapsed else 0.0 + print(f" Built timelines for {processed}/{total} allele(s)... ({rate:.0f}/s, {elapsed:.0f}s elapsed)") + + +def _migrate_variant( + db: Session, + variant_id: int, + mvs: list[MappedVariant], + stats: Counter, + skipped: dict[str, list[int]], + annotation_manager: AnnotationStatusManager, + existing_records: set[tuple[int, date]], + allele_cache: dict[str, Allele], +) -> set[int]: + """Reconstruct as much of a variant's record history as its data supports, returning the ids of + every allele this variant authoritatively touched. + + Also records one ``VRS_MAPPING`` event per version created (variant-subject) and one + ``CLINGEN_ALLELE_ID`` / ``MAPPED_HGVS`` event per allele-touch (allele-subject), all reason + :data:`EventReason.MIGRATED` — see :func:`do_migration`'s docstring for why this is scoped to the + mapping/identity side only, not gnomAD/ClinVar/VEP. + + The caller only accumulates these bare ids (not the annotation payload itself) into a global, + whole-run set — a dedup key is all a deferred, allele-batched annotation pass + (:func:`_backfill_annotation_timelines`) needs to know which alleles to revisit; the actual + gnomAD/ClinVar/VEP observations are re-read fresh from the DB per allele-batch there, not carried + in memory from here. Carrying full ``_AnnotationObservation`` payloads for the whole run was the + previous design and is what overloaded the heap at scale — see that function's docstring. Returns + an empty set for a variant with no current version at all — that one is a full skip, since there is + no live state to reconstruct. + + A version with no resolvable sequence level is skipped **individually**, not the whole variant: + ``ValidTime`` history is exactly the mechanism for retaining partial provenance, so one bad version + should not cost its siblings their records. A window's ``valid_from``/``valid_to`` only depend on + its neighbors' ``mapped_date`` (computed up front in :func:`_reconstruct_windows`), never on + whether a record actually got created for those neighbors — so skipping a window here doesn't + distort the boundaries around it. Already-migrated windows (from a prior partial run that + stopped here before this fix, or a live-level failure that keeps this variant eligible for retry — + see :func:`_variant_ids_to_migrate`) are detected via ``existing_records`` (bulk pre-fetched once per + chunk by the caller — see :func:`do_migration`) and skipped as a no-op, so a re-run is safe to + resume mid-variant. Their annotations are not re-derived here; run ``rebuild-annotations`` after a + resumed run to backfill any timelines missed by the skip. + + Every ``MappingRecord``/``Allele``/``MappingRecordAllele`` this variant needs is only ``db.add``'d + here, not flushed, until one ``db.flush()`` at the end covering the whole variant — cut from one + flush per *window* (this was the dominant cost at scale: round trips, not query time, were the + bottleneck). ``MappingRecordAllele`` is built from the ``record``/``allele`` *objects*, not their + ``.id``, specifically so it doesn't need them flushed first — SQLAlchemy resolves the FKs at flush + time. Observations and ``MIGRATED`` events (which need real ``allele.id``/``variant_id`` values) are + built in a second pass over ``pending`` *after* that single flush, once ids exist. + + Skips/anomalies are recorded into ``skipped`` (reason → variant ids), not logged per variant: a + single unmappable score set (e.g. a large pairwise-haplotype library the mapper never converted) + can contribute hundreds of thousands of skips, so :func:`do_migration` rolls them up into one line + per score set at the end rather than emitting a warning each. + """ + result = _reconstruct_windows(mvs) + if result is None: + stats["variants_skipped_no_current"] += 1 + skipped["no_current"].append(variant_id) + logger.debug("Variant %s has mapped_variants but none current; skipped.", variant_id) + return set() + + if result.live_precedes_history: + # Not fatal — the clamp in _reconstruct_windows keeps windows non-inverted — but it silently + # collapses a historical window to zero-width, so surface it loudly: this shape means the + # live mapped_variant's mapped_date is older than a superseded one, which shouldn't happen + # absent a source data-quality issue, and likely wants a re-map to fix at the source. + stats["variants_live_precedes_history"] += 1 + skipped["live_precedes_history"].append(variant_id) + + # (mv, record, allele-or-None, hgvs_assay_level, valid_from, valid_to) per window actually created + # this pass — resolved to real ids by one flush below, then walked again to emit events and collect + # touched allele ids. + pending: list[tuple[MappedVariant, MappingRecord, Optional[Allele], Optional[str], datetime, Optional[datetime]]] = [] + + for mv, valid_from, valid_to in result.windows: + if (variant_id, mv.mapped_date) in existing_records: + continue # already migrated in a prior pass over this variant; resuming, not redoing + + level = _resolve_level(mv) + if level is None: + is_live = valid_to is None + reason = "no_level_live" if is_live else "no_level_historical" + stats[f"windows_skipped_{reason}"] += 1 + skipped[reason].append(variant_id) + logger.debug( + "Variant %s has an unattributable mapped_variant (id=%s, live=%s); window skipped.", + variant_id, + mv.id, + is_live, + ) + continue + + hgvs_assay_level, hgvs_source = _hgvs_assay_level_with_source(mv, level) + if hgvs_source == "level_column": + stats["hgvs_recovered_from_level_column"] += 1 + elif hgvs_source == "vrs_expression": + stats["hgvs_recovered_from_vrs_expression"] += 1 + + record = MappingRecord( + variant_id=variant_id, + vrs_digest=(mv.pre_mapped or {}).get("id"), + pre_mapped=mv.pre_mapped or None, + assay_level=level, + hgvs_assay_level=hgvs_assay_level, + mapped_date=mv.mapped_date, + vrs_version=mv.vrs_version, + mapping_api_version=mv.mapping_api_version, + target_gene_mapping_id=mv.target_gene_mapping_id, + alignment_level=mv.alignment_level, + at_mismatched_locus=mv.at_mismatched_locus, + near_gap=mv.near_gap, + valid_from=valid_from, + valid_to=valid_to, + ) + db.add(record) + stats["records"] += 1 + + allele = _authoritative_allele(db, mv, level, allele_cache) + if allele is not None: + db.add( + MappingRecordAllele( + mapping_record=record, + allele=allele, + is_authoritative=True, + valid_from=valid_from, + valid_to=valid_to, + ) + ) + stats["record_alleles"] += 1 + + pending.append((mv, record, allele, hgvs_assay_level, valid_from, valid_to)) + + if not pending: + return set() + + db.flush() # one round trip resolves every record/allele/link id added for this whole variant + stats["variants"] += 1 + + touched_allele_ids: set[int] = set() + for mv, record, allele, hgvs_assay_level, valid_from, valid_to in pending: + # VRS_MAPPING: variant-subject, written for every version regardless of allele — mirrors the + # live job (mapping.py), whose FAILED/ABSENT outcomes get a record but no allele too. + if mv.error_message: + mapping_disposition = Disposition.FAILED + elif allele is not None: + mapping_disposition = Disposition.PRESENT + else: + mapping_disposition = Disposition.ABSENT + annotation_manager.record_event( + AnnotationType.VRS_MAPPING, + variant_id=variant_id, + disposition=mapping_disposition, + reason=EventReason.MIGRATED.value, + source_version=mv.mapping_api_version, + ) + stats["annotation_events_written"] += 1 + + if allele is not None: + # CLINGEN_ALLELE_ID / MAPPED_HGVS: allele-subject, only meaningful once an allele exists. + # allele.id is real now — this loop runs after the single flush above. + annotation_manager.record_event( + AnnotationType.CLINGEN_ALLELE_ID, + allele_id=allele.id, + disposition=Disposition.PRESENT if mv.clingen_allele_id else Disposition.ABSENT, + reason=EventReason.MIGRATED.value, + source_version=mv.mapping_api_version, + ) + annotation_manager.record_event( + AnnotationType.MAPPED_HGVS, + allele_id=allele.id, + disposition=Disposition.PRESENT if hgvs_assay_level else Disposition.ABSENT, + reason=EventReason.MIGRATED.value, + source_version=mv.mapping_api_version, + ) + stats["annotation_events_written"] += 2 + + touched_allele_ids.add(allele.id) + + # Flushed once for the whole variant (not per event) — still inside this variant's own SAVEPOINT + # (the caller's `with db.begin_nested():` wraps this entire call), so a buffered event never + # outlives the rollback of the record/allele it references. See do_migration's except-block for the + # backstop: if something *between* here and there still raised, pending events for this variant are + # explicitly discarded rather than leaking into a later, unrelated flush. + annotation_manager.flush() + return touched_allele_ids + + +def _variant_ids_to_migrate(db: Session, score_set_urn: Optional[str]) -> list[int]: + """Variant ids that have ``mapped_variants`` but no *live* ``MappingRecord`` yet. + + Keyed on a live record specifically, not "any record at all": a variant can have historical + records from a prior pass but still be missing its live one (the live version's mapped_variant had + no resolvable sequence level — see :func:`_migrate_variant`), and that variant must stay eligible + for retry until a re-map (or a QC fix) lets its live window resolve. The parallel-tables invariant + still makes "has mapped_variants and no live mapping record" the exact set of variants with + incomplete substrate; native new-model variants have no ``mapped_variants``. + """ + has_live_record = sa.select(MappingRecord.id).where( + MappingRecord.variant_id == Variant.id, MappingRecord.valid_to.is_(None) + ) + query = ( + sa.select(Variant.id) + .join(MappedVariant, MappedVariant.variant_id == Variant.id) + .where(~has_live_record.exists()) + ) + if score_set_urn is not None: + query = query.join(ScoreSet, ScoreSet.id == Variant.score_set_id).where(ScoreSet.urn == score_set_urn) + return list(db.scalars(query.distinct().order_by(Variant.id)).all()) + + +_SKIP_REASONS = { + "no_current": "mapped_variants present but none current", + "no_level_historical": "a superseded mapped_variant with no resolvable sequence level (window skipped, siblings rescued)", + "no_level_live": "the LIVE mapped_variant has no resolvable sequence level — no live record could be created; remains eligible for retry", + "live_precedes_history": "the live mapped_variant's mapped_date precedes a historical one (data anomaly — a re-map candidate)", +} + + +def _log_skip_summary(db: Session, skipped: dict[str, list[int]], *, chunk: int = 5000) -> None: + """Roll skipped variants up into one log line per (reason, score set). + + Skips cluster hard by score set — a whole library the mapper couldn't convert skips en masse — so + a per-variant warning drowns the log while a per-score-set count pinpoints exactly which set to + chase. Variant ids are resolved to score set URNs in chunks to keep the ``IN`` clauses bounded. + """ + for reason, variant_ids in skipped.items(): + if not variant_ids: + continue + counts: Counter = Counter() + for offset in range(0, len(variant_ids), chunk): + part = variant_ids[offset : offset + chunk] + for urn, n in db.execute( + sa.select(ScoreSet.urn, sa.func.count()) + .join(Variant, Variant.score_set_id == ScoreSet.id) + .where(Variant.id.in_(part)) + .group_by(ScoreSet.urn) + ).all(): + counts[urn] += n + detail = _SKIP_REASONS.get(reason, reason) + logger.warning("Skipped %d variant(s) across %d score set(s) — %s:", len(variant_ids), len(counts), detail) + for urn, n in counts.most_common(): + logger.warning(" %s: %d variant(s) skipped (%s).", urn, n, reason) + + +def do_migration( + db: Session, + *, + score_set_urn: Optional[str] = None, + batch_size: int = 1000, + dry_run: bool = False, + run_at: Optional[datetime] = None, +) -> None: + """Backfill the allele substrate from ``mapped_variants``, committing per batch. + + ``run_at`` (default: now) is the single cutover instant used to force-close any protein-level + ClinVar link that would otherwise land live — see :func:`_write_allele_timeline`. Exposed as a + parameter mainly for deterministic tests; a real run just wants "now". + + Also backfills the ``annotation_event`` audit log — but only for the mapping/identity side + (``VRS_MAPPING``, ``CLINGEN_ALLELE_ID``, ``MAPPED_HGVS``), not gnomAD/ClinVar/VEP. That scope split + is deliberate: the mapping side is a genuine, non-lossy reconstruction (one real historical event + per ``mapped_variants`` version, the same data already trusted enough to build ``MappingRecord``/ + ``Allele`` from), whereas gnomAD/ClinVar/VEP would need to guess at a created/reconfirmed/skipped + cadence legacy data never recorded — and those sources get real, fresh events again once each score + set is re-annotated, so the audit gap there is temporary, not permanent. See + :data:`EventReason.MIGRATED`. + """ + print("Backfilling the Allele substrate from mapped_variants...") + if dry_run: + print(" DRY RUN — no changes will be committed.") + run_at = run_at or datetime.now(timezone.utc) + annotation_manager = AnnotationStatusManager(db) + + variant_ids = _variant_ids_to_migrate(db, score_set_urn) + print(f"Found {len(variant_ids)} variants to migrate.") + + stats: Counter = Counter() + # Wall-clock spent in each phase, so a run reports where the time goes (batch build vs. the + # final annotation pass) and whether the per-batch rate degrades as the tables grow. + phase_seconds: Counter = Counter() + # Ids only — a deduped allele can be shared by variants in different chunks, so its annotation + # timeline can only be assembled once every contributing variant has been migrated, but the actual + # gnomAD/ClinVar/VEP payload doesn't need to ride along in memory to get there: it's re-read fresh + # from the DB per allele-batch in the deferred pass below (_backfill_annotation_timelines). Holding + # just ids here (not full _AnnotationObservation objects) keeps this set's footprint tiny even at + # millions of variants — see that function's docstring for why the old accumulate-everything design + # degraded badly at scale. + touched_allele_ids: set[int] = set() + # Skipped variant ids by reason, rolled up into a per-score-set summary at the end instead of a + # per-variant warning (one unmappable library can skip hundreds of thousands of variants). + skipped: dict[str, list[int]] = defaultdict(list) + + run_start = perf_counter() + for offset in range(0, len(variant_ids), batch_size): + chunk = variant_ids[offset : offset + batch_size] + + mark = perf_counter() + mvs_by_variant: dict[int, list[MappedVariant]] = {vid: [] for vid in chunk} + for mv in db.scalars(sa.select(MappedVariant).where(MappedVariant.variant_id.in_(chunk))).all(): + mvs_by_variant[mv.variant_id].append(mv) + phase_seconds["fetch_mapped_variants"] += perf_counter() - mark + + # Bulk pre-fetch, once per chunk instead of once per window — this (not query execution time) + # was the actual bottleneck: a per-window round trip for "does this record already exist" plus + # another for "does this allele already exist" dominates wall-clock at millions-of-rows scale, + # far more than the cost of the queries themselves. + mark = perf_counter() + existing_records: set[tuple[int, date]] = set( + db.execute( + sa.select(MappingRecord.variant_id, MappingRecord.mapped_date).where( + MappingRecord.variant_id.in_(chunk) + ) + ).all() + ) + candidate_digests = { + (mv.post_mapped or {}).get("id") for mvs in mvs_by_variant.values() for mv in mvs + } + candidate_digests.discard(None) + allele_cache: dict[str, Allele] = {} + if candidate_digests: + for allele in db.scalars(sa.select(Allele).where(Allele.vrs_digest.in_(candidate_digests))).all(): + allele_cache[allele.vrs_digest] = allele + phase_seconds["prefetch"] += perf_counter() - mark + + mark = perf_counter() + for variant_id in chunk: + # A per-variant SAVEPOINT so one bad variant rolls back only its own inserts (not the + # whole batch), and its touched allele ids are simply never added rather than referencing + # rolled-back rows. + pending_events_before = len(annotation_manager._pending) + try: + with db.begin_nested(): + variant_touched_allele_ids = _migrate_variant( + db, + variant_id, + mvs_by_variant[variant_id], + stats, + skipped, + annotation_manager, + existing_records, + allele_cache, + ) + except Exception as exc: # noqa: BLE001 — one bad variant must not abort the batch + # Discard any events this variant buffered but never got to flush before raising — they + # would otherwise linger in the manager's queue and get inserted by a later, unrelated + # flush, still referencing this variant/allele's now-rolled-back id (an FK violation far + # from its actual cause). _migrate_variant flushes its own events before returning + # normally, so this is a no-op on the success path. + del annotation_manager._pending[pending_events_before:] + stats["variants_errored"] += 1 + logger.exception("Failed to migrate variant %s: %s", variant_id, exc) + continue + touched_allele_ids.update(variant_touched_allele_ids) + phase_seconds["build_records"] += perf_counter() - mark + + mark = perf_counter() + if dry_run: + db.rollback() + touched_allele_ids.clear() # nothing persisted this batch; these ids are now invalid + else: + db.commit() + phase_seconds["commit"] += perf_counter() - mark + + processed = min(offset + batch_size, len(variant_ids)) + elapsed = perf_counter() - run_start + rate = processed / elapsed if elapsed else 0.0 + print(f" Processed {processed}/{len(variant_ids)} variants... ({rate:.0f}/s, {elapsed:.0f}s elapsed)") + + if not dry_run and touched_allele_ids: + print(f"Building annotation timelines for {len(touched_allele_ids)} allele(s)...") + mark = perf_counter() + _backfill_annotation_timelines( + db, sorted(touched_allele_ids), batch_size=batch_size, run_at=run_at, stats=stats + ) + phase_seconds["annotation_timelines"] += perf_counter() - mark + + print("\nMigration completed:") + for key in ( + "variants", + "records", + "record_alleles", + "hgvs_recovered_from_level_column", + "hgvs_recovered_from_vrs_expression", + "annotation_events_written", + "gnomad_links", + "protein_gnomad_links_closed", + "clinvar_links", + "protein_clinvar_links_closed", + "vep_consequences", + "vep_source_version_resolved", + "vep_source_version_legacy_fallback", + "annotation_conflicts", + "variants_skipped_no_current", + "windows_skipped_no_level_historical", + "windows_skipped_no_level_live", + "variants_live_precedes_history", + "variants_errored", + ): + print(f" {key}: {stats[key]}") + + _log_skip_summary(db, skipped) + + total = perf_counter() - run_start + print("\nTiming by phase:") + for phase in ("fetch_mapped_variants", "build_records", "commit", "annotation_timelines"): + seconds = phase_seconds[phase] + share = f"{100 * seconds / total:.0f}%" if total else "n/a" + print(f" {phase}: {seconds:.1f}s ({share})") + variants_done = stats["variants"] + overall_rate = f"{variants_done / total:.0f}/s" if total else "n/a" + print(f" total: {total:.1f}s ({overall_rate})") + + +def _allele_ids_to_rebuild(db: Session, score_set_urn: Optional[str]) -> list[int]: + """Every allele id with an authoritative link, optionally scoped to one score set's variants.""" + query = sa.select(sa.distinct(MappingRecordAllele.allele_id)).where(MappingRecordAllele.is_authoritative) + if score_set_urn is not None: + query = ( + query.join(MappingRecord, MappingRecordAllele.mapping_record_id == MappingRecord.id) + .join(Variant, Variant.id == MappingRecord.variant_id) + .join(ScoreSet, ScoreSet.id == Variant.score_set_id) + .where(ScoreSet.urn == score_set_urn) + ) + return list(db.scalars(query.order_by(MappingRecordAllele.allele_id)).all()) + + +def rebuild_annotation_timelines( + db: Session, *, score_set_urn: Optional[str] = None, batch_size: int = 1000, run_at: Optional[datetime] = None +) -> None: + """(Re)build annotation link timelines for already-migrated variants, independently of record + creation. Idempotent (existence-checked, savepoint-isolated). Use to recover after a crash in the + ``migrate`` annotation pass, or to refresh timelines after fixing annotation logic. + + Rebuilds strictly from the legacy proxy data (``mapped_variants`` associations), same as + ``migrate`` — it is not the cross-level fan-out job — so it applies the same protein-level ClinVar + closure rule (see :func:`_write_allele_timeline`) using its own ``run_at`` (default: now). Batches by + allele id via :func:`_backfill_annotation_timelines`, the same bounded-memory pass ``migrate`` itself + uses — this used to accumulate every observation for the whole rebuild in memory before writing + anything, which had the identical unbounded-growth problem the ``migrate`` pass was fixed for. + """ + print("Rebuilding annotation timelines from existing records...") + run_at = run_at or datetime.now(timezone.utc) + allele_ids = _allele_ids_to_rebuild(db, score_set_urn) + print(f"Found {len(allele_ids)} allele(s) with authoritative links.") + + stats: Counter = Counter() + _backfill_annotation_timelines(db, allele_ids, batch_size=batch_size, run_at=run_at, stats=stats) + + print("\nRebuild completed:") + for key in ( + "gnomad_links", + "protein_gnomad_links_closed", + "clinvar_links", + "protein_clinvar_links_closed", + "vep_consequences", + "vep_source_version_resolved", + "vep_source_version_legacy_fallback", + "annotation_conflicts", + ): + print(f" {key}: {stats[key]}") + + +def verify_migration(db: Session, *, score_set_urn: Optional[str] = None) -> None: + """Report substrate coverage without writing.""" + print("\nVerifying backfill...") + + remaining = len(_variant_ids_to_migrate(db, score_set_urn)) + total_mapped_variant_variants = db.scalar(sa.select(sa.func.count(sa.distinct(MappedVariant.variant_id)))) + total_records = db.scalar(sa.select(sa.func.count(MappingRecord.id))) + live_records = db.scalar(sa.select(sa.func.count(MappingRecord.id)).where(MappingRecord.valid_to.is_(None))) + total_alleles = db.scalar(sa.select(sa.func.count(Allele.id))) + live_auth_links = db.scalar( + sa.select(sa.func.count(MappingRecordAllele.id)).where( + MappingRecordAllele.is_authoritative, MappingRecordAllele.valid_to.is_(None) + ) + ) + gnomad_links = db.scalar(sa.select(sa.func.count(GnomadAlleleLink.id))) + clinvar_links = db.scalar(sa.select(sa.func.count(ClinvarAlleleLink.id))) + vep_consequences = db.scalar(sa.select(sa.func.count(VepAlleleConsequence.id))) + migrated_events = db.scalar( + sa.select(sa.func.count(AnnotationEvent.id)).where(AnnotationEvent.reason == EventReason.MIGRATED.value) + ) + + print(f" Variants with mapped_variants: {total_mapped_variant_variants}") + print(f" Variants still needing backfill: {remaining}") + print(f" MappingRecords (total / live): {total_records} / {live_records}") + print(f" Alleles (deduped): {total_alleles}") + print(f" Live authoritative links: {live_auth_links}") + print(f" gnomAD / ClinVar / VEP annotations: {gnomad_links} / {clinvar_links} / {vep_consequences}") + print(f" MIGRATED annotation_event rows: {migrated_events}") + if remaining == 0: + print(" ✓ Every legacy variant has a MappingRecord.") + else: + print(f" ⚠ {remaining} variants remain (see logged anomalies, or re-run migrate).") + + +def rollback_migration(db: Session) -> None: + """Delete backfilled ``MappingRecord``s (cascading their allele links). + + Scoped to variants that also have ``mapped_variants`` — the exact discriminator + for backfilled records, since native new-model variants have no legacy rows. + Deduped ``Allele``s and annotation links are left in place: alleles are content + -addressed and harmless if orphaned (a re-run reuses them), and annotation + links re-existence-check. Delete those manually if a full teardown is needed. + + ``AnnotationEvent`` rows (reason ``MIGRATED``) are left in place too, on purpose: it is an + append-only audit log, and deleting them on rollback would erase the very audit trail it exists to + keep — a re-run after rollback just appends a fresh set (matching how ``MappingRecord`` itself gets + deleted and recreated), not a duplicate-avoidance concern the way the ValidTime domain tables are. + """ + print("Rolling back backfilled MappingRecords (scoped to variants with mapped_variants)...") + legacy_variant_ids = sa.select(sa.distinct(MappedVariant.variant_id)) + count = db.scalar( + sa.select(sa.func.count(MappingRecord.id)).where(MappingRecord.variant_id.in_(legacy_variant_ids)) + ) + # ON DELETE CASCADE on mapping_record_alleles.mapping_record_id removes the links. + db.execute(sa.delete(MappingRecord).where(MappingRecord.variant_id.in_(legacy_variant_ids))) + db.commit() + print(f"Deleted {count} MappingRecords (and their allele links via cascade).") + print("Note: deduped Alleles, annotation links, and MIGRATED annotation_event rows were left in place") + print("(re-runnable / harmless orphans; the event log is append-only by design).") + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Backfill the Allele substrate from mapped_variants.") + parser.add_argument( + "command", nargs="?", default="migrate", choices=["migrate", "verify", "rollback", "rebuild-annotations"] + ) + parser.add_argument("--score-set", dest="score_set_urn", default=None, help="Limit to one score set URN.") + parser.add_argument("--batch-size", type=int, default=1000, help="Variants per commit (default 1000).") + parser.add_argument("--dry-run", action="store_true", help="Run without committing (rolls back each batch).") + return parser.parse_args() + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + args = _parse_args() + + if args.command == "rollback": + print("WARNING: this deletes all backfilled MappingRecords and their allele links.") + if input("Are you sure you want to continue? (y/N): ").lower() == "y": + with SessionLocal() as session: + rollback_migration(session) + else: + print("Rollback cancelled.") + elif args.command == "verify": + with SessionLocal() as session: + verify_migration(session, score_set_urn=args.score_set_urn) + elif args.command == "rebuild-annotations": + with SessionLocal() as session: + rebuild_annotation_timelines(session, score_set_urn=args.score_set_urn, batch_size=args.batch_size) + verify_migration(session, score_set_urn=args.score_set_urn) + else: + with SessionLocal() as session: + do_migration( + session, + score_set_urn=args.score_set_urn, + batch_size=args.batch_size, + dry_run=args.dry_run, + ) + verify_migration(session, score_set_urn=args.score_set_urn) diff --git a/src/mavedb/models/enums/event_reason.py b/src/mavedb/models/enums/event_reason.py index 35aa23af4..2082a6163 100644 --- a/src/mavedb/models/enums/event_reason.py +++ b/src/mavedb/models/enums/event_reason.py @@ -10,8 +10,17 @@ class EventReason(str, Enum): contributes its own pre-existing domain enum instead of duplicating here: ``mapping`` uses ``MappingOutcome`` (mapped / intronic / no_protein_consequence / failed), which mirrors the external dcd-mapping vocabulary. The full ``reason`` vocabulary is this enum plus that one. + + ``MIGRATED`` is the one exception to "reason means what happened": it deliberately does not + distinguish created/preexisting/reconfirmed etc., because a one-off data migration reconstructing + state from a legacy source cannot know which of those actually happened historically — only + ``disposition`` (present/absent/failed) is knowable there. Using it elsewhere would misrepresent a + live job's real observation as a backfill guess. """ + # spans present / absent / failed — a reconstructed observation, not a live run + MIGRATED = "migrated" # state reconstructed from a world lacking now required context; no per-run history + # present — we hold the result / the step succeeded CREATED = "created" # linked/registered this run (gnomAD, ClinVar, CAR) PREEXISTING = "preexisting" # already held before this run (gnomAD, ClinVar, CAR) From b3fd1ba21a0a1d044f5c2d06f3ed5aea97154111 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Mon, 20 Jul 2026 17:14:49 -0700 Subject: [PATCH 098/252] feat(serving): rewrite annotation views and stats MV onto mapping_records The read layer (v_variant_annotations, published_variants MV, statistics router) still joined the legacy mapped_variants table, so it never resolved the mapping_records/alleles substrate the mapping job now writes to. This completes the read-side cutover onto the new tables. - rewrite v_variant_annotations to pivot the record's live allele links into the flat hgvs_g/hgvs_c/hgvs_p triple, and pull clingen_allele_id and VEP consequence from the authoritative allele, via correlated scalar subqueries - rebuild published_variants_materialized_view on mapping_record_id / current_mapping_record, keeping every record version (not just the live one) so history-aware counts still work - update statistics router and tests to the renamed columns - freeze all historical migrations' view/MV SQL as inline literals instead of importing from the model, so replaying an old revision builds the shape that existed at that point in history, not whatever the model looks like after later migrations rewrite it - add view-level tests for the flat-triple reconstruction, protein-only assays, and unmapped variants under the new substrate --- ..._simplify_published_variants_mv_indexes.py | 7 +- ...rite_v_variant_annotations_onto_alleles.py | 91 ++++++++++++++ ...uild_published_variants_mv_onto_records.py | 86 +++++++++++++ ...aterialized_view_for_variant_statistics.py | 59 +++++---- ...b3c4d5e6_add_v_variant_annotations_view.py | 33 ++++- src/mavedb/models/published_variant.py | 22 ++-- src/mavedb/models/variant_annotation_view.py | 111 ++++++++++++++--- src/mavedb/routers/statistics.py | 10 +- tests/conftest.py | 4 +- tests/models/test_variant_annotation_view.py | 115 ++++++++++++++++++ tests/routers/test_statistics.py | 12 +- 11 files changed, 485 insertions(+), 65 deletions(-) create mode 100644 alembic/versions/a1d4f7c2e9b0_rewrite_v_variant_annotations_onto_alleles.py create mode 100644 alembic/versions/b2e5a8d3f0c1_rebuild_published_variants_mv_onto_records.py create mode 100644 tests/models/test_variant_annotation_view.py diff --git a/alembic/versions/4726e4dddde8_simplify_published_variants_mv_indexes.py b/alembic/versions/4726e4dddde8_simplify_published_variants_mv_indexes.py index 479c51a33..1ce888933 100644 --- a/alembic/versions/4726e4dddde8_simplify_published_variants_mv_indexes.py +++ b/alembic/versions/4726e4dddde8_simplify_published_variants_mv_indexes.py @@ -8,8 +8,6 @@ from alembic import op -from mavedb.models.published_variant import signature - # revision identifiers, used by Alembic. revision = "4726e4dddde8" @@ -17,6 +15,11 @@ branch_labels = None depends_on = None +# Frozen literal, not imported from the model: this migration replaces the historical +# mapped_variants-based MV indexes at its point in history. A later migration rebuilds the MV onto +# the mapping_records substrate and re-indexes it, so the column names below stay the legacy ones. +signature = "published_variants_materialized_view" + def upgrade(): # ### commands auto generated by Alembic - please adjust! ### diff --git a/alembic/versions/a1d4f7c2e9b0_rewrite_v_variant_annotations_onto_alleles.py b/alembic/versions/a1d4f7c2e9b0_rewrite_v_variant_annotations_onto_alleles.py new file mode 100644 index 000000000..c8f702d72 --- /dev/null +++ b/alembic/versions/a1d4f7c2e9b0_rewrite_v_variant_annotations_onto_alleles.py @@ -0,0 +1,91 @@ +"""rewrite v_variant_annotations onto the mapping_records/alleles substrate + +Revision ID: a1d4f7c2e9b0 +Revises: e7b2c9a1f4d3 +Create Date: 2026-07-18 + +Drops the legacy ``mapped_variants``-based ``v_variant_annotations`` view and recreates it over the +``MappingRecord`` / ``Allele`` / ``MappingRecordAllele`` substrate (part of the MappedVariant +read-path teardown). Column changes vs. the legacy shape: + * ``mapped_variant_id`` -> ``mapping_record_id`` (the mapping identity is now the mapping record). + * ``mapping_error`` dropped — there is no per-mapping-record error column; per-subject annotation + dispositions live in ``v_current_annotation_events`` and score-set-level mapping errors on + ``scoresets.mapping_errors``. + +Both directions are frozen inline as literal SQL — the migration imports nothing from the model layer. +A migration is an immutable historical record: importing ``variant_annotation_view.definition`` would +couple this fixed revision to mutable code, so the next model edit (the following view rewrite) would +make a replay of *this* revision build that later shape — against tables that may not exist yet at this +point in history — and fail. ``_UPGRADE_DEFINITION`` below was compiled once from the model at +authoring time (``CreateView(signature, definition).compile(postgresql.dialect())``) and must not be +edited to track the model; the next shape change gets its own migration. +""" + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "a1d4f7c2e9b0" +down_revision = "e7b2c9a1f4d3" +branch_labels = None +depends_on = None + +SIGNATURE = "v_variant_annotations" + +# New (mapping_records/alleles) definition, frozen at this revision. Compiled from the model once at +# authoring time; do not edit to track the model. Allele-derived columns are correlated scalar +# subqueries so the grain stays one row per (variant, annotation_type), matching the legacy view. +_UPGRADE_DEFINITION = """ +SELECT variants.urn AS variant_urn, scoresets.urn AS score_set_urn, variants.hgvs_nt, variants.hgvs_pro, + variants.hgvs_splice, mapping_records.id AS mapping_record_id, (SELECT alleles.clingen_allele_id +FROM mapping_record_alleles JOIN alleles ON alleles.id = mapping_record_alleles.allele_id +WHERE mapping_record_alleles.mapping_record_id = mapping_records.id AND mapping_record_alleles.is_authoritative IS true AND mapping_record_alleles.valid_to IS NULL + LIMIT 1) AS clingen_allele_id, mapping_records.hgvs_assay_level, (SELECT alleles.hgvs_g +FROM mapping_record_alleles JOIN alleles ON alleles.id = mapping_record_alleles.allele_id +WHERE mapping_record_alleles.mapping_record_id = mapping_records.id AND mapping_record_alleles.valid_to IS NULL AND alleles.level = 'genomic' + LIMIT 1) AS hgvs_g, (SELECT alleles.hgvs_c +FROM mapping_record_alleles JOIN alleles ON alleles.id = mapping_record_alleles.allele_id +WHERE mapping_record_alleles.mapping_record_id = mapping_records.id AND mapping_record_alleles.valid_to IS NULL AND alleles.level = 'cdna' + LIMIT 1) AS hgvs_c, (SELECT alleles.hgvs_p +FROM mapping_record_alleles JOIN alleles ON alleles.id = mapping_record_alleles.allele_id +WHERE mapping_record_alleles.mapping_record_id = mapping_records.id AND mapping_record_alleles.valid_to IS NULL AND alleles.level = 'protein' + LIMIT 1) AS hgvs_p, (SELECT vep_allele_consequences.functional_consequence +FROM mapping_record_alleles JOIN alleles ON alleles.id = mapping_record_alleles.allele_id JOIN vep_allele_consequences ON vep_allele_consequences.allele_id = alleles.id AND vep_allele_consequences.valid_to IS NULL +WHERE mapping_record_alleles.mapping_record_id = mapping_records.id AND mapping_record_alleles.is_authoritative IS true AND mapping_record_alleles.valid_to IS NULL + LIMIT 1) AS vep_functional_consequence, (SELECT vep_allele_consequences.access_date +FROM mapping_record_alleles JOIN alleles ON alleles.id = mapping_record_alleles.allele_id JOIN vep_allele_consequences ON vep_allele_consequences.allele_id = alleles.id AND vep_allele_consequences.valid_to IS NULL +WHERE mapping_record_alleles.mapping_record_id = mapping_records.id AND mapping_record_alleles.is_authoritative IS true AND mapping_record_alleles.valid_to IS NULL + LIMIT 1) AS vep_access_date, mapping_records.mapped_date, mapping_records.mapping_api_version, mapping_records.vrs_version, variant_annotation_status.annotation_type, variant_annotation_status.status AS annotation_status, variant_annotation_status.failure_category, variant_annotation_status.error_message AS annotation_error, variant_annotation_status.version AS annotation_version, variant_annotation_status.annotation_metadata, variant_annotation_status.current, variant_annotation_status.created_at AS annotation_created_at, variant_annotation_status.updated_at AS annotation_updated_at +FROM variants JOIN scoresets ON scoresets.id = variants.scoreset_id + LEFT OUTER JOIN mapping_records ON mapping_records.variant_id = variants.id AND mapping_records.valid_to IS NULL + LEFT OUTER JOIN variant_annotation_status ON variant_annotation_status.variant_id = variants.id AND variant_annotation_status.current = true +""" + +# Legacy (mapped_variants-based) definition, inlined so the downgrade can restore the prior shape. +# ``mapped_variants`` still exists at this revision (dropped in a later teardown migration), so this +# remains runnable on downgrade. +_LEGACY_DEFINITION = """ +SELECT variants.urn AS variant_urn, scoresets.urn AS score_set_urn, variants.hgvs_nt, variants.hgvs_pro, + variants.hgvs_splice, mapped_variants.id AS mapped_variant_id, mapped_variants.clingen_allele_id, + mapped_variants.hgvs_assay_level, mapped_variants.hgvs_g, mapped_variants.hgvs_c, mapped_variants.hgvs_p, + mapped_variants.vep_functional_consequence, mapped_variants.vep_access_date, mapped_variants.mapped_date, + mapped_variants.mapping_api_version, mapped_variants.vrs_version, mapped_variants.error_message AS mapping_error, + variant_annotation_status.annotation_type, variant_annotation_status.status AS annotation_status, + variant_annotation_status.failure_category, variant_annotation_status.error_message AS annotation_error, + variant_annotation_status.version AS annotation_version, variant_annotation_status.annotation_metadata, + variant_annotation_status.current, variant_annotation_status.created_at AS annotation_created_at, + variant_annotation_status.updated_at AS annotation_updated_at +FROM variants JOIN scoresets ON scoresets.id = variants.scoreset_id + LEFT OUTER JOIN mapped_variants ON mapped_variants.variant_id = variants.id AND mapped_variants.current = true + LEFT OUTER JOIN variant_annotation_status + ON variant_annotation_status.variant_id = variants.id AND variant_annotation_status.current = true +""" + + +def upgrade() -> None: + op.execute(f"DROP VIEW {SIGNATURE}") + op.execute(f"CREATE VIEW {SIGNATURE} AS {_UPGRADE_DEFINITION}") + + +def downgrade() -> None: + op.execute(f"DROP VIEW {SIGNATURE}") + op.execute(f"CREATE VIEW {SIGNATURE} AS {_LEGACY_DEFINITION}") diff --git a/alembic/versions/b2e5a8d3f0c1_rebuild_published_variants_mv_onto_records.py b/alembic/versions/b2e5a8d3f0c1_rebuild_published_variants_mv_onto_records.py new file mode 100644 index 000000000..2f9dad721 --- /dev/null +++ b/alembic/versions/b2e5a8d3f0c1_rebuild_published_variants_mv_onto_records.py @@ -0,0 +1,86 @@ +"""rebuild published_variants MV onto the mapping_records substrate + +Revision ID: b2e5a8d3f0c1 +Revises: a1d4f7c2e9b0 +Create Date: 2026-07-18 + +Rebuilds the ``published_variants_materialized_view`` off the legacy ``mapped_variants`` join onto the +``mapping_records`` substrate (part of the MappedVariant read-path teardown). The MV keeps its grain +(one row per published variant per mapping-record version, unmapped variants retained with a NULL +mapping id) and its published filter. Column changes: + * ``mapped_variant_id`` -> ``mapping_record_id`` + * ``current_mapped_variant`` -> ``current_mapping_record`` (``valid_to IS NULL``) + +Both directions are frozen inline as literal SQL — the migration imports nothing from the model layer. +A migration is an immutable historical record: importing ``published_variant.definition`` would couple +this fixed revision to mutable code, so a later model edit would make a replay of *this* revision build +that later shape (possibly against tables that no longer exist by then) and fail. ``_UPGRADE_DEFINITION`` +below was compiled once from the model at authoring time and must not be edited to track the model. +""" + +from alembic_utils.pg_materialized_view import PGMaterializedView + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "b2e5a8d3f0c1" +down_revision = "a1d4f7c2e9b0" +branch_labels = None +depends_on = None + +SIGNATURE = "published_variants_materialized_view" + +# New (mapping_records-based) definition, frozen at this revision. Compiled from the model once at +# authoring time; do not edit to track the model. The LEFT OUTER JOIN to mapping_records carries no +# liveness filter on purpose — every record version is retained and current-ness is exposed via the +# ``current_mapping_record`` flag (``valid_to IS NULL``). +_UPGRADE_DEFINITION = """ +SELECT variants.id AS variant_id, variants.urn AS variant_urn, mapping_records.id AS mapping_record_id, + scoresets.id AS score_set_id, scoresets.urn AS score_set_urn, scoresets.published_date AS published_date, + mapping_records.valid_to IS NULL AS current_mapping_record +FROM variants LEFT OUTER JOIN mapping_records ON variants.id = mapping_records.variant_id + JOIN scoresets ON scoresets.id = variants.scoreset_id +WHERE scoresets.published_date IS NOT NULL +""" + +# Legacy (mapped_variants-based) definition, inlined for the downgrade path. ``mapped_variants`` still +# exists at this revision (dropped in a later teardown migration), so downgrade remains runnable. +_LEGACY_DEFINITION = """ +SELECT variants.id AS variant_id, variants.urn AS variant_urn, mapped_variants.id AS mapped_variant_id, + scoresets.id AS score_set_id, scoresets.urn AS score_set_urn, scoresets.published_date AS published_date, + mapped_variants.current AS current_mapped_variant +FROM variants LEFT OUTER JOIN mapped_variants ON variants.id = mapped_variants.variant_id + JOIN scoresets ON scoresets.id = variants.scoreset_id +WHERE scoresets.published_date IS NOT NULL +""" + + +def upgrade(): + # Dropping the MV cascades to its owned indexes (the legacy idx_..._ids on mapped_variant_id). + op.drop_entity( + PGMaterializedView(schema="public", signature=SIGNATURE, definition=_LEGACY_DEFINITION, with_data=True) + ) + op.create_entity( + PGMaterializedView(schema="public", signature=SIGNATURE, definition=_UPGRADE_DEFINITION, with_data=True) + ) + op.create_index( + f"idx_{SIGNATURE}_ids", + SIGNATURE, + ["mapping_record_id", "variant_id", "score_set_id"], + unique=True, + ) + + +def downgrade(): + op.drop_entity( + PGMaterializedView(schema="public", signature=SIGNATURE, definition=_UPGRADE_DEFINITION, with_data=True) + ) + op.create_entity( + PGMaterializedView(schema="public", signature=SIGNATURE, definition=_LEGACY_DEFINITION, with_data=True) + ) + op.create_index( + f"idx_{SIGNATURE}_ids", + SIGNATURE, + ["mapped_variant_id", "variant_id", "score_set_id"], + unique=True, + ) diff --git a/alembic/versions/b85bc7b1bec7_materialized_view_for_variant_statistics.py b/alembic/versions/b85bc7b1bec7_materialized_view_for_variant_statistics.py index 7ce51c88d..3b3855e7a 100644 --- a/alembic/versions/b85bc7b1bec7_materialized_view_for_variant_statistics.py +++ b/alembic/versions/b85bc7b1bec7_materialized_view_for_variant_statistics.py @@ -2,13 +2,16 @@ Revision ID: b85bc7b1bec7 Revises: c404b6719110 Create Date: 2025-03-14 01:53:19.898198 + +The MV SQL is inlined as a frozen literal rather than imported from the model. At this revision the +MV joins the (now legacy) ``mapped_variants`` table; a later migration rebuilds it onto the +``mapping_records`` substrate. Importing the live model definition here would make a fresh replay +build *that* later shape at this early revision — before the substrate tables exist — and fail. +Freezing the historical SQL keeps replay green; the rebuild happens in its own migration. """ from alembic import op from alembic_utils.pg_materialized_view import PGMaterializedView -from sqlalchemy.dialects import postgresql - -from mavedb.models.published_variant import signature, definition # revision identifiers, used by Alembic. @@ -17,59 +20,71 @@ branch_labels = None depends_on = None +SIGNATURE = "published_variants_materialized_view" + +# Historical definition, frozen at this revision. Do not edit to track the model. +DEFINITION = """ +SELECT variants.id AS variant_id, variants.urn AS variant_urn, mapped_variants.id AS mapped_variant_id, + scoresets.id AS score_set_id, scoresets.urn AS score_set_urn, scoresets.published_date AS published_date, + mapped_variants.current AS current_mapped_variant +FROM variants LEFT OUTER JOIN mapped_variants ON variants.id = mapped_variants.variant_id + JOIN scoresets ON scoresets.id = variants.scoreset_id +WHERE scoresets.published_date IS NOT NULL +""" + def upgrade(): op.create_entity( PGMaterializedView( schema="public", - signature=signature, - definition=definition.compile(dialect=postgresql.dialect()).string, + signature=SIGNATURE, + definition=DEFINITION, with_data=True, ) ) op.create_index( - f"idx_{signature}_variant_id", - signature, + f"idx_{SIGNATURE}_variant_id", + SIGNATURE, ["variant_id"], unique=False, ) op.create_index( - f"idx_{signature}_variant_urn", - signature, + f"idx_{SIGNATURE}_variant_urn", + SIGNATURE, ["variant_urn"], unique=False, ) op.create_index( - f"idx_{signature}_score_set_id", - signature, + f"idx_{SIGNATURE}_score_set_id", + SIGNATURE, ["score_set_id"], unique=False, ) op.create_index( - f"idx_{signature}_score_set_urn", - signature, + f"idx_{SIGNATURE}_score_set_urn", + SIGNATURE, ["score_set_urn"], unique=False, ) op.create_index( - f"idx_{signature}_mapped_variant_id", - signature, + f"idx_{SIGNATURE}_mapped_variant_id", + SIGNATURE, ["mapped_variant_id"], unique=True, ) def downgrade(): - op.drop_index(f"idx_{signature}_variant_id", signature) - op.drop_index(f"idx_{signature}_variant_urn", signature) - op.drop_index(f"idx_{signature}_mapped_variant_id", signature) - op.drop_index(f"idx_{signature}_score_set_id", signature) - op.drop_index(f"idx_{signature}_score_set_urn", signature) + op.drop_index(f"idx_{SIGNATURE}_variant_id", SIGNATURE) + op.drop_index(f"idx_{SIGNATURE}_variant_urn", SIGNATURE) + op.drop_index(f"idx_{SIGNATURE}_mapped_variant_id", SIGNATURE) + op.drop_index(f"idx_{SIGNATURE}_score_set_id", SIGNATURE) + op.drop_index(f"idx_{SIGNATURE}_score_set_urn", SIGNATURE) op.drop_entity( PGMaterializedView( schema="public", - signature=signature, - definition=definition.compile(dialect=postgresql.dialect()).string, + signature=SIGNATURE, + definition=DEFINITION, with_data=True, ) ) diff --git a/alembic/versions/f2a1b3c4d5e6_add_v_variant_annotations_view.py b/alembic/versions/f2a1b3c4d5e6_add_v_variant_annotations_view.py index 12ffd574d..2d377def7 100644 --- a/alembic/versions/f2a1b3c4d5e6_add_v_variant_annotations_view.py +++ b/alembic/versions/f2a1b3c4d5e6_add_v_variant_annotations_view.py @@ -7,23 +7,46 @@ Creates the v_variant_annotations convenience view, which joins variants, mapped_variants, variant_annotation_status, and scoresets into a single flat row per (variant, annotation_type). Intended for operator queries and the variant_annotations CLI script. + +The view SQL is inlined as a frozen literal rather than imported from the model. At this point in +history the view joins the (now legacy) ``mapped_variants`` table; a later migration rewrites it onto +the ``mapping_records`` / ``alleles`` substrate. Importing the live model definition here would make a +fresh replay build *that* later shape at this early revision — before the substrate tables exist — +and fail. Freezing the historical SQL keeps replay green; the rewrite happens in its own migration. """ from alembic import op -from mavedb.db.view import CreateView, DropView -from mavedb.models.variant_annotation_view import definition, signature - # revision identifiers, used by Alembic. revision = "f2a1b3c4d5e6" down_revision = "e3a7b9f1d2c5" branch_labels = None depends_on = None +SIGNATURE = "v_variant_annotations" + +# Historical definition, frozen at this revision. Do not edit to track the model. +DEFINITION = """ +SELECT variants.urn AS variant_urn, scoresets.urn AS score_set_urn, variants.hgvs_nt, variants.hgvs_pro, + variants.hgvs_splice, mapped_variants.id AS mapped_variant_id, mapped_variants.clingen_allele_id, + mapped_variants.hgvs_assay_level, mapped_variants.hgvs_g, mapped_variants.hgvs_c, mapped_variants.hgvs_p, + mapped_variants.vep_functional_consequence, mapped_variants.vep_access_date, mapped_variants.mapped_date, + mapped_variants.mapping_api_version, mapped_variants.vrs_version, mapped_variants.error_message AS mapping_error, + variant_annotation_status.annotation_type, variant_annotation_status.status AS annotation_status, + variant_annotation_status.failure_category, variant_annotation_status.error_message AS annotation_error, + variant_annotation_status.version AS annotation_version, variant_annotation_status.annotation_metadata, + variant_annotation_status.current, variant_annotation_status.created_at AS annotation_created_at, + variant_annotation_status.updated_at AS annotation_updated_at +FROM variants JOIN scoresets ON scoresets.id = variants.scoreset_id + LEFT OUTER JOIN mapped_variants ON mapped_variants.variant_id = variants.id AND mapped_variants.current = true + LEFT OUTER JOIN variant_annotation_status + ON variant_annotation_status.variant_id = variants.id AND variant_annotation_status.current = true +""" + def upgrade() -> None: - op.execute(CreateView(signature, definition, materialized=False)) + op.execute(f"CREATE VIEW {SIGNATURE} AS {DEFINITION}") def downgrade() -> None: - op.execute(DropView(signature, materialized=False)) + op.execute(f"DROP VIEW {SIGNATURE}") diff --git a/src/mavedb/models/published_variant.py b/src/mavedb/models/published_variant.py index eff749bd7..503d3fa30 100644 --- a/src/mavedb/models/published_variant.py +++ b/src/mavedb/models/published_variant.py @@ -1,25 +1,29 @@ -from sqlalchemy import select, join +from sqlalchemy import join, select from mavedb.db.view import MaterializedView, view - +from mavedb.models.mapping_record import MappingRecord from mavedb.models.score_set import ScoreSet from mavedb.models.variant import Variant -from mavedb.models.mapped_variant import MappedVariant - signature = "published_variants_materialized_view" + +# One row per (published variant, mapping-record version). The LEFT OUTER JOIN to mapping_records +# carries NO liveness filter on purpose: every record version (live and superseded) is retained so +# the statistics layer can count either the current mappings or the full history via the +# ``current_mapping_record`` flag (``valid_to IS NULL``). Unmapped variants stay in the view with a +# NULL ``mapping_record_id``. definition = ( select( Variant.id.label("variant_id"), Variant.urn.label("variant_urn"), - MappedVariant.id.label("mapped_variant_id"), + MappingRecord.id.label("mapping_record_id"), ScoreSet.id.label("score_set_id"), ScoreSet.urn.label("score_set_urn"), ScoreSet.published_date.label("published_date"), - MappedVariant.current.label("current_mapped_variant"), + MappingRecord.valid_to.is_(None).label("current_mapping_record"), ) .select_from( - join(Variant, MappedVariant, Variant.id == MappedVariant.variant_id, isouter=True).join( + join(Variant, MappingRecord, Variant.id == MappingRecord.variant_id, isouter=True).join( ScoreSet, ScoreSet.id == Variant.score_set_id ) ) @@ -38,8 +42,8 @@ class PublishedVariantsMV(MaterializedView): variant_id = __table__.c.variant_id variant_urn = __table__.c.variant_urn - mapped_variant_id = __table__.c.mapped_variant_id + mapping_record_id = __table__.c.mapping_record_id score_set_id = __table__.c.score_set_id score_set_urn = __table__.c.score_set_urn published_date = __table__.c.published_date - current_mapped_variant = __table__.c.current_mapped_variant + current_mapping_record = __table__.c.current_mapping_record diff --git a/src/mavedb/models/variant_annotation_view.py b/src/mavedb/models/variant_annotation_view.py index c7c77d6f8..6b6e52963 100644 --- a/src/mavedb/models/variant_annotation_view.py +++ b/src/mavedb/models/variant_annotation_view.py @@ -2,13 +2,89 @@ from mavedb.db.base import Base from mavedb.db.view import view -from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.allele import Allele +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.score_set import ScoreSet from mavedb.models.variant import Variant from mavedb.models.variant_annotation_status import VariantAnnotationStatus +from mavedb.models.vep_allele_consequence import VepAlleleConsequence signature = "v_variant_annotations" +# All allele-derived columns are correlated scalar subqueries keyed on the outer MappingRecord, so the +# outer query's FROM stays Variant / ScoreSet / MappingRecord / VariantAnnotationStatus and never +# references Allele / MappingRecordAllele directly. That keeps the grain at one row per +# (variant, annotation_type) — matching the legacy view — and, crucially, avoids ``aliased()`` at +# module import (which would force full mapper configuration before every model is registered, breaking +# a bare ``import`` of this module by Alembic). + + +def _auth_allele(column): + """Scalar subquery: ``column`` of the outer record's live **authoritative** (measured) allele. + + Source of the single-valued ``clingen_allele_id``. The authoritative link is 1:1 with the record + (the mapping job upholds one live authoritative link per record), so this resolves a single allele. + """ + return ( + select(column) + .select_from(MappingRecordAllele) + .join(Allele, Allele.id == MappingRecordAllele.allele_id) + .where( + MappingRecordAllele.mapping_record_id == MappingRecord.id, + MappingRecordAllele.is_authoritative.is_(True), + MappingRecordAllele.valid_to.is_(None), + ) + .limit(1) + .correlate(MappingRecord) + .scalar_subquery() + ) + + +def _auth_vep(column): + """Scalar subquery: ``column`` of the live VEP consequence on the record's authoritative allele.""" + return ( + select(column) + .select_from(MappingRecordAllele) + .join(Allele, Allele.id == MappingRecordAllele.allele_id) + .join( + VepAlleleConsequence, + and_(VepAlleleConsequence.allele_id == Allele.id, VepAlleleConsequence.valid_to.is_(None)), + ) + .where( + MappingRecordAllele.mapping_record_id == MappingRecord.id, + MappingRecordAllele.is_authoritative.is_(True), + MappingRecordAllele.valid_to.is_(None), + ) + .limit(1) + .correlate(MappingRecord) + .scalar_subquery() + ) + + +def _record_hgvs(level: str, column): + """Scalar subquery: the canonical HGVS at ``level`` for the outer record. + + Each mapped level (genomic / cdna / protein) is a distinct deduplicated :class:`Allele` linked to + the record. This constructs a flat ``hgvs_g`` / ``hgvs_c`` / ``hgvs_p`` triple by pivoting the record's + live allele links by level — yielding ``NULL`` where the record has no allele at that level. + """ + return ( + select(column) + .select_from(MappingRecordAllele) + .join(Allele, Allele.id == MappingRecordAllele.allele_id) + .where( + MappingRecordAllele.mapping_record_id == MappingRecord.id, + MappingRecordAllele.valid_to.is_(None), + Allele.level == level, + ) + .limit(1) + .correlate(MappingRecord) + .scalar_subquery() + ) + + +# Flat operator/CLI convenience view: one row per (variant, current annotation_type). definition = ( select( Variant.urn.label("variant_urn"), @@ -16,18 +92,17 @@ Variant.hgvs_nt, Variant.hgvs_pro, Variant.hgvs_splice, - MappedVariant.id.label("mapped_variant_id"), - MappedVariant.clingen_allele_id, - MappedVariant.hgvs_assay_level, - MappedVariant.hgvs_g, - MappedVariant.hgvs_c, - MappedVariant.hgvs_p, - MappedVariant.vep_functional_consequence, - MappedVariant.vep_access_date, - MappedVariant.mapped_date, - MappedVariant.mapping_api_version, - MappedVariant.vrs_version, - MappedVariant.error_message.label("mapping_error"), + MappingRecord.id.label("mapping_record_id"), + _auth_allele(Allele.clingen_allele_id).label("clingen_allele_id"), + MappingRecord.hgvs_assay_level, + _record_hgvs("genomic", Allele.hgvs_g).label("hgvs_g"), + _record_hgvs("cdna", Allele.hgvs_c).label("hgvs_c"), + _record_hgvs("protein", Allele.hgvs_p).label("hgvs_p"), + _auth_vep(VepAlleleConsequence.functional_consequence).label("vep_functional_consequence"), + _auth_vep(VepAlleleConsequence.access_date).label("vep_access_date"), + MappingRecord.mapped_date, + MappingRecord.mapping_api_version, + MappingRecord.vrs_version, VariantAnnotationStatus.annotation_type, VariantAnnotationStatus.status.label("annotation_status"), VariantAnnotationStatus.failure_category, @@ -40,10 +115,9 @@ ) .select_from(Variant) .join(ScoreSet, ScoreSet.id == Variant.score_set_id) - .outerjoin( - MappedVariant, - and_(MappedVariant.variant_id == Variant.id, MappedVariant.current == True), # noqa: E712 - ) + # The variant's live mapping record. live-ness rides the ON clause (never a WHERE) so the outer + # join is preserved for unmapped variants — a WHERE would reject the null-record row. + .outerjoin(MappingRecord, and_(MappingRecord.variant_id == Variant.id, MappingRecord.valid_to.is_(None))) .outerjoin( VariantAnnotationStatus, and_(VariantAnnotationStatus.variant_id == Variant.id, VariantAnnotationStatus.current == True), # noqa: E712 @@ -59,7 +133,7 @@ class VariantAnnotationView(Base): hgvs_nt = __table__.c.hgvs_nt hgvs_pro = __table__.c.hgvs_pro hgvs_splice = __table__.c.hgvs_splice - mapped_variant_id = __table__.c.mapped_variant_id + mapping_record_id = __table__.c.mapping_record_id clingen_allele_id = __table__.c.clingen_allele_id hgvs_assay_level = __table__.c.hgvs_assay_level hgvs_g = __table__.c.hgvs_g @@ -70,7 +144,6 @@ class VariantAnnotationView(Base): mapped_date = __table__.c.mapped_date mapping_api_version = __table__.c.mapping_api_version vrs_version = __table__.c.vrs_version - mapping_error = __table__.c.mapping_error annotation_type = __table__.c.annotation_type annotation_status = __table__.c.annotation_status failure_category = __table__.c.failure_category diff --git a/src/mavedb/routers/statistics.py b/src/mavedb/routers/statistics.py index e52d2582d..5deea5636 100644 --- a/src/mavedb/routers/statistics.py +++ b/src/mavedb/routers/statistics.py @@ -345,7 +345,7 @@ def record_mapped_variant_counts(db: Session = Depends(get_db)) -> dict[str, int within a given record. """ variants = db.execute( - select(PublishedVariantsMV.score_set_urn, func.count(PublishedVariantsMV.mapped_variant_id)) + select(PublishedVariantsMV.score_set_urn, func.count(PublishedVariantsMV.mapping_record_id)) .group_by(PublishedVariantsMV.score_set_urn) .order_by(PublishedVariantsMV.score_set_urn) ).all() @@ -650,10 +650,10 @@ def mapped_variant_counts( """ # Fast path: total distinct mapped variants (optionally only current) without per-date aggregation. if group is None: - total_stmt = select(func.count(func.distinct(PublishedVariantsMV.mapped_variant_id))) + total_stmt = select(func.count(func.distinct(PublishedVariantsMV.mapping_record_id))) if onlyCurrent: - total_stmt = total_stmt.where(PublishedVariantsMV.current_mapped_variant.is_(True)) + total_stmt = total_stmt.where(PublishedVariantsMV.current_mapping_record.is_(True)) total = db.execute(total_stmt).scalar_one() # type: ignore return OrderedDict([("count", total)]) @@ -661,11 +661,11 @@ def mapped_variant_counts( # Grouped path: materialize distinct counts per published_date, then roll up. per_date_stmt = select( PublishedVariantsMV.published_date, - func.count(func.distinct(PublishedVariantsMV.mapped_variant_id)), + func.count(func.distinct(PublishedVariantsMV.mapping_record_id)), ) if onlyCurrent: - per_date_stmt = per_date_stmt.where(PublishedVariantsMV.current_mapped_variant.is_(True)) + per_date_stmt = per_date_stmt.where(PublishedVariantsMV.current_mapping_record.is_(True)) per_date = db.execute( per_date_stmt.group_by(PublishedVariantsMV.published_date).order_by(PublishedVariantsMV.published_date) diff --git a/tests/conftest.py b/tests/conftest.py index 34e366392..d2825fcdb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -95,11 +95,11 @@ def session(postgresql): Base.metadata.create_all(bind=engine) # Create a unique index for the published_variants_materialized_view to - # enforce uniqueness on (variant_id, mapped_variant_id, score_set_id). This + # enforce uniqueness on (variant_id, mapping_record_id, score_set_id). This # allows us to test mat view refreshes that require this constraint. session.execute( text("""CREATE UNIQUE INDEX IF NOT EXISTS published_variants_mv_unique_idx - ON published_variants_materialized_view (variant_id, mapped_variant_id, score_set_id)"""), + ON published_variants_materialized_view (variant_id, mapping_record_id, score_set_id)"""), ) session.commit() diff --git a/tests/models/test_variant_annotation_view.py b/tests/models/test_variant_annotation_view.py new file mode 100644 index 000000000..d1c2f1d72 --- /dev/null +++ b/tests/models/test_variant_annotation_view.py @@ -0,0 +1,115 @@ +# ruff: noqa: E402 +"""Tests for the v_variant_annotations view after its rewrite onto the MappingRecord/Allele substrate. + +The view is a flat operator convenience projection: one row per (variant, current annotation_type). It +reconstructs the legacy ``mapped_variants`` row's flat ``hgvs_g``/``hgvs_c``/``hgvs_p`` triple by +pivoting the record's live allele links by level, and carries the single-valued ``clingen_allele_id`` +and VEP consequence from the record's authoritative allele. +""" + +import pytest + +pytest.importorskip("psycopg2") + +from sqlalchemy import select + +from mavedb.models.variant import Variant +from mavedb.models.variant_annotation_view import VariantAnnotationView +from tests.helpers.constants import TEST_MINIMAL_VARIANT +from tests.helpers.util.annotation import AlleleSpec, seed_mapping_record + + +def _rows_for(session, variant_urn): + # Select explicit columns (not the whole entity): the view is keyless, so an all-null-identity row + # (an unmapped variant) would materialize as a single ``None`` entity. Column rows sidestep that. + stmt = select( + VariantAnnotationView.mapping_record_id, + VariantAnnotationView.hgvs_assay_level, + VariantAnnotationView.hgvs_g, + VariantAnnotationView.hgvs_c, + VariantAnnotationView.hgvs_p, + VariantAnnotationView.clingen_allele_id, + VariantAnnotationView.vep_functional_consequence, + ).where(VariantAnnotationView.variant_urn == variant_urn) + return list(session.execute(stmt).all()) + + +def test_reconstructs_flat_triple_and_authoritative_fields(session, setup_lib_db_with_variant): + """A cdna-assay record with genomic/cdna/protein alleles fills all three HGVS slots; the + authoritative (cdna) allele supplies clingen_allele_id and the VEP consequence.""" + variant = setup_lib_db_with_variant + record = seed_mapping_record( + session, + variant, + assay_level="cdna", + hgvs_assay_level="NM_000001.1:c.1A>T", + alleles=[ + AlleleSpec( + digest="d-cdna", + level="cdna", + is_authoritative=True, + clingen_allele_id="CA123", + hgvs_c="NM_000001.1:c.1A>T", + vep_consequence="missense_variant", + ), + AlleleSpec(digest="d-genomic", level="genomic", hgvs_g="NC_000001.11:g.100A>T"), + AlleleSpec(digest="d-protein", level="protein", hgvs_p="NP_000001.1:p.Met1Leu"), + ], + ) + + rows = _rows_for(session, variant.urn) + assert len(rows) == 1 + row = rows[0] + assert row.mapping_record_id == record.id + assert row.hgvs_assay_level == "NM_000001.1:c.1A>T" + assert row.hgvs_g == "NC_000001.11:g.100A>T" + assert row.hgvs_c == "NM_000001.1:c.1A>T" + assert row.hgvs_p == "NP_000001.1:p.Met1Leu" + assert row.clingen_allele_id == "CA123" + assert row.vep_functional_consequence == "missense_variant" + + +def test_protein_assay_leaves_nucleotide_slots_null(session, setup_lib_db_with_variant): + """A protein-assay record has only a protein allele: hgvs_p is filled, the nucleotide slots stay + null, and the authoritative protein allele still supplies clingen_allele_id.""" + variant = setup_lib_db_with_variant + seed_mapping_record( + session, + variant, + assay_level="protein", + hgvs_assay_level="NP_000001.1:p.Met1Leu", + alleles=[ + AlleleSpec( + digest="d-prot-only", + level="protein", + is_authoritative=True, + clingen_allele_id="PA9", + hgvs_p="NP_000001.1:p.Met1Leu", + ), + ], + ) + + row = _rows_for(session, variant.urn)[0] + assert row.hgvs_p == "NP_000001.1:p.Met1Leu" + assert row.hgvs_g is None + assert row.hgvs_c is None + assert row.clingen_allele_id == "PA9" + + +def test_unmapped_variant_retained_with_null_mapping(session, setup_lib_db_with_score_set): + """The outer join keeps a variant with no live mapping record — it still appears with a null + mapping_record_id and empty mapped fields.""" + variant = Variant( + **TEST_MINIMAL_VARIANT, urn=f"{setup_lib_db_with_score_set.urn}#7", score_set_id=setup_lib_db_with_score_set.id + ) + session.add(variant) + session.commit() + session.refresh(variant) + + row = _rows_for(session, variant.urn)[0] + assert row.mapping_record_id is None + assert row.hgvs_g is None + assert row.hgvs_c is None + assert row.hgvs_p is None + assert row.clingen_allele_id is None + assert row.vep_functional_consequence is None diff --git a/tests/routers/test_statistics.py b/tests/routers/test_statistics.py index 69be6ffba..21ae62676 100644 --- a/tests/routers/test_statistics.py +++ b/tests/routers/test_statistics.py @@ -21,7 +21,12 @@ TEST_PUBMED_IDENTIFIER, VALID_GENE, ) -from tests.helpers.util.score_set import publish_score_set, create_acc_score_set, create_seq_score_set +from tests.helpers.util.score_set import ( + publish_score_set, + create_acc_score_set, + create_seq_score_set, + seed_annotation_substrate, +) from tests.helpers.util.experiment import create_experiment from tests.helpers.util.variant import mock_worker_variant_insertion, create_mapped_variants_for_score_set @@ -65,7 +70,12 @@ def setup_seq_scoreset(setup_router_db, session, data_provider, client, data_fil unpublished_score_set = mock_worker_variant_insertion( client, session, data_provider, unpublished_score_set, data_files / "scores.csv" ) + # Fully-mapped state: target post_mapped_metadata + mapping_state (read by the target-gene + # statistics). MappedVariant rows are seeded here too but are no longer read by the MV. create_mapped_variants_for_score_set(session, unpublished_score_set["urn"], TEST_MINIMAL_MAPPED_VARIANT) + # The published_variants MV reads the MappingRecord/Allele substrate now (not MappedVariant), so + # give each variant a live mapping record for the mapped-variant statistics to count. + seed_annotation_substrate(session, unpublished_score_set) with patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as worker_queue: publish_score_set(client, unpublished_score_set["urn"]) From 9a3ded6fb4a6a79eb800974124f24149c3a7196d Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 21 Jul 2026 23:41:03 -0700 Subject: [PATCH 099/252] perf(gnomad): batch allele lookup in gnomAD linker to end per-row scans link_gnomad_variants_to_alleles queried alleles once per gnomAD variant row inside the loop. The predicate wraps clingen_allele_id in regexp_replace to bridge zero-padded (MaveDB) and stripped (gnomAD dump, #722) CAIDs, which is non-sargable and forces a sequential scan of the alleles table on every execution. Over a linking run this becomes N full scans, saturating database IO (observed as sustained IO:DataFileRead above max vCPUs on the enlarged post-backfill alleles table). Resolve all rows in a single IN scan before the loop and group the result in Python by normalize_caid, so the loop reads from an in-memory map. N scans collapse to one per run, with no schema change. Grouping reuses normalize_caid on both the input and result sides, so the SQL and Python normalization forms are now exercised together. - add test_links_a_batch_of_rows_in_one_pass covering multi-row batching: distinct CAIDs, zero-padding match (#722), and shared-CAID fan-out preserved across a single call --- src/mavedb/lib/gnomad.py | 23 +++++++++++------ tests/lib/test_gnomad.py | 53 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 67 insertions(+), 9 deletions(-) diff --git a/src/mavedb/lib/gnomad.py b/src/mavedb/lib/gnomad.py index 5fd4d888c..9985e9934 100644 --- a/src/mavedb/lib/gnomad.py +++ b/src/mavedb/lib/gnomad.py @@ -1,6 +1,7 @@ import logging import os import re +from collections import defaultdict from enum import Enum from typing import Any, Sequence, Union @@ -188,20 +189,26 @@ def link_gnomad_variants_to_alleles( save_to_logging_context({"num_gnomad_variant_rows": len(gnomad_variant_data)}) logger.debug(msg="Linking gnomAD variants to alleles", extra=logging_context()) + # Resolve all rows' alleles in one query, then group in Python. The regexp_replace predicate + # (normalizing the zero-padded stored CAID to the dump's stripped form, #722) is non-sargable, so + # a per-row query would seq-scan `alleles` once per variant; one IN scan replaces those N scans. + target_caids = {normalize_caid(row.caid) for row in gnomad_variant_data} + alleles_by_caid: dict[str, list[Allele]] = defaultdict(list) + if target_caids: + for allele in db.scalars( + select(Allele).where( + func.regexp_replace(Allele.clingen_allele_id, _CAID_LEADING_ZERO_RE, r"\1\2").in_(target_caids) + ) + ).all(): + alleles_by_caid[normalize_caid(allele.clingen_allele_id)].append(allele) + verdicts: dict[int, GnomadLinkVerdict] = {} for index, row in enumerate(gnomad_variant_data, start=1): logger.info( msg=f"Processing gnomAD variant row {index}/{len(gnomad_variant_data)}: {row.caid}", extra=logging_context() ) - # Match on the unpadded CAID: the dump's caid is already stripped, while the stored CAID may - # be zero-padded, so normalize both sides (issue #722). regexp_replace mirrors normalize_caid. - alleles_with_caid = db.scalars( - select(Allele).where( - func.regexp_replace(Allele.clingen_allele_id, _CAID_LEADING_ZERO_RE, r"\1\2") - == normalize_caid(row.caid) - ) - ).all() + alleles_with_caid = alleles_by_caid.get(normalize_caid(row.caid), []) if not alleles_with_caid: continue diff --git a/tests/lib/test_gnomad.py b/tests/lib/test_gnomad.py index f9c8b17aa..b3dc62c28 100644 --- a/tests/lib/test_gnomad.py +++ b/tests/lib/test_gnomad.py @@ -1,6 +1,6 @@ # ruff: noqa: E402 -from unittest.mock import patch +from unittest.mock import Mock, patch import pytest from sqlalchemy import select @@ -22,6 +22,7 @@ from tests.helpers.constants import ( TEST_GNOMAD_DATA_VERSION, TEST_GNOMAD_VARIANT, + TEST_MAVEDB_ATHENA_ROW, ) ### Tests for gnomad_identifier function ### @@ -148,6 +149,18 @@ def _make_allele(session, caid, *, vrs_digest, level="genomic"): return allele +def _make_gnomad_row(**overrides): + """Build an Athena-style gnomAD row Mock from the canonical test row, applying overrides. + + Mirrors the ``mocked_gnomad_variant_row`` fixture but takes overrides so a test can construct a + batch of distinct rows (different CAIDs, loci) in one call. + """ + row = Mock() + for key, value in {**TEST_MAVEDB_ATHENA_ROW, **overrides}.items(): + setattr(row, key, value) + return row + + def _live_links_for(session, allele_id): return session.scalars( select(GnomadAlleleLink).where( @@ -324,6 +337,44 @@ def test_links_allele_when_dump_strips_leading_zero_from_caid(session, mocked_gn assert len(_live_links_for(session, allele.id)) == 1 +def test_links_a_batch_of_rows_in_one_pass(session): + """Multiple rows in one call are resolved by a single batched allele lookup, then grouped by + normalized CAID. Exercises everything the batching must preserve at once: distinct CAIDs each + reach their own allele, a zero-padded stored CAID still matches a dump-stripped row (#722), and a + CAID shared across alleles fans its variant out to each — all without a per-row query.""" + # Row A: allele stored zero-padded, dump delivers the stripped form. + allele_a = _make_allele(session, "CA025094", vrs_digest="vrs-a") + row_a = _make_gnomad_row(caid="CA25094") + row_a.__setattr__("locus.position", "87961093") # identifier 10-87961093-A-G + + # Row B: a different CAID at a different locus, shared by two alleles (cross-score-set dedup). + allele_b1 = _make_allele(session, "CA341478553", vrs_digest="vrs-b1") + allele_b2 = _make_allele(session, "CA341478553", vrs_digest="vrs-b2", level="cdna") + row_b = _make_gnomad_row(caid="CA341478553") + row_b.__setattr__("locus.position", "87961094") # identifier 10-87961094-A-G + + with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): + result = link_gnomad_variants_to_alleles(session, [row_a, row_b]) + assert result == { + allele_a.id: GnomadLinkVerdict.CREATED, + allele_b1.id: GnomadLinkVerdict.CREATED, + allele_b2.id: GnomadLinkVerdict.CREATED, + } + session.commit() + + # Each allele ends with exactly one live link. + for allele in (allele_a, allele_b1, allele_b2): + assert len(_live_links_for(session, allele.id)) == 1 + + # Two distinct gnomAD variants (one per row's identifier); B's two alleles share row B's variant. + assert len(session.scalars(select(GnomADVariant)).all()) == 2 + b1_variant = _live_links_for(session, allele_b1.id)[0].gnomad_variant_id + b2_variant = _live_links_for(session, allele_b2.id)[0].gnomad_variant_id + a_variant = _live_links_for(session, allele_a.id)[0].gnomad_variant_id + assert b1_variant == b2_variant + assert a_variant != b1_variant + + def test_returns_empty_map_when_no_alleles_match(session, mocked_gnomad_variant_row): result = link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) assert result == {} From 0f6ba0b8234e6f5907631229f4bcf746d5237e74 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 22 Jul 2026 15:56:34 -0700 Subject: [PATCH 100/252] feat(scripts): gate unenriched selection on a usable transcript The --select unenriched path enqueues annotate_score_set, whose reverse-translation step needs a transcript. It previously selected any mapped-but-not-enriched score set, so ones with no usable transcript were re-selected on every run and burned worker cycles skipping every variant as transcript_unresolved. - split _needs_enrichment into _mapped_without_enrichment and a new _has_usable_transcript predicate (cdna TargetGeneMapping reference_accession, or a live NP_/XP_ RefSeq protein MappingRecord), mirroring the two transcript sources in reverse_translation.py - require a usable transcript for unenriched eligibility - report the count of mapped score sets skipped for lack of a transcript so they are visibly dropped rather than silently lost --- src/mavedb/scripts/run_score_set_pipelines.py | 74 ++++++++++++++++++- 1 file changed, 70 insertions(+), 4 deletions(-) diff --git a/src/mavedb/scripts/run_score_set_pipelines.py b/src/mavedb/scripts/run_score_set_pipelines.py index 169f17031..6a7571562 100644 --- a/src/mavedb/scripts/run_score_set_pipelines.py +++ b/src/mavedb/scripts/run_score_set_pipelines.py @@ -9,8 +9,10 @@ ones, whose records carry the old ``mapped_date`` and so need a real re-map under the upgraded pipeline. * ``--select unenriched`` → runs ``annotate_score_set`` over mapped score sets that have not yet had - the reverse-translation fan-out (no derived, non-authoritative live allele link). This is the - post-backfill enrichment half. + the reverse-translation fan-out (no derived, non-authoritative live allele link) **and** have a + transcript reverse translation can use. The transcript gate excludes score sets whose every + variant would skip as ``transcript_unresolved``, which would otherwise be re-selected forever and + waste worker cycles. This is the post-backfill enrichment half. **Throughput — gene-grouped ordering.** Score sets whose variants resolve to the same CAIDs/PAIDs cache the expensive ClinGen CAR resolution (24h TTL) and reuse already-deduplicated alleles. So by @@ -41,16 +43,18 @@ import asyncclick as click from arq import create_pool -from sqlalchemy import and_, not_, select +from sqlalchemy import and_, func, not_, or_, select from sqlalchemy.orm import Session, selectinload from mavedb.db.session import SessionLocal from mavedb.lib.workflow.kickoff import enqueue_pipeline_for_score_set from mavedb.models.enums.processing_state import ProcessingState +from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.models.mapping_record import MappingRecord from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.score_set import ScoreSet from mavedb.models.target_gene import TargetGene +from mavedb.models.target_gene_mapping import TargetGeneMapping from mavedb.models.user import User from mavedb.models.variant import Variant from mavedb.worker.settings import RedisWorkerSettings @@ -104,7 +108,7 @@ def _needs_mapping(remap_before: datetime.date): return variant_needing_mapping.exists() -def _needs_enrichment(): +def _mapped_without_enrichment(): """Predicate: the score set is mapped (a live MappingRecord) but has no reverse-translation-derived (non-authoritative) live allele link yet.""" has_live_record = ( @@ -126,6 +130,63 @@ def _needs_enrichment(): return and_(has_live_record.exists(), not_(has_derived_link.exists())) +def _has_usable_transcript(): + """Predicate: the score set has a transcript that reverse translation can use. + + Mirrors the two transcript sources in + ``worker/jobs/variant_processing/reverse_translation.py``: a cdna ``TargetGeneMapping``'s + ``reference_accession`` (the mapper-supplied coding transcript), and a RefSeq protein + (``NP_``/``XP_``) ``hgvs_assay_level`` on a live ``MappingRecord`` (resolved ``NP_``→``NM_`` via + UTA). A score set with neither would have every variant skip as ``transcript_unresolved``, so + the unenriched selection excludes it rather than re-queue it forever.""" + has_cdna_transcript = ( + select(TargetGeneMapping.id) + .join(TargetGene, TargetGene.id == TargetGeneMapping.target_gene_id) + .where( + TargetGene.score_set_id == ScoreSet.id, + TargetGeneMapping.alignment_level == SequenceLevel.cdna, + TargetGeneMapping.reference_accession.isnot(None), + ) + ) + # `_` is a LIKE wildcard, so escape it to match the literal underscore in the RefSeq prefix. + has_refseq_protein_record = ( + select(MappingRecord.id) + .join(Variant, Variant.id == MappingRecord.variant_id) + .where( + Variant.score_set_id == ScoreSet.id, + MappingRecord.valid_to.is_(None), + or_( + MappingRecord.hgvs_assay_level.like(r"NP\_%", escape="\\"), + MappingRecord.hgvs_assay_level.like(r"XP\_%", escape="\\"), + ), + ) + ) + return or_(has_cdna_transcript.exists(), has_refseq_protein_record.exists()) + + +def _needs_enrichment(): + """Predicate: the score set is mapped-but-not-enriched AND has a usable transcript, so + ``annotate_score_set``'s reverse-translation step has something to translate against.""" + return and_(_mapped_without_enrichment(), _has_usable_transcript()) + + +def count_unenrichable_without_transcript(db: Session, *, only_published: bool = True) -> int: + """Count mapped-but-not-enriched score sets that ``--select unenriched`` skips for lack of a + usable transcript, so the operator sees they were intentionally dropped (not lost).""" + query = ( + select(func.count()) + .select_from(ScoreSet) + .where( + ScoreSet.processing_state == ProcessingState.success, + _mapped_without_enrichment(), + not_(_has_usable_transcript()), + ) + ) + if only_published: + query = query.where(ScoreSet.private.is_(False), ScoreSet.published_date.isnot(None)) + return db.scalar(query) or 0 + + def select_score_sets( db: Session, *, @@ -227,6 +288,11 @@ async def main( ) click.echo(f"{len(score_sets)} score set(s) selected ({phase}) for pipeline '{pipeline}'.") + if phase == "unenriched" and not force: + skipped = count_unenrichable_without_transcript(db, only_published=not include_unpublished) + if skipped: + click.echo(f" ({skipped} mapped score set(s) skipped: no usable transcript for reverse translation.)") + if dry_run: for score_set in score_sets: click.echo(f" {_grouping_key(score_set):<12} {score_set.urn}") From 6f0a8e263da69ee5df34c404abc20ba7686e8573 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 22 Jul 2026 23:13:24 -0700 Subject: [PATCH 101/252] feat(variants): stream variant-details NDJSON, add gnomAD variant links - add preMapped/postMapped VRS pair to VariantDetail and switch GET /variants/{urn} to exclude_none=False so absent fields serialize as null instead of being dropped - add streaming NDJSON GET /score-sets/{urn}/variant-details, replacing the retired GET /score-sets/{urn}/mapped-variants - replace MappedVariant-keyed gnomAD lookups with a substrate-based get_gnomad_variants_with_variant_urns, pairing each gnomAD variant with the score-set variant URNs it links to (GnomADVariantWithVariantLinks) - add as_of time-travel support to the CSV variants export and the gnomad-variants endpoint, echoed via X-As-Of response headers - extract mapped_hgvs_by_level/get_protein_hgvs_by_record helpers out of score_set_variants for reuse by the CSV export --- src/mavedb/lib/gnomad.py | 73 ++++- src/mavedb/lib/score_set_variants.py | 132 +++++---- src/mavedb/lib/variant_detail.py | 7 + src/mavedb/routers/score_sets.py | 234 ++++++++++----- src/mavedb/routers/variants.py | 5 +- src/mavedb/view_models/gnomad_variant.py | 25 +- src/mavedb/view_models/variant_detail.py | 28 +- tests/helpers/constants.py | 2 +- tests/helpers/util/annotation.py | 5 +- tests/routers/test_score_set.py | 358 +++++++++++++++++++---- tests/routers/test_variant.py | 16 +- 11 files changed, 680 insertions(+), 205 deletions(-) diff --git a/src/mavedb/lib/gnomad.py b/src/mavedb/lib/gnomad.py index 9985e9934..830cc62d9 100644 --- a/src/mavedb/lib/gnomad.py +++ b/src/mavedb/lib/gnomad.py @@ -2,8 +2,10 @@ import os import re from collections import defaultdict +from dataclasses import dataclass +from datetime import datetime from enum import Enum -from typing import Any, Sequence, Union +from typing import Any, Optional, Sequence, Union from sqlalchemy import Connection, Row, func, select, text from sqlalchemy.orm import Session @@ -13,6 +15,9 @@ from mavedb.models.allele import Allele from mavedb.models.gnomad_allele_link import GnomadAlleleLink from mavedb.models.gnomad_variant import GnomADVariant +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.variant import Variant logger = logging.getLogger(__name__) @@ -39,6 +44,72 @@ class GnomadLinkVerdict(str, Enum): UNCHANGED = "unchanged" # a live link already pointed at the resolved variant; left untouched +@dataclass(frozen=True) +class GnomadVariantLink: + """One (score-set variant, annotated allele) pair a gnomAD frequency record reaches. + + Mirrors :class:`mavedb.lib.clinical_controls.ControlVariantLink`. ``allele_digest`` is the VRS digest + of the allele gnomAD's frequency was linked to. + """ + + variant_urn: str + allele_digest: str + + +def get_gnomad_variants_with_variant_urns( + db: Session, + score_set_id: int, + *, + as_of: Optional[datetime] = None, + db_version: Optional[str] = None, +) -> list[tuple[GnomADVariant, list[GnomadVariantLink]]]: + """The live gnomAD frequency records for a score set, each paired with the score-set variants that link + to it. + + Walks ``GnomadAlleleLink → Allele → MappingRecordAllele → MappingRecord → Variant`` with every + ``ValidTime`` hop constrained to the same instant (``as_of``, defaulting to currently-live). Optionally + narrowed to one gnomAD release (``db_version``). Records come back in first-seen order; each + record's links preserve query order. + """ + stmt = ( + select( + GnomADVariant, + Variant.urn.label("variant_urn"), + Allele.vrs_digest.label("allele_digest"), + ) + .join(GnomadAlleleLink, GnomadAlleleLink.gnomad_variant_id == GnomADVariant.id) + .join(Allele, Allele.id == GnomadAlleleLink.allele_id) + .join(MappingRecordAllele, MappingRecordAllele.allele_id == Allele.id) + .join(MappingRecord, MappingRecord.id == MappingRecordAllele.mapping_record_id) + .join(Variant, Variant.id == MappingRecord.variant_id) + .where(GnomadAlleleLink.live_at(as_of)) + .where(MappingRecordAllele.live_at(as_of)) + .where(MappingRecord.live_at(as_of)) + .where(Variant.score_set_id == score_set_id) + .distinct() + ) + if db_version is not None: + stmt = stmt.where(GnomADVariant.db_version == db_version) + + variants: dict[int, GnomADVariant] = {} + links_by_variant: dict[int, list[GnomadVariantLink]] = {} + + for gnomad_variant, variant_urn, allele_digest in db.execute(stmt).tuples(): + # TODO#372: nullable URN + if variant_urn is None: + continue + + if gnomad_variant.id not in variants: + variants[gnomad_variant.id] = gnomad_variant + links_by_variant[gnomad_variant.id] = [] + + links_by_variant[gnomad_variant.id].append( + GnomadVariantLink(variant_urn=variant_urn, allele_digest=allele_digest) + ) + + return [(gnomad_variant, links_by_variant[gnomad_variant.id]) for gnomad_variant in variants.values()] + + def gnomad_identifier(contig: str, position: Union[str, int], alleles: list[str]) -> str: """ Generate a gnomAD variant identifier based on contig, position, and alleles. diff --git a/src/mavedb/lib/score_set_variants.py b/src/mavedb/lib/score_set_variants.py index f0c75390f..c1f45c545 100644 --- a/src/mavedb/lib/score_set_variants.py +++ b/src/mavedb/lib/score_set_variants.py @@ -118,43 +118,104 @@ def _hgvs_field_for_str(hgvs: Optional[str]) -> Optional[HgvsField]: return HgvsField(hgvs=hgvs, position=block.position, ref=block.ref, alt=block.alt) -def _mapped_triple( +def mapped_hgvs_by_level( *, assay_level: Optional[SequenceLevel], assay_level_hgvs: Optional[str], sibling_level: Optional[str], sibling_hgvs: Optional[str], protein_hgvs: Optional[str], -) -> MappedTriple: - """Assemble the canonical :class:`MappedTriple` for one variant from its per-row query fields. +) -> dict[str, Optional[str]]: + """The canonical mapped HGVS **string** at each populated level slot, keyed by ``SequenceLevel`` value. - - Measured slot (``mapped[assay_level]``) ← ``assay_level_hgvs`` (the record's mapped assay HGVS). + The canonical projection of a measured allele onto various levels relative to its reference frame: + - Measured slot (``[assay_level]``) ← ``assay_level_hgvs`` (the record's mapped assay HGVS). - Other nucleotide slot ← the authoritative link's ``projection_group`` sibling (``sibling_level`` / ``sibling_hgvs``), populated only on a nucleotide assay and only where the pairing is recorded. - ``protein`` slot ← the separate protein-apex subquery. A **protein** assay fills only ``protein`` (the measured slot is protein itself); its nucleotide - fan-out is ambiguous, so ``cdna``/``genomic`` are deliberately left ``None``. An unmapped variant - (``assay_level is None``) yields an empty triple. + fan-out is ambiguous, so ``cdna``/``genomic`` are deliberately absent. An unmapped variant + (``assay_level is None``) yields an empty mapping. """ - slots: dict[str, Optional[HgvsField]] = {} - # Protein assay: the measured slot *is* protein. No canonical c/g — do not fabricate one. + slots: dict[str, Optional[str]] = {} if assay_level == SequenceLevel.protein: - slots[SequenceLevel.protein.value] = _hgvs_field_for_str(assay_level_hgvs) + slots[SequenceLevel.protein.value] = assay_level_hgvs elif assay_level in (SequenceLevel.cdna, SequenceLevel.genomic): - slots[assay_level.value] = _hgvs_field_for_str(assay_level_hgvs) + slots[assay_level.value] = assay_level_hgvs + slots[SequenceLevel.protein.value] = protein_hgvs # The other nucleotide level, from the projection_group sibling (null where unpopulated). if sibling_level is not None: - slots[sibling_level] = _hgvs_field_for_str(sibling_hgvs) - slots[SequenceLevel.protein.value] = _hgvs_field_for_str(protein_hgvs) + slots[sibling_level] = sibling_hgvs + + return slots + +def _mapped_triple( + *, + assay_level: Optional[SequenceLevel], + assay_level_hgvs: Optional[str], + sibling_level: Optional[str], + sibling_hgvs: Optional[str], + protein_hgvs: Optional[str], +) -> MappedTriple: + """Assemble the canonical :class:`MappedTriple` for one variant, wrapping each slot from + :func:`mapped_hgvs_by_level` as an :class:`HgvsField`.""" + slots = mapped_hgvs_by_level( + assay_level=assay_level, + assay_level_hgvs=assay_level_hgvs, + sibling_level=sibling_level, + sibling_hgvs=sibling_hgvs, + protein_hgvs=protein_hgvs, + ) return MappedTriple( - genomic=slots.get(SequenceLevel.genomic.value), - cdna=slots.get(SequenceLevel.cdna.value), - protein=slots.get(SequenceLevel.protein.value), + genomic=_hgvs_field_for_str(slots.get(SequenceLevel.genomic.value)), + cdna=_hgvs_field_for_str(slots.get(SequenceLevel.cdna.value)), + protein=_hgvs_field_for_str(slots.get(SequenceLevel.protein.value)), ) +def get_protein_hgvs_by_record(db: Session, score_set_id: int, *, as_of: Optional[datetime] = None) -> dict[int, str]: + """The canonical protein-level mapped HGVS for each of a score set's live mapping records, keyed by + ``mapping_record.id``. Records with no protein projection (UTR/intronic) are simply absent from the + map. Shared by the lean whole-set view and the CSV export so both resolve the protein slot the same way. + + This runs as its **own** query, never a join into a per-variant projection. Joined in, it forces an + O(N^2) plan: the ``DISTINCT ON`` cardinality is opaque to the planner, so it lands on the inner side + of a nested loop and is rescanned in full once per variant — historically ~97% of the lean endpoint's + total time. Pulling it out and stitching by ``mapping_record_id`` in Python keeps both queries O(N). + """ + prot_link = aliased(MappingRecordAllele) + prot_allele = aliased(Allele) + prot_record = aliased(MappingRecord) + prot_variant = aliased(Variant) + protein_statement = ( + select( + prot_link.mapping_record_id.label("mapping_record_id"), + prot_allele.hgvs_p.label("protein_hgvs"), + ) + # DISTINCT ON -> at most one protein level row per record. + .distinct(prot_link.mapping_record_id) + # link -> allele: level and hgvs_p live on the *allele*. + .join(prot_allele, prot_allele.id == prot_link.allele_id) + # link -> record -> variant: the bridge to scoreset_id, to scope the query. + .join(prot_record, prot_record.id == prot_link.mapping_record_id) + .join(prot_variant, prot_variant.id == prot_record.variant_id) + # Scope to this score set so the cost scales with the response. + .where(prot_variant.score_set_id == score_set_id) + # Live links only. A live link implies a live parent record (ValidTime invariant), so this also + # covers record-liveness — no prot_record.live_at needed; the MR join is just the scoping bridge. + .where(prot_link.live_at(as_of)) + # Only the protein-level allele. + .where(prot_allele.level == SequenceLevel.protein.value) + # DISTINCT ON requires ORDER BY to lead with its column; allele.id then picks which row survives + # per record — inert today (only one protein allele exists). + .order_by(prot_link.mapping_record_id, prot_allele.id) + ) + # Keyed by record id (a globally-unique PK -> the right protein row, no cross-set risk). + return {row.mapping_record_id: row.protein_hgvs for row in db.execute(protein_statement)} + + def get_lean_score_set_variants( db: Session, score_set: ScoreSet, *, as_of: Optional[datetime] = None ) -> list[LeanVariantRecord]: @@ -236,44 +297,9 @@ def get_lean_score_set_variants( .order_by(cast(func.split_part(Variant.urn, "#", 2), Integer), Variant.id) ) - # The protein projection runs as its own query, NOT a join into the base statement. Joined in, it forces - # an O(N^2) plan: the DISTINCT ON cardinality is opaque to the planner (est. ~18 vs ~11.5k actual), so it - # buried the projection on the inner side of a nested loop and rescanned it in full once per variant — - # ~97% of the endpoint's total time. MATERIALIZED in the query above fixes the *construction* but not the - # per-variant *rescan*. Pulling out and stitching these queries by mapping_record_id in Python enables both - # queries to run in O(N) time, making this function (and the endpoint it serves) considerably faster. - prot_link = aliased(MappingRecordAllele) - prot_allele = aliased(Allele) - prot_record = aliased(MappingRecord) - prot_variant = aliased(Variant) - protein_statement = ( - select( - prot_link.mapping_record_id.label("mapping_record_id"), - prot_allele.hgvs_p.label("protein_hgvs"), - ) - # DISTINCT ON -> at most one protein level row per record. - .distinct(prot_link.mapping_record_id) - # link -> allele: level and hgvs_p live on the *allele*. - .join(prot_allele, prot_allele.id == prot_link.allele_id) - # link -> record -> variant: the bridge to scoreset_id, to scope the query. - .join(prot_record, prot_record.id == prot_link.mapping_record_id) - .join(prot_variant, prot_variant.id == prot_record.variant_id) - # Scope to this score set so the cost scales with the response. - .where(prot_variant.score_set_id == score_set.id) - # Live links only. A live link implies a live parent record (ValidTime invariant), so this also - # covers record-liveness — no prot_record.live_at needed; the MR join is just the scoping bridge. - .where(prot_link.live_at(as_of)) - # Only the protein-level allele. - .where(prot_allele.level == SequenceLevel.protein.value) - # DISTINCT ON requires ORDER BY to lead with its column; allele.id then picks which row survives - # per record — inert today (only one protein allele exists). - .order_by(prot_link.mapping_record_id, prot_allele.id) - ) - # Keyed by record id (a globally-unique PK -> the right protein row, no cross-set risk). Some records - # have no protein projection (UTR/intronic) -> absent from the map -> null protein field below. - protein_hgvs_by_record: dict[int, str] = { - row.mapping_record_id: row.protein_hgvs for row in db.execute(protein_statement) - } + # The protein projection runs as its own query, NOT a join into the base statement (see + # get_protein_hgvs_by_record for why), stitched back by mapping_record_id below. + protein_hgvs_by_record = get_protein_hgvs_by_record(db, score_set.id, as_of=as_of) return [ LeanVariantRecord( diff --git a/src/mavedb/lib/variant_detail.py b/src/mavedb/lib/variant_detail.py index 567fcf226..cf9a95cc0 100644 --- a/src/mavedb/lib/variant_detail.py +++ b/src/mavedb/lib/variant_detail.py @@ -70,6 +70,11 @@ class VariantDetail: assay_level_digest: Optional[str] clingen_allele_id: Optional[str] + # The raw GA4GH VRS pair, surfaced flat so a bulk/VRS consumer reads them directly rather than + # digging the measured allele out of the Cat-VRS below. + pre_mapped: Optional[dict[str, Any]] + post_mapped: Optional[dict[str, Any]] + # Spec-pure GA4GH Cat-VRS (no MaveDB fields inside), plus the MaveDB layer riding alongside: the # per-allele identity sidecar, keyed by VRS digest — one entry per linked allele (the record-scoped, # all-levels set, sharing keys with `annotations`), carrying level + HGVS + ClinGen id + relation. @@ -258,6 +263,8 @@ def get_variant_detail( reference_hgvs=reference_hgvs, assay_level_digest=assay_level_digest, clingen_allele_id=clingen_allele_id, + pre_mapped=record.pre_mapped if record is not None else None, + post_mapped=authoritative.allele.post_mapped if authoritative is not None else None, molecular_representation=molecular_representation, mode=mode, alleles=alleles, diff --git a/src/mavedb/routers/score_sets.py b/src/mavedb/routers/score_sets.py index 7baeb0893..6bbed79f8 100644 --- a/src/mavedb/routers/score_sets.py +++ b/src/mavedb/routers/score_sets.py @@ -38,6 +38,7 @@ from mavedb.lib.contributors import find_or_create_contributor from mavedb.lib.exceptions import MixedTargetError, NonexistentOrcidUserError from mavedb.lib.experiments import enrich_experiment_with_num_score_sets +from mavedb.lib.gnomad import get_gnomad_variants_with_variant_urns from mavedb.lib.identifiers import ( create_external_gene_identifier_offset, find_or_create_doi_identifier, @@ -71,13 +72,12 @@ generate_experiment_urn, generate_score_set_urn, ) +from mavedb.lib.variant_detail import get_variant_detail from mavedb.lib.workflow.kickoff import enqueue_pipeline_for_score_set from mavedb.models.contributor import Contributor from mavedb.models.enums.processing_state import ProcessingState from mavedb.models.experiment import Experiment -from mavedb.models.gnomad_variant import GnomADVariant from mavedb.models.license import License -from mavedb.models.mapped_variant import MappedVariant from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.score_set import ScoreSet from mavedb.models.target_accession import TargetAccession @@ -92,7 +92,7 @@ PUBLIC_ERROR_RESPONSES, ROUTER_BASE_PREFIX, ) -from mavedb.view_models import clinical_control, gnomad_variant, mapped_variant, score_set +from mavedb.view_models import clinical_control, gnomad_variant, score_set from mavedb.view_models.contributor import ContributorCreate from mavedb.view_models.doi_identifier import DoiIdentifierCreate from mavedb.view_models.lean_variant import LeanVariant @@ -100,6 +100,7 @@ from mavedb.view_models.score_set_dataset_columns import DatasetColumnMetadata from mavedb.view_models.search import ScoreSetsSearch, ScoreSetsSearchFilterOptionsResponse, ScoreSetsSearchResponse from mavedb.view_models.target_gene import TargetGeneCreate +from mavedb.view_models.variant_detail import VariantDetail TAG_NAME = "Score Sets" logger = logging.getLogger(__name__) @@ -869,6 +870,108 @@ async def get_score_set_lean_variants( return get_lean_score_set_variants(db, score_set, as_of=as_of) +def _stream_score_set_variant_details( + db: Session, + variants: Sequence[Variant], + superseding_score_set: Optional[ScoreSet], + visible_calibration_ids: set[int], + *, + as_of: Optional[datetime] = None, +): + """Serialize the whole-set variant-detail export as NDJSON — one :class:`VariantDetail` per line. + + Assembles each variant's detail envelope (the flat assay fields + ``preMapped``/``postMapped`` VRS + pair + the spec-pure GA4GH CategoricalVariant + the digest-keyed VEP/gnomAD/ClinVar annotation map) + one at a time, so a large score set streams rather than building every envelope up front. + ``superseding_score_set`` / ``visible_calibration_ids`` are resolved once for the whole set and + threaded into every per-variant build. + """ + for variant in variants: + detail = get_variant_detail( + db, + variant, + superseding_score_set=superseding_score_set, + visible_calibration_ids=visible_calibration_ids, + as_of=as_of, + ) + # get_variant_detail returns the lib transit dataclass; coerce it through the view model. + # exclude_none=False keeps a stable key set per line for programmatic consumers. + yield VariantDetail.model_validate(detail).model_dump_json(by_alias=True, exclude_none=False) + "\n" + + +@router.get( + "/score-sets/{urn}/variant-details", + status_code=200, + responses={ + 200: { + "content": {"application/x-ndjson": {}}, + "description": ( + "Newline-delimited JSON: one VariantDetail per mapped variant — the same envelope the " + "single-variant GET /variants/{urn} route serves, carrying the flat preMapped/postMapped " + "VRS pair, the spec-pure GA4GH CategoricalVariant, and the digest-keyed VEP/gnomAD/ClinVar " + "annotation map." + ), + }, + **ACCESS_CONTROL_ERROR_RESPONSES, + }, + summary="Download a score set's variant details (VRS + Cat-VRS + annotations)", +) +async def get_score_set_variant_details( + *, + urn: str, + response: Response, + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the molecular layer (VRS, Cat-VRS membership + VEP/gnomAD/ClinVar annotations) " + "as it stood at this instant, over the score set's fixed scores. ISO 8601, ideally " + "timezone-aware. Content valid-time only — it never re-selects a score-set version. Defaults " + "to current." + ), + ), + db: Session = Depends(deps.get_db), + user_data: Optional[UserData] = Depends(get_current_user), +) -> Any: + """Download the score set's variant details — the whole-set streaming pair of the single-variant + ``GET /variants/{urn}`` detail endpoint, and the substrate-faithful replacement for the retired + ``/mapped-variants`` export. + + One record per *mapped* variant (unmapped variants carry no VRS and are omitted): the same + VariantDetail envelope the single-variant route serves — the flat ``preMapped``/``postMapped`` VRS + pair for VRS consumers, plus the spec-pure GA4GH CategoricalVariant and the digest-keyed + VEP/gnomAD/ClinVar annotation map for the full molecular picture. + + Streamed as NDJSON (like the annotated-variant exports) so a large score set downloads without + materializing every envelope server-side and a client can process it line by line. ``as_of`` + time-travels the molecular layer only (scores/classifications are immutable); the resolved value is + echoed in ``X-As-Of`` and the variant count in ``X-Total-Count``. + """ + save_to_logging_context({"requested_resource": urn, "resource_property": "variant-details", "as_of": as_of}) + + score_set = await fetch_score_set_by_urn(db, urn, user_data, None, False) + # fetch_score_set_by_urn already blanks a non-readable superseding version and filters + # score_calibrations to the readable ones, so both are visibility-resolved here. + visible_calibration_ids = {sc.id for sc in score_set.score_calibrations if sc.id is not None} + variants = get_annotatable_variants(db, score_set, as_of=as_of) + return StreamingResponse( + _stream_score_set_variant_details( + db, + variants, + score_set.superseding_score_set, + visible_calibration_ids, + as_of=as_of, + ), + media_type="application/x-ndjson", + headers={ + "X-As-Of": as_of.isoformat() if as_of is not None else "current", + "X-Total-Count": str(len(variants)), + "X-Processing-Started": datetime.now().isoformat(), + "X-Stream-Type": "variant-detail", + "Access-Control-Expose-Headers": "X-As-Of, X-Total-Count, X-Processing-Started, X-Stream-Type", + }, + ) + + @router.get( "/score-sets/{urn}/variants/data", status_code=200, @@ -899,6 +1002,14 @@ def get_score_set_variants_csv( drop_na_columns: Optional[bool] = None, include_custom_columns: Optional[bool] = None, include_post_mapped_hgvs: Optional[bool] = None, + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the annotation layer (post-mapped HGVS, VEP, gnomAD, ClinVar) as it stood at this " + "instant, over the variant's immutable submitted HGVS/scores/counts. ISO 8601, ideally " + "timezone-aware. No effect on the scores/counts namespaces. Defaults to current." + ), + ), db: Session = Depends(deps.get_db), user_data: Optional[UserData] = Depends(get_current_user), ) -> Any: @@ -940,6 +1051,7 @@ def get_score_set_variants_csv( "start": start, "limit": limit, "drop_na_columns": drop_na_columns, + "as_of": as_of, } ) @@ -982,8 +1094,13 @@ def get_score_set_variants_csv( drop_na_columns, include_custom_columns, include_post_mapped_hgvs, + as_of, + ) + return StreamingResponse( + iter([csv_str]), + media_type="text/csv", + headers={"X-As-Of": as_of.isoformat() if as_of is not None else "current"}, ) - return StreamingResponse(iter([csv_str]), media_type="text/csv") @router.get( @@ -1102,51 +1219,6 @@ async def get_score_set_counts_csv( return StreamingResponse(iter([csv_str]), media_type="text/csv") -@router.get( - "/score-sets/{urn}/mapped-variants", - status_code=200, - response_model=list[mapped_variant.MappedVariant], - responses={**ACCESS_CONTROL_ERROR_RESPONSES}, - summary="Get mapped variants from score set by URN", -) -def get_score_set_mapped_variants( - *, - urn: str, - db: Session = Depends(deps.get_db), - user_data: Optional[UserData] = Depends(get_current_user), -) -> list[MappedVariant]: - """ - Return mapped variants from a score set, identified by URN. - """ - save_to_logging_context({"requested_resource": urn, "resource_property": "mapped-variants"}) - - score_set = db.query(ScoreSet).filter(ScoreSet.urn == urn).first() - if not score_set: - logger.info( - msg="Could not fetch the requested mapped variants; No such score set exist.", extra=logging_context() - ) - raise HTTPException(status_code=404, detail=f"score set with URN {urn} not found") - - assert_permission(user_data, score_set, Action.READ) - - mapped_variants = ( - db.query(MappedVariant) - .filter(ScoreSet.urn == urn) - .filter(ScoreSet.id == Variant.score_set_id) - .filter(Variant.id == MappedVariant.variant_id) - .all() - ) - - if not mapped_variants: - logger.info(msg="No mapped variants are associated with the requested score set.", extra=logging_context()) - raise HTTPException( - status_code=404, - detail=f"No mapped variant associated with score set URN {urn} was found", - ) - - return mapped_variants - - def _stream_generated_annotations(db, variants, annotation_function, as_of=None): """ Generator function to stream annotations as pure NDJSON data. @@ -2587,7 +2659,7 @@ async def get_clinical_controls_options_for_score_set( @router.get( "/score-sets/{urn}/gnomad-variants", status_code=200, - response_model=list[gnomad_variant.GnomADVariantWithMappedVariants], + response_model=list[gnomad_variant.GnomADVariantWithVariantLinks], response_model_exclude_none=True, responses={**ACCESS_CONTROL_ERROR_RESPONSES}, summary="Get gnomad variants for a score set", @@ -2595,16 +2667,26 @@ async def get_clinical_controls_options_for_score_set( async def get_gnomad_variants_for_score_set( *, urn: str, + response: Response, + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the allele → gnomAD link state as it stood at this instant. " + "ISO 8601, ideally timezone-aware. Defaults to current." + ), + ), db: Session = Depends(deps.get_db), user_data: UserData = Depends(get_current_user), version: Optional[str] = None, -) -> Sequence[GnomADVariant]: +) -> list[gnomad_variant.GnomADVariantWithVariantLinks]: """ - Fetch relevant gnomad variants for a given score set. + Fetch relevant gnomad variants for a given score set, each paired with the score-set variants (and + annotated allele digests) it links to over the allele substrate. """ - save_to_logging_context({"requested_resource": urn, "resource_property": "gnomad_variants"}) + save_to_logging_context({"requested_resource": urn, "resource_property": "gnomad_variants", "as_of": as_of}) + response.headers["X-As-Of"] = as_of.isoformat() if as_of is not None else "current" - # Rename user facing kwargs for consistency with code base naming conventions. My-py doesn't care for us redefining db. + # Rename user facing kwargs for consistency with code base naming conventions. db_version = version item: Optional[ScoreSet] = db.scalars(select(ScoreSet).where(ScoreSet.urn == urn)).one_or_none() @@ -2617,28 +2699,12 @@ async def get_gnomad_variants_for_score_set( assert_permission(user_data, item, Action.READ) - gnomad_variants_query = ( - select(GnomADVariant) - .join(MappedVariant, GnomADVariant.mapped_variants) - .join(Variant) - .where(Variant.score_set_id == item.id) - ) - if db_version is not None: save_to_logging_context({"db_version": db_version}) - gnomad_variants_query = gnomad_variants_query.where(GnomADVariant.db_version == db_version) - gnomad_variants_for_item: Sequence[GnomADVariant] = db.scalars(gnomad_variants_query).all() - gnomad_variants_with_mapped_variant = [] - for gnomad_variant_in_item in gnomad_variants_for_item: - gnomad_variant_in_item.mapped_variants = [ - mv for mv in gnomad_variant_in_item.mapped_variants if mv.current and mv.variant.score_set_id == item.id - ] - - if gnomad_variant_in_item.mapped_variants: - gnomad_variants_with_mapped_variant.append(gnomad_variant_in_item) + gnomad_variants = get_gnomad_variants_with_variant_urns(db, item.id, as_of=as_of, db_version=db_version) - if not gnomad_variants_with_mapped_variant: + if not gnomad_variants: logger.info( msg="No gnomad variants matching the provided filters are associated with the requested score set.", extra=logging_context(), @@ -2648,6 +2714,26 @@ async def get_gnomad_variants_for_score_set( detail=f"No gnomad variants matching the provided filters associated with score set URN {urn} were found", ) - save_to_logging_context({"resource_count": len(gnomad_variants_for_item)}) + save_to_logging_context({"resource_count": len(gnomad_variants)}) - return gnomad_variants_for_item + return [ + gnomad_variant.GnomADVariantWithVariantLinks.model_validate( + { + "id": gv.id, + "db_name": gv.db_name, + "db_identifier": gv.db_identifier, + "db_version": gv.db_version, + "allele_count": gv.allele_count, + "allele_number": gv.allele_number, + "allele_frequency": gv.allele_frequency, + "faf95_max": gv.faf95_max, + "faf95_max_ancestry": gv.faf95_max_ancestry, + "creation_date": gv.creation_date, + "modification_date": gv.modification_date, + "variant_links": [ + {"variant_urn": link.variant_urn, "allele_digest": link.allele_digest} for link in links + ], + } + ) + for gv, links in gnomad_variants + ] diff --git a/src/mavedb/routers/variants.py b/src/mavedb/routers/variants.py index 78144594c..70e4bf788 100644 --- a/src/mavedb/routers/variants.py +++ b/src/mavedb/routers/variants.py @@ -136,7 +136,10 @@ def lookup_variants_by_vrs_identifier( status_code=200, response_model=VariantDetail, responses={**ACCESS_CONTROL_ERROR_RESPONSES}, - response_model_exclude_none=True, + # Emit nulls rather than omitting absent fields: this shape is a stable, self-describing envelope + # (its bulk pair, GET /score-sets/{urn}/variant-details, streams the same object one-per-line for + # dataframe/columnar consumption), so every response carries the same key set. + response_model_exclude_none=False, summary="Fetch assayed variant detail by URN", ) def get_variant( diff --git a/src/mavedb/view_models/gnomad_variant.py b/src/mavedb/view_models/gnomad_variant.py index 97dd675e6..116a4e21b 100644 --- a/src/mavedb/view_models/gnomad_variant.py +++ b/src/mavedb/view_models/gnomad_variant.py @@ -8,7 +8,7 @@ from mavedb.view_models.base.base import BaseModel if TYPE_CHECKING: - from mavedb.view_models.mapped_variant import MappedVariant, MappedVariantCreate, SavedMappedVariant + from mavedb.view_models.mapped_variant import MappedVariantCreate class GnomADVariantBase(BaseModel): @@ -54,10 +54,21 @@ class Config: from_attributes = True -class SavedGnomADVariantWithMappedVariants(SavedGnomADVariant): - """Saved GnomAD variant records with mapped variants.""" +class GnomadVariantLink(BaseModel): + """One score-set variant a gnomAD frequency record reaches, tagged with the annotated allele's digest. - mapped_variants: Sequence["SavedMappedVariant"] + Mirrors :class:`clinical_control.ClinvarVariantLink`: a gnomAD variant fans out to every allele that + resolved to it, and each allele belongs to a score-set variant. + """ + + variant_urn: str + allele_digest: Optional[str] = None + + +class SavedGnomADVariantWithVariantLinks(SavedGnomADVariant): + """Saved gnomAD variant paired with the score-set variants (and annotated allele digests) it links to.""" + + variant_links: Sequence[GnomadVariantLink] class GnomADVariant(SavedGnomADVariant): @@ -66,7 +77,7 @@ class GnomADVariant(SavedGnomADVariant): pass -class GnomADVariantWithMappedVariants(SavedGnomADVariantWithMappedVariants): - """GnomAD variant view model with mapped variants for non-admin clients.""" +class GnomADVariantWithVariantLinks(SavedGnomADVariantWithVariantLinks): + """GnomAD variant + its score-set variant links, for non-admin clients.""" - mapped_variants: Sequence["MappedVariant"] + pass diff --git a/src/mavedb/view_models/variant_detail.py b/src/mavedb/view_models/variant_detail.py index d9bc08344..9603bc7b3 100644 --- a/src/mavedb/view_models/variant_detail.py +++ b/src/mavedb/view_models/variant_detail.py @@ -35,15 +35,22 @@ class VariantDetail(BaseModel): """The assayed variant-detail envelope (``GET /variants/{urn}``). Two tiers: flat, UI-ergonomic assay fields (the ``targetHgvs``/``referenceHgvs`` coordinate pair - is a client-side toggle, no refetch) plus the spec-pure GA4GH ``molecularRepresentation`` - (``CategoricalVariant``, no MaveDB fields inside). The MaveDB layer rides alongside, keyed by VRS - digest: the ``alleles`` identity sidecar (per-allele ``level`` / ``hgvs`` / ``clingenAlleleId`` / - ``relation`` — one entry per linked allele, sharing keys with ``annotations``) and the - ``annotations`` map. ``isCurrent``/``supersededByScoreSet`` let a superseded variant self-describe: - ``supersededByScoreSet`` is the superseding *score set*'s URN, not a variant URN. Supersession is - versioned at the score-set level, and a newer version may add, drop, or renumber variants — so there - is no stable superseding-*variant* pointer to hand back; a consumer resolves the current measurement - by looking this variant up within that score set. Absent fields are omitted. + is a client-side toggle, no refetch; the ``preMapped``/``postMapped`` raw VRS pair lets a + VRS/bulk consumer read the assayed-level and measured VRS directly) plus the spec-pure GA4GH + ``molecularRepresentation`` (``CategoricalVariant``, no MaveDB fields inside). The MaveDB layer + rides alongside, keyed by VRS digest: the ``alleles`` identity sidecar (per-allele ``level`` / + ``hgvs`` / ``clingenAlleleId`` / ``relation`` — one entry per linked allele, sharing keys with + ``annotations``) and the ``annotations`` map. ``isCurrent``/``supersededByScoreSet`` let a + superseded variant self-describe: ``supersededByScoreSet`` is the superseding *score set*'s URN, + not a variant URN. Supersession is versioned at the score-set level, and a newer version may add, + drop, or renumber variants — so there is no stable superseding-*variant* pointer to hand back; a + consumer resolves the current measurement by looking this variant up within that score set. + + Unlike most MaveDB response models, this one serializes with ``exclude_none=False`` on both routes + that emit it (``GET /variants/{urn}`` and the bulk ``GET /score-sets/{urn}/variant-details`` NDJSON + stream): the shape is a stable, self-describing envelope, so every record carries the same key set + and an unmapped variant reads ``preMapped``/``postMapped``/``molecularRepresentation`` as ``null`` + rather than dropping them. The two routes are kept in lockstep — the same object, one shape. """ urn: str @@ -57,6 +64,9 @@ class VariantDetail(BaseModel): assay_level_digest: Optional[str] = None clingen_allele_id: Optional[str] = None + pre_mapped: Optional[dict[str, Any]] = None + post_mapped: Optional[dict[str, Any]] = None + molecular_representation: Optional[dict[str, Any]] = None mode: Optional[str] = None alleles: dict[str, AlleleIdentity] = {} diff --git a/tests/helpers/constants.py b/tests/helpers/constants.py index 8766c2f53..7f00ed5f8 100644 --- a/tests/helpers/constants.py +++ b/tests/helpers/constants.py @@ -2147,7 +2147,7 @@ "faf95MaxAncestry": TEST_GNOMAD_FAF95_MAX_ANCESTRY, "creationDate": date.today().isoformat(), "modificationDate": date.today().isoformat(), - "recordType": "GnomADVariantWithMappedVariants", + "recordType": "GnomADVariantWithVariantLinks", "id": 1, # Presuming this is the only gnomAD variant in the database } diff --git a/tests/helpers/util/annotation.py b/tests/helpers/util/annotation.py index f845cdc1c..24b9c11f3 100644 --- a/tests/helpers/util/annotation.py +++ b/tests/helpers/util/annotation.py @@ -60,11 +60,13 @@ def seed_mapping_record( assay_level: str = "cdna", hgvs_assay_level: Optional[str] = None, mapping_api_version: str = "test.0.0", + pre_mapped: Optional[dict] = None, valid_from: Optional[datetime] = None, ) -> MappingRecord: """Give ``variant`` a live mapping record carrying ``alleles`` and their annotations. - ``variant`` may be a ``Variant`` instance or its URN. When ``valid_from`` is set it stamps the + ``variant`` may be a ``Variant`` instance or its URN. ``pre_mapped`` sets the record's assayed-level + VRS (the flat ``preMapped`` field on the variant detail). When ``valid_from`` is set it stamps the record, its allele links, and every annotation link, so ``as_of`` reconstruction tests can place the whole graph at a chosen instant (absent it, ``valid_from`` defaults to ``now()`` and the row is live). Returns the created ``MappingRecord``. @@ -79,6 +81,7 @@ def seed_mapping_record( assay_level=assay_level, hgvs_assay_level=hgvs_assay_level, mapping_api_version=mapping_api_version, + pre_mapped=pre_mapped, ) if valid_from is not None: record.valid_from = valid_from diff --git a/tests/routers/test_score_set.py b/tests/routers/test_score_set.py index 77150ba4c..2bc830db3 100644 --- a/tests/routers/test_score_set.py +++ b/tests/routers/test_score_set.py @@ -25,6 +25,7 @@ from mavedb.models.experiment import Experiment as ExperimentDbModel from mavedb.models.job_run import JobRun from mavedb.models.pipeline import Pipeline +from mavedb.models.gnomad_variant import GnomADVariant as GnomADVariantDbModel from mavedb.models.score_set import ScoreSet as ScoreSetDbModel from mavedb.models.variant import Variant as VariantDbModel from mavedb.view_models.lean_variant import LeanVariant @@ -61,6 +62,8 @@ TEST_SAVED_GNOMAD_VARIANT, TEST_SAVED_TAXONOMY, TEST_USER, + TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X, + TEST_VALID_PRE_MAPPED_VRS_ALLELE_VRS2_X, VALID_CLINGEN_CA_ID, ) from tests.helpers.dependency_overrider import DependencyOverrider @@ -82,14 +85,12 @@ create_seq_score_set_with_mapped_variants, create_seq_score_set_with_variants, link_clinical_controls_to_alleles, - link_clinvar_control_to_mapped_variant, - link_gnomad_variants_to_mapped_variants, publish_score_set, seed_annotation_substrate, + seed_csv_substrate, ) from tests.helpers.util.user import change_ownership from tests.helpers.util.variant import ( - create_mapped_variants_for_score_set, mock_worker_variant_insertion, ) @@ -3399,7 +3400,17 @@ def test_download_variants_data_file( score_set = create_seq_score_set(client, experiment["urn"]) score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") if mapped_variant is not None: - create_mapped_variants_for_score_set(session, score_set["urn"], mapped_variant) + if has_hgvs_g: + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g=mapped_variant["hgvs_g"], + hgvs_c=mapped_variant["hgvs_c"], + hgvs_p=mapped_variant["hgvs_p"], + ) + elif has_hgvs_p: + seed_csv_substrate(session, score_set, assay_level="protein", hgvs_p=mapped_variant["hgvs_p"]) with patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as worker_queue: published_score_set = publish_score_set(client, score_set["urn"]) @@ -3584,7 +3595,17 @@ def test_download_scores_counts_and_post_mapped_variants_file( client, session, data_provider, score_set, data_files / "scores.csv", data_files / "counts.csv" ) if mapped_variant is not None: - create_mapped_variants_for_score_set(session, score_set["urn"], mapped_variant) + if has_hgvs_g: + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g=mapped_variant["hgvs_g"], + hgvs_c=mapped_variant["hgvs_c"], + hgvs_p=mapped_variant["hgvs_p"], + ) + elif has_hgvs_p: + seed_csv_substrate(session, score_set, assay_level="protein", hgvs_p=mapped_variant["hgvs_p"]) with patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as worker_queue: published_score_set = publish_score_set(client, score_set["urn"]) @@ -3622,8 +3643,14 @@ def test_download_vep_file_in_variant_data_path(session, data_provider, client, score_set = mock_worker_variant_insertion( client, session, data_provider, score_set, data_files / "scores.csv", data_files / "counts.csv" ) - # Create mapped variants with VEP consequence populated - create_mapped_variants_for_score_set(session, score_set["urn"], TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION) + # Seed mapped alleles with a VEP consequence on the authoritative allele. + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g=TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION["hgvs_g"], + vep_consequence="missense_variant", + ) with patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as worker_queue: published_score_set = publish_score_set(client, score_set["urn"]) @@ -3646,13 +3673,15 @@ def test_download_clingen_file_in_variant_data_path(session, data_provider, clie score_set = mock_worker_variant_insertion( client, session, data_provider, score_set, data_files / "scores.csv", data_files / "counts.csv" ) - # Create mapped variants then set ClinGen allele id for first mapped variant - create_mapped_variants_for_score_set(session, score_set["urn"], TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION) - db_score_set = session.query(ScoreSetDbModel).filter(ScoreSetDbModel.urn == score_set["urn"]).one() - first_mapped_variant = db_score_set.variants[0].mapped_variants[0] - first_mapped_variant.clingen_allele_id = VALID_CLINGEN_CA_ID - session.add(first_mapped_variant) - session.commit() + # Seed mapped alleles, with a ClinGen id on only the first variant's authoritative allele. + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g=TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION["hgvs_g"], + clingen_allele_id=VALID_CLINGEN_CA_ID, + annotate="first", + ) with patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as worker_queue: published_score_set = publish_score_set(client, score_set["urn"]) @@ -3670,11 +3699,31 @@ def test_download_clingen_file_in_variant_data_path(session, data_provider, clie def test_download_gnomad_file_in_variant_data_path(session, data_provider, client, setup_router_db, data_files): experiment = create_experiment(client) - # Link a gnomAD variant to the first mapped variant (version may not match export filter) - score_set = create_seq_score_set_with_mapped_variants( - client, session, data_provider, experiment["urn"], data_files / "scores.csv" + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") + + # The CSV gnomAD column reflects the served release (v4.1); seed one and link it to the first variant's + # authoritative allele. + gnomad_variant = GnomADVariantDbModel( + db_name="gnomAD", + db_identifier="10-1-A-G", + db_version="v4.1", + allele_count=3, + allele_number=1613510, + allele_frequency=3 / 1613510, + faf95_max=6.8e-07, + faf95_max_ancestry="nfe", + ) + session.add(gnomad_variant) + session.commit() + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g=TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION["hgvs_g"], + gnomad_variant_ids=[gnomad_variant.id], + annotate="first", ) - link_gnomad_variants_to_mapped_variants(session, score_set) with patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as worker_queue: published_score_set = publish_score_set(client, score_set["urn"]) @@ -3686,6 +3735,9 @@ def test_download_gnomad_file_in_variant_data_path(session, data_provider, clien assert response.status_code == 200 reader = csv.DictReader(StringIO(response.text)) assert "gnomad.gnomad_af" in reader.fieldnames + rows = list(reader) + assert rows[0]["gnomad.gnomad_af"] == str(3 / 1613510) + assert all(row["gnomad.gnomad_af"] == "NA" for row in rows[1:]) def test_download_clingen_and_vep_file_in_variant_data_path( @@ -3696,13 +3748,16 @@ def test_download_clingen_and_vep_file_in_variant_data_path( score_set = mock_worker_variant_insertion( client, session, data_provider, score_set, data_files / "scores.csv", data_files / "counts.csv" ) - # Create mapped variants with VEP consequence populated - create_mapped_variants_for_score_set(session, score_set["urn"], TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION) - db_score_set = session.query(ScoreSetDbModel).filter(ScoreSetDbModel.urn == score_set["urn"]).one() - first_mapped_variant = db_score_set.variants[0].mapped_variants[0] - first_mapped_variant.clingen_allele_id = VALID_CLINGEN_CA_ID - session.add(first_mapped_variant) - session.commit() + # Seed mapped alleles with a VEP consequence (all variants) and a ClinGen id (first variant only). + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g=TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION["hgvs_g"], + vep_consequence="missense_variant", + clingen_allele_id=VALID_CLINGEN_CA_ID, + annotate="first", + ) with patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as worker_queue: published_score_set = publish_score_set(client, score_set["urn"]) @@ -3728,13 +3783,15 @@ def test_download_clingen_and_scores_file_in_variant_data_path( score_set = mock_worker_variant_insertion( client, session, data_provider, score_set, data_files / "scores.csv", data_files / "counts.csv" ) - # Create mapped variants with VEP consequence populated - create_mapped_variants_for_score_set(session, score_set["urn"], TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION) - db_score_set = session.query(ScoreSetDbModel).filter(ScoreSetDbModel.urn == score_set["urn"]).one() - first_mapped_variant = db_score_set.variants[0].mapped_variants[0] - first_mapped_variant.clingen_allele_id = VALID_CLINGEN_CA_ID - session.add(first_mapped_variant) - session.commit() + # Seed mapped alleles with a ClinGen id on only the first variant's authoritative allele. + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g=TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION["hgvs_g"], + clingen_allele_id=VALID_CLINGEN_CA_ID, + annotate="first", + ) with patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as worker_queue: published_score_set = publish_score_set(client, score_set["urn"]) @@ -3766,10 +3823,18 @@ def test_download_clinvar_namespace_in_variant_data_path(session, data_provider, # The ClinVar control seeded in setup_router_db has db_version="11_2024", mapping to namespace clinvar.2024_11. clinvar_namespace = "clinvar.2024_11" experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( + score_set = create_seq_score_set_with_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) - link_clinvar_control_to_mapped_variant(session, score_set) + # ClinVar control id=1 (db_version 11_2024) links to only the first variant's authoritative allele. + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g="NC_000018.10:g.1A>G", + clinvar_control_ids=[1], + annotate="first", + ) with patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as worker_queue: published_score_set = publish_score_set(client, score_set["urn"]) @@ -3800,10 +3865,17 @@ def test_download_clinvar_namespace_with_no_matching_version( # clinvar.2023_01 does not match the seeded control (11_2024), so all rows should be NA. clinvar_namespace = "clinvar.2023_01" experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( + score_set = create_seq_score_set_with_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) - link_clinvar_control_to_mapped_variant(session, score_set) + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g="NC_000018.10:g.1A>G", + clinvar_control_ids=[1], + annotate="first", + ) with patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as worker_queue: published_score_set = publish_score_set(client, score_set["urn"]) @@ -3830,10 +3902,17 @@ def test_download_multiple_clinvar_namespaces_in_variant_data_path( matching_ns = "clinvar.2024_11" # matches db_version="11_2024" seeded in setup_router_db non_matching_ns = "clinvar.2023_01" # no controls with this version experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( + score_set = create_seq_score_set_with_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) - link_clinvar_control_to_mapped_variant(session, score_set) + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g="NC_000018.10:g.1A>G", + clinvar_control_ids=[1], + annotate="first", + ) with patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as worker_queue: published_score_set = publish_score_set(client, score_set["urn"]) @@ -3861,6 +3940,49 @@ def test_download_multiple_clinvar_namespaces_in_variant_data_path( assert all(row[f"{non_matching_ns}.clinical_review_status"] == "NA" for row in rows) +def test_download_variant_data_as_of_reconstructs_annotation_layer( + session, data_provider, client, setup_router_db, data_files +): + """`as_of` time-travels the CSV annotation layer: an instant before the annotation was live yields NA, + while the current view yields the value. The resolved instant is echoed in X-As-Of.""" + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") + + # Seed the whole substrate live as of 2020 — live "now" but not at an as_of in 2000. + seeded_at = datetime(2020, 1, 1, tzinfo=timezone.utc) + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g=TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION["hgvs_g"], + vep_consequence="missense_variant", + valid_from=seeded_at, + ) + + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as worker_queue: + published_score_set = publish_score_set(client, score_set["urn"]) + worker_queue.assert_called_once() + + base_url = f"/api/v1/score-sets/{published_score_set['urn']}/variants/data" + + # Current: the annotation is live, and X-As-Of echoes "current". + current = client.get(base_url, params={"namespaces": "vep", "drop_na_columns": "false"}) + assert current.status_code == 200 + assert current.headers["X-As-Of"] == "current" + current_rows = list(csv.DictReader(StringIO(current.text))) + assert any(row["vep.vep_functional_consequence"] == "missense_variant" for row in current_rows) + + # An instant before the annotation existed: nothing live -> every VEP value is NA, and X-As-Of echoes it. + past = datetime(2000, 1, 1, tzinfo=timezone.utc) + before = client.get(base_url, params={"namespaces": "vep", "drop_na_columns": "false", "as_of": past.isoformat()}) + assert before.status_code == 200 + assert datetime.fromisoformat(before.headers["X-As-Of"]) == past + before_rows = list(csv.DictReader(StringIO(before.text))) + assert before_rows # variants (and their immutable scores) are still present + assert all(row["vep.vep_functional_consequence"] == "NA" for row in before_rows) + + def test_invalid_clinvar_namespace_returns_422(client, setup_router_db, data_files): """A clinvar namespace with an out-of-range month (13) is rejected with 422.""" experiment = create_experiment(client) @@ -4067,6 +4189,106 @@ def test_clinical_control_options_exclude_non_current(client, setup_router_db, s assert response.status_code == 404 +######################################################################################################################## +# Downloading a score set's variant details (VRS + Cat-VRS + annotations) +######################################################################################################################## + + +def test_get_score_set_variant_details_returns_ndjson(client, session, data_provider, data_files, setup_router_db): + """The whole-set variant-detail export streams NDJSON — one VariantDetail per mapped variant, each + carrying the flat preMapped/postMapped VRS pair and the spec-pure Cat-VRS.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + variant_urns = { + variant.urn + for variant in seed_annotation_substrate(session, score_set, pre_mapped=TEST_VALID_PRE_MAPPED_VRS_ALLELE_VRS2_X) + } + + response = client.get(f"/api/v1/score-sets/{score_set['urn']}/variant-details") + assert response.status_code == 200 + assert response.headers["content-type"].startswith("application/x-ndjson") + assert response.headers["X-As-Of"] == "current" + assert response.headers["X-Total-Count"] == str(len(variant_urns)) + assert response.headers["X-Stream-Type"] == "variant-detail" + + records = parse_ndjson_response(response) + assert len(records) == len(variant_urns) + assert {record["urn"] for record in records} == variant_urns + for record in records: + # The flat VRS pair for VRS consumers, and the spec-pure Cat-VRS for the full picture. + assert record["preMapped"] == TEST_VALID_PRE_MAPPED_VRS_ALLELE_VRS2_X + assert record["postMapped"] == TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X + assert record.get("molecularRepresentation") is not None + + +def test_get_score_set_variant_details_omits_unmapped_variants( + client, session, data_provider, data_files, setup_router_db +): + """Only mapped variants carry VRS/detail, so an un-mapped variant is omitted — the record count is + the mapped subset, not the whole score set.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + variants = seed_annotation_substrate(session, score_set, skip_first=True) + unmapped_urn = variants[0].urn # read before the request expires the instances + + response = client.get(f"/api/v1/score-sets/{score_set['urn']}/variant-details") + assert response.status_code == 200 + + records = parse_ndjson_response(response) + assert len(records) == score_set["numVariants"] - 1 + assert unmapped_urn not in {record["urn"] for record in records} + + +def test_get_score_set_variant_details_for_unmapped_returns_empty( + client, session, data_provider, data_files, setup_router_db +): + """A score set with no live mapping records has no annotatable variants — an empty NDJSON stream with + X-Total-Count 0, not a 404.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + # No new-substrate mapping records → no annotatable variants. + + response = client.get(f"/api/v1/score-sets/{score_set['urn']}/variant-details") + assert response.status_code == 200 + assert response.headers["X-Total-Count"] == "0" + assert parse_ndjson_response(response) == [] + + +def test_cannot_get_variant_details_for_nonexistent_score_set(client, setup_router_db): + response = client.get("/api/v1/score-sets/urn:mavedb:00000000-a-1/variant-details") + assert response.status_code == 404 + + +def test_get_score_set_variant_details_honors_as_of(client, session, data_provider, data_files, setup_router_db): + """``as_of`` time-travels the molecular layer: a far-future instant sees the freshly-seeded substrate + as live (the full set), a far-past instant sees no live mapping (an empty stream). Either way 200 — + ``as_of`` is a filter, so an empty match is an empty collection, never a 404.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + seed_annotation_substrate(session, score_set) + url = f"/api/v1/score-sets/{score_set['urn']}/variant-details" + + future = datetime(2999, 1, 1, tzinfo=timezone.utc) + future_response = client.get(url, params={"as_of": future.isoformat()}) + assert future_response.status_code == 200 + assert datetime.fromisoformat(future_response.headers["X-As-Of"]) == future + assert len(parse_ndjson_response(future_response)) == score_set["numVariants"] + + past = datetime(2000, 1, 1, tzinfo=timezone.utc) + past_response = client.get(url, params={"as_of": past.isoformat()}) + assert past_response.status_code == 200 + assert past_response.headers["X-Total-Count"] == "0" + assert parse_ndjson_response(past_response) == [] + + ######################################################################################################################## # Fetching annotated variants for a score set ######################################################################################################################## @@ -4605,19 +4827,32 @@ def test_annotated_functional_study_result_exists_for_score_set_when_some_varian def test_can_fetch_current_gnomad_variants_for_score_set(client, setup_router_db, session, data_provider, data_files): experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( + score_set = create_seq_score_set_with_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) - link_gnomad_variants_to_mapped_variants(session, score_set) + # gnomAD variant id=1 (seeded by setup_router_db) links to the first variant's authoritative allele. + variants = seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g="NC_000018.10:g.1A>G", + gnomad_variant_ids=[1], + annotate="first", + ) + expected_urn = variants[0].urn + expected_digest = f"csv-auth-{variants[0].id}" response = client.get(f"/api/v1/score-sets/{score_set['urn']}/gnomad-variants") assert response.status_code == 200 + assert response.headers["x-as-of"] == "current" response_data = response.json() assert len(response_data) == 1 for gnomad_variant in response_data: - mapped_variants = gnomad_variant.pop("mappedVariants") - assert len(mapped_variants) == 1 + variant_links = gnomad_variant.pop("variantLinks") + assert len(variant_links) == 1 + assert variant_links[0]["variantUrn"] == expected_urn + assert variant_links[0]["alleleDigest"] == expected_digest gnomad_variant_items = sorted(gnomad_variant.items()) assert gnomad_variant_items == sorted(TEST_SAVED_GNOMAD_VARIANT.items()) @@ -4626,10 +4861,17 @@ def test_can_fetch_current_gnomad_variants_for_score_set_with_version( client, setup_router_db, session, data_provider, data_files ): experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( + score_set = create_seq_score_set_with_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) - link_gnomad_variants_to_mapped_variants(session, score_set) + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g="NC_000018.10:g.1A>G", + gnomad_variant_ids=[1], + annotate="first", + ) response = client.get(f"/api/v1/score-sets/{score_set['urn']}/gnomad-variants?version={TEST_GNOMAD_DATA_VERSION}") assert response.status_code == 200 @@ -4637,8 +4879,8 @@ def test_can_fetch_current_gnomad_variants_for_score_set_with_version( response_data = response.json() assert len(response_data) == 1 for gnomad_variant in response_data: - mapped_variants = gnomad_variant.pop("mappedVariants") - assert len(mapped_variants) == 1 + variant_links = gnomad_variant.pop("variantLinks") + assert len(variant_links) == 1 gnomad_variant_items = sorted(gnomad_variant.items()) assert gnomad_variant_items == sorted(TEST_SAVED_GNOMAD_VARIANT.items()) @@ -4647,10 +4889,17 @@ def test_cannot_fetch_current_gnomad_variants_for_score_set_with_nonexistent_ver client, setup_router_db, session, data_provider, data_files ): experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( + score_set = create_seq_score_set_with_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) - link_gnomad_variants_to_mapped_variants(session, score_set) + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g="NC_000018.10:g.1A>G", + gnomad_variant_ids=[1], + annotate="first", + ) response = client.get(f"/api/v1/score-sets/{score_set['urn']}/gnomad-variants?version=nonexistent_version") assert response.status_code == 404 @@ -4667,10 +4916,17 @@ def test_cannot_fetch_gnomad_variants_for_nonexistent_score_set( client, setup_router_db, session, data_provider, data_files ): experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( + score_set = create_seq_score_set_with_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) - link_gnomad_variants_to_mapped_variants(session, score_set) + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g="NC_000018.10:g.1A>G", + gnomad_variant_ids=[1], + annotate="first", + ) response = client.get(f"/api/v1/score-sets/{score_set['urn'] + 'xxx'}/gnomad-variants") @@ -4683,7 +4939,7 @@ def test_cannot_fetch_gnomad_variants_for_score_set_when_none_exist( client, setup_router_db, session, data_provider, data_files ): experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( + score_set = create_seq_score_set_with_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) diff --git a/tests/routers/test_variant.py b/tests/routers/test_variant.py index a422bbe24..bb96bec08 100644 --- a/tests/routers/test_variant.py +++ b/tests/routers/test_variant.py @@ -140,11 +140,13 @@ def test_get_variant_detail_envelope(client, session, data_provider, data_files, assert body["alleles"]["cdna-digest"]["level"] == "cdna" assert body["alleles"]["cdna-digest"]["hgvs"] == "NM_000546.6:c.1216G>A" assert body["alleles"]["cdna-digest"]["clingenAlleleId"] == "CA123" - assert "relation" not in body["alleles"]["cdna-digest"] # null relation dropped by exclude_none + assert ( + body["alleles"]["cdna-digest"]["relation"] is None + ) # measured allele: no relation to itself (null, not dropped) # The measured allele is the focus (isFocus); it carries no derivation (that axis describes the # *other* members relative to it), and pairs with its genomic projection sibling (projectionOf). assert body["alleles"]["cdna-digest"]["isFocus"] is True - assert "derivation" not in body["alleles"]["cdna-digest"] # null derivation dropped by exclude_none + assert body["alleles"]["cdna-digest"]["derivation"] is None # null, not dropped (exclude_none=False) assert body["alleles"]["cdna-digest"]["projectionOf"] == "gen-digest" assert body["alleles"]["gen-digest"]["isFocus"] is False assert body["alleles"]["gen-digest"]["relation"] == "coordinate_representation_of" @@ -153,16 +155,16 @@ def test_get_variant_detail_envelope(client, session, data_provider, data_files, assert body["alleles"]["prot-digest"]["level"] == "protein" assert body["alleles"]["prot-digest"]["hgvs"] == "NP_000537.3:p.Ala406Thr" assert body["alleles"]["prot-digest"]["relation"] == "translation_of" - # The apex is a deterministic projection here but pairs with nothing (projectionOf dropped as null). + # The apex is a deterministic projection here but pairs with nothing (projectionOf is null). assert body["alleles"]["prot-digest"]["derivation"] == "projection" - assert "projectionOf" not in body["alleles"]["prot-digest"] + assert body["alleles"]["prot-digest"]["projectionOf"] is None # The synonymous cousin (different projection group) surfaces as a member wearing co_encodes and is # labelled `convergent` (a distinct change sharing the consequence, not an ambiguous candidate). assert body["alleles"]["cousin-digest"]["relation"] == "co_encodes" assert body["alleles"]["cousin-digest"]["derivation"] == "convergent" assert body["annotations"]["cdna-digest"]["vep"]["consequence"] == "missense_variant" assert body["isCurrent"] is True - assert "supersededByScoreSet" not in body # dropped by exclude_none when current + assert body["supersededByScoreSet"] is None # null when current (stable envelope, not dropped) assert response.headers["X-As-Of"] == "current" @@ -182,8 +184,8 @@ def test_get_variant_detail_echoes_as_of_header(client, session, data_provider, assert historical.status_code == 200 assert historical.headers["X-As-Of"] == "2020-01-01T00:00:00+00:00" - # Before any mapping existed, the molecular layer is empty. - assert "molecularRepresentation" not in historical.json() + # Before any mapping existed, the molecular layer is empty — the stable envelope carries it as null. + assert historical.json()["molecularRepresentation"] is None def test_superseded_variant_self_describes(client, session, data_provider, data_files, setup_router_db): From cdbaeab715ea29184ef41ee80dbf95a70772579a Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 22 Jul 2026 23:17:37 -0700 Subject: [PATCH 102/252] feat(score-sets): rewrite CSV export onto the allele substrate - replace the MappedVariant-keyed CSV joins with a substrate query (_csv_substrate_query) over the live mapping record, its authoritative allele, and the projection-group nucleotide sibling - add _gnomad_by_allele/_clinvar_by_allele batch fetches keyed by allele id, replacing the old mapped_variant_id-keyed lookups - reuse mapped_hgvs_by_level/get_protein_hgvs_by_record from score_set_variants so the CSV resolves post-mapped HGVS per level the same way the lean whole-set view does, instead of falling back to a per-row VRS parse - extract column planning (_plan_csv_columns) and header assembly (_csv_header_columns) out of get_score_set_variants_as_csv - thread as_of through the CSV export to time-travel the annotation layer (post-mapped HGVS, VEP, gnomAD, ClinVar) independently of the immutable scores/counts namespaces - add seed_csv_substrate test helper for the substrate-based fixtures --- src/mavedb/lib/score_sets.py | 528 ++++++++++++++++++++------------ tests/helpers/util/score_set.py | 119 ++++--- 2 files changed, 412 insertions(+), 235 deletions(-) diff --git a/src/mavedb/lib/score_sets.py b/src/mavedb/lib/score_sets.py index 3c1559510..e684efd7f 100644 --- a/src/mavedb/lib/score_sets.py +++ b/src/mavedb/lib/score_sets.py @@ -3,6 +3,7 @@ import logging import re from collections import Counter, defaultdict +from dataclasses import dataclass from datetime import datetime from operator import attrgetter from typing import TYPE_CHECKING, Any, BinaryIO, Iterable, List, Optional, Sequence @@ -10,11 +11,12 @@ import numpy as np import pandas as pd from pandas.testing import assert_index_equal -from sqlalchemy import Integer, and_, cast, func, or_, select +from sqlalchemy import Integer, Select, and_, cast, func, or_, select from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Query, Session, aliased, contains_eager, joinedload, selectinload from mavedb.lib.exceptions import ValidationError +from mavedb.lib.gnomad import GNOMAD_DATA_VERSION, GNOMAD_DB_NAME from mavedb.lib.logging.context import logging_context, save_to_logging_context from mavedb.lib.mave.constants import ( HGVS_NT_COLUMN, @@ -26,24 +28,25 @@ ) from mavedb.lib.mave.utils import is_csv_null from mavedb.lib.permissions import Action, has_permission +from mavedb.lib.score_set_variants import get_protein_hgvs_by_record, mapped_hgvs_by_level from mavedb.lib.types.authentication import UserData from mavedb.lib.validation.constants.general import null_values_list from mavedb.lib.validation.utilities import is_null as validate_is_null -from mavedb.lib.variants import get_digest_from_post_mapped, get_hgvs_from_post_mapped, is_hgvs_g, is_hgvs_p from mavedb.models.allele import Allele from mavedb.models.clinical_control import ClinvarControl -from mavedb.models.clinical_control_mapped_variant import mapped_variants_clinical_controls_association_table +from mavedb.models.clinvar_allele_link import ClinvarAlleleLink from mavedb.models.contributor import Contributor from mavedb.models.controlled_keyword import ControlledKeyword from mavedb.models.doi_identifier import DoiIdentifier from mavedb.models.ensembl_identifier import EnsemblIdentifier from mavedb.models.ensembl_offset import EnsemblOffset +from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.models.experiment import Experiment from mavedb.models.experiment_controlled_keyword import ExperimentControlledKeywordAssociation from mavedb.models.experiment_publication_identifier import ExperimentPublicationIdentifierAssociation from mavedb.models.experiment_set import ExperimentSet +from mavedb.models.gnomad_allele_link import GnomadAlleleLink from mavedb.models.gnomad_variant import GnomADVariant -from mavedb.models.mapped_variant import MappedVariant from mavedb.models.mapping_record import MappingRecord from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.publication_identifier import PublicationIdentifier @@ -61,6 +64,7 @@ from mavedb.models.uniprot_offset import UniprotOffset from mavedb.models.user import User from mavedb.models.variant import Variant +from mavedb.models.vep_allele_consequence import VepAlleleConsequence from mavedb.view_models.search import ControlledKeywordFilterOption, ScoreSetsSearch if TYPE_CHECKING: @@ -74,6 +78,9 @@ # e.g. "clinvar.2024_01" for January 2024. CLINVAR_NS_PATTERN = re.compile(r"^clinvar\.(\d+)_(0[1-9]|1[0-2])$") +# CSV rows come out in variant-number order (the integer after '#'); id breaks ties stably. +_VARIANT_NUMBER_ORDER = (cast(func.split_part(Variant.urn, "#", 2), Integer), Variant.id) + class HGVSColumns: NUCLEOTIDE: str = "hgvs_nt" # dataset.constants.hgvs_nt_column @@ -606,6 +613,287 @@ def get_annotatable_variants( ) +@dataclass(frozen=True) +class _CsvMappedRow: + """The mapping-derived CSV fields for one variant, resolved from its live mapping record's + authoritative allele (post-mapped HGVS per level, assay-level HGVS, VRS digest, VEP consequence, + ClinGen id). gnomAD frequency and ClinVar assertions ride alongside as separate parallel lists + (they fan out per allele, not 1:1 like these).""" + + hgvs_g: Optional[str] + hgvs_c: Optional[str] + hgvs_p: Optional[str] + hgvs_assay_level: Optional[str] + vrs_digest: Optional[str] + vep_functional_consequence: Optional[str] + clingen_allele_id: Optional[str] + + +@dataclass(frozen=True) +class _CsvVariantData: + """The per-variant inputs to the CSV row builder, as positionally-aligned sequences (one entry per + variant, in output order). ``mappings`` / ``gnomad_data`` / ``clinvar_per_variant`` are ``None`` when the + requested namespaces don't need that dimension.""" + + variants: list[Variant] + mappings: Optional[list[Optional[_CsvMappedRow]]] + gnomad_data: Optional[list[Optional[GnomADVariant]]] + clinvar_per_variant: Optional[list[Optional[dict[str, Optional[ClinvarControl]]]]] + + +def _plan_csv_columns( + score_set: ScoreSet, + namespaces: List[str], + *, + include_custom_columns: Optional[bool], + include_post_mapped_hgvs: Optional[bool], +) -> tuple[dict[str, list[str]], dict[str, str]]: + """Plan the CSV's columns from the requested namespaces. Returns the ``namespace -> [column, ...]`` map + (in emission order) and the ``clinvar_namespace -> db_version`` map parsed out of any + ``clinvar.YEAR_MONTH`` namespaces (``db_version`` is stored on the control as ``MONTH_YEAR``).""" + assert type(score_set.dataset_columns) is dict + + columns: dict[str, list[str]] = { + "core": ["accession", "hgvs_nt", "hgvs_splice", "hgvs_pro"], + "mavedb": [], + } + + if include_post_mapped_hgvs: + columns["mavedb"] += [ + "post_mapped_hgvs_g", + "post_mapped_hgvs_p", + "post_mapped_hgvs_c", + "post_mapped_hgvs_at_assay_level", + "post_mapped_vrs_digest", + ] + + for namespace in namespaces: + columns[namespace] = [] + + if include_custom_columns: + if "scores" in columns: + columns["scores"] = [str(x) for x in list(score_set.dataset_columns.get("score_columns", []))] + if "counts" in columns: + columns["counts"] = [str(x) for x in list(score_set.dataset_columns.get("count_columns", []))] + elif "scores" in columns: + columns["scores"].append(REQUIRED_SCORE_COLUMN) + + if "vep" in columns: + columns["vep"].append("vep_functional_consequence") + if "gnomad" in columns: + columns["gnomad"].append("gnomad_af") + if "clingen" in columns: + columns["clingen"].append("clingen_allele_id") + + clinvar_namespaces: dict[str, str] = {} + for cv_ns in namespaces: + m = CLINVAR_NS_PATTERN.match(cv_ns) + if m: + year, month = m.group(1), m.group(2) + clinvar_namespaces[cv_ns] = f"{month}_{year}" + columns[cv_ns] = ["clinical_significance", "clinical_review_status"] + + return columns, clinvar_namespaces + + +def _csv_header_columns(columns: dict[str, list[str]], *, namespaced: Optional[bool]) -> list[str]: + """Flatten the planned column map into the ordered CSV header. ClinVar-versioned namespaces are always + prefixed (to avoid collisions when multiple versions are requested); other non-``core`` namespaces are + prefixed only when ``namespaced`` is set; ``core`` is never prefixed.""" + header: list[str] = [] + for namespace, cols in columns.items(): + for col in cols: + if CLINVAR_NS_PATTERN.match(namespace) or (namespaced and namespace != "core"): + header.append(f"{namespace}.{col}") + else: + header.append(col) + + return header + + +def _csv_substrate_query(score_set_id: int, *, as_of: Optional[datetime]) -> Select[Any]: + """The base per-variant query for the annotation namespaces: each variant LEFT-joined to its live mapping + record, its authoritative (measured) allele (digest / ClinGen id / VEP consequence), and the + projection-group nucleotide sibling — mirroring the lean whole-set view's join + (:func:`mavedb.lib.score_set_variants.get_lean_score_set_variants`). LEFT joins with ``live_at`` in the ON + clause keep unmapped variants (with null mapped fields); a WHERE would collapse them out. The protein slot + and the gnomAD/ClinVar dimensions are resolved by separate fetches and stitched on by id.""" + sibling_link = aliased(MappingRecordAllele) + sibling_allele = aliased(Allele) + return ( + select( + Variant, + MappingRecord.id.label("mapping_record_id"), + MappingRecord.assay_level, + MappingRecord.hgvs_assay_level, + Allele.id.label("allele_id"), + Allele.vrs_digest, + Allele.clingen_allele_id, + VepAlleleConsequence.functional_consequence, + sibling_allele.level.label("sibling_level"), + # Exactly one of the sibling's hgvs_g/hgvs_c is populated (it is a nucleotide allele), so + # coalesce yields its canonical string regardless of which nucleotide level it sits at. + func.coalesce(sibling_allele.hgvs_g, sibling_allele.hgvs_c).label("sibling_hgvs"), + ) + .outerjoin(MappingRecord, and_(MappingRecord.variant_id == Variant.id, MappingRecord.live_at(as_of))) + .outerjoin( + MappingRecordAllele, + and_( + MappingRecordAllele.mapping_record_id == MappingRecord.id, + MappingRecordAllele.is_authoritative.is_(True), + MappingRecordAllele.live_at(as_of), + ), + ) + .outerjoin(Allele, Allele.id == MappingRecordAllele.allele_id) + .outerjoin( + VepAlleleConsequence, + and_(VepAlleleConsequence.allele_id == Allele.id, VepAlleleConsequence.live_at(as_of)), + ) + .outerjoin( + sibling_link, + and_( + sibling_link.mapping_record_id == MappingRecord.id, + sibling_link.projection_group == MappingRecordAllele.projection_group, + sibling_link.is_authoritative.is_(False), + sibling_link.live_at(as_of), + ), + ) + .outerjoin(sibling_allele, sibling_allele.id == sibling_link.allele_id) + .where(Variant.score_set_id == score_set_id) + .order_by(*_VARIANT_NUMBER_ORDER) + ) + + +def _apply_pagination(query: Select[Any], start: Optional[int], limit: Optional[int]) -> Select[Any]: + """Apply the shared ``start``/``limit`` window (offset/limit) to a variant query. A falsy ``start`` or + ``limit`` leaves that bound off.""" + if start: + query = query.offset(start) + if limit: + query = query.limit(limit) + + return query + + +def _gnomad_by_allele(db: Session, allele_ids: list[int], *, as_of: Optional[datetime]) -> dict[int, GnomADVariant]: + """The live gnomAD frequency record at the currently-served release (:data:`GNOMAD_DATA_VERSION`) for each + of ``allele_ids``, keyed by allele id — at most one per allele (single-live link, narrowed to the release + the CSV's ``gnomad_af`` column reports).""" + if not allele_ids: + return {} + + rows = db.execute( + select(GnomadAlleleLink.allele_id, GnomADVariant) + .join(GnomADVariant, GnomADVariant.id == GnomadAlleleLink.gnomad_variant_id) + .where(GnomadAlleleLink.allele_id.in_(allele_ids)) + .where(GnomadAlleleLink.live_at(as_of)) + .where(GnomADVariant.db_name == GNOMAD_DB_NAME, GnomADVariant.db_version == GNOMAD_DATA_VERSION) + ) + return {allele_id: gnomad_variant for allele_id, gnomad_variant in rows.tuples()} + + +def _clinvar_by_allele( + db: Session, allele_ids: list[int], clinvar_namespaces: dict[str, str], *, as_of: Optional[datetime] +) -> dict[str, dict[int, ClinvarControl]]: + """For each requested ClinVar namespace, the live control at that release for each of ``allele_ids`` + (``allele id -> control``). ClinVar links are multi-live (one per release), so each namespace's query is + narrowed to its own ``db_version``.""" + by_namespace: dict[str, dict[int, ClinvarControl]] = {} + for ns, db_version in clinvar_namespaces.items(): + per_allele: dict[int, ClinvarControl] = {} + if allele_ids: + rows = db.execute( + select(ClinvarAlleleLink.allele_id, ClinvarControl) + .join(ClinvarControl, ClinvarControl.id == ClinvarAlleleLink.clinvar_control_id) + .where(ClinvarAlleleLink.allele_id.in_(allele_ids)) + .where(ClinvarAlleleLink.live_at(as_of)) + .where(ClinvarControl.db_name == "ClinVar", ClinvarControl.db_version == db_version) + ) + per_allele = {allele_id: control for allele_id, control in rows.tuples()} + + by_namespace[ns] = per_allele + + return by_namespace + + +def _csv_mapped_row(row: Any, protein_hgvs_by_record: dict[int, str]) -> _CsvMappedRow: + """Assemble one variant's :class:`_CsvMappedRow` from a substrate-query row: the canonical post-mapped + HGVS at each level (reconstructed via the shared projection rule) plus the assay-level string, VRS digest, + VEP consequence, and ClinGen id off the authoritative allele.""" + slots = mapped_hgvs_by_level( + assay_level=SequenceLevel(row.assay_level) if row.assay_level else None, + assay_level_hgvs=row.hgvs_assay_level, + sibling_level=row.sibling_level, + sibling_hgvs=row.sibling_hgvs, + protein_hgvs=protein_hgvs_by_record.get(row.mapping_record_id), + ) + return _CsvMappedRow( + hgvs_g=slots.get(SequenceLevel.genomic.value), + hgvs_c=slots.get(SequenceLevel.cdna.value), + hgvs_p=slots.get(SequenceLevel.protein.value), + hgvs_assay_level=row.hgvs_assay_level, + vrs_digest=row.vrs_digest, + vep_functional_consequence=row.functional_consequence, + clingen_allele_id=row.clingen_allele_id, + ) + + +def _fetch_csv_variant_data( + db: Session, + score_set: ScoreSet, + *, + need_mappings: bool, + need_gnomad: bool, + clinvar_namespaces: dict[str, str], + start: Optional[int], + limit: Optional[int], + as_of: Optional[datetime], +) -> _CsvVariantData: + """Fetch the paginated per-variant CSV inputs. When no annotation namespace is requested this is a plain + scan of the score set's variants (no substrate join, ``as_of`` irrelevant); otherwise it joins the live + mapping substrate and batch-fetches the gnomAD/ClinVar dimensions keyed by authoritative allele id.""" + if not need_mappings: + # Fast path: scores/counts (and bare core) are a straight paginated scan in variant-number order, + # untouched by as_of (it only reconstructs the — absent here — annotation layer). + query = _apply_pagination( + select(Variant).where(Variant.score_set_id == score_set.id).order_by(*_VARIANT_NUMBER_ORDER), + start, + limit, + ) + return _CsvVariantData(list(db.scalars(query).all()), None, None, None) + + result = db.execute(_apply_pagination(_csv_substrate_query(score_set.id, as_of=as_of), start, limit)).all() + + # Protein slot: the standalone protein subquery over the whole set, stitched back by record id. + protein_hgvs_by_record = get_protein_hgvs_by_record(db, score_set.id, as_of=as_of) + # The authoritative allele ids on this page — the join key for the batch gnomAD/ClinVar fetches. + allele_ids = [row.allele_id for row in result if row.allele_id is not None] + gnomad_by_allele = _gnomad_by_allele(db, allele_ids, as_of=as_of) if need_gnomad else {} + clinvar_by_allele = _clinvar_by_allele(db, allele_ids, clinvar_namespaces, as_of=as_of) + + variants: list[Variant] = [] + mappings: list[Optional[_CsvMappedRow]] = [] + gnomad_data: Optional[list[Optional[GnomADVariant]]] = [] if need_gnomad else None + clinvar_per_variant: Optional[list[Optional[dict[str, Optional[ClinvarControl]]]]] = ( + [] if clinvar_namespaces else None + ) + for row in result: + variants.append(row.Variant) + allele_id = row.allele_id + mappings.append(_csv_mapped_row(row, protein_hgvs_by_record) if row.mapping_record_id is not None else None) + if gnomad_data is not None: + gnomad_data.append(gnomad_by_allele.get(allele_id) if allele_id is not None else None) + if clinvar_per_variant is not None: + clinvar_per_variant.append( + { + ns: (clinvar_by_allele[ns].get(allele_id) if allele_id is not None else None) + for ns in clinvar_namespaces + } + ) + + return _CsvVariantData(variants, mappings, gnomad_data, clinvar_per_variant) + + def get_score_set_variants_as_csv( db: Session, score_set: ScoreSet, @@ -616,6 +904,7 @@ def get_score_set_variants_as_csv( drop_na_columns: Optional[bool] = None, include_custom_columns: Optional[bool] = True, include_post_mapped_hgvs: Optional[bool] = False, + as_of: Optional[datetime] = None, ) -> str: """ Get the variant data from a score set as a CSV string. @@ -643,180 +932,47 @@ def get_score_set_variants_as_csv( include_post_mapped_hgvs : bool, optional Whether to include post-mapped HGVS notations and VEP functional consequence in the output. Defaults to False. If True, the output will include columns for post-mapped HGVS genomic (g.) and protein (p.) notations, and VEP functional consequence. + as_of : datetime, optional + Reconstruct the annotation layer (post-mapped HGVS, VEP, gnomAD, ClinVar) as it stood at this + instant, over the variant's immutable submitted HGVS/scores/counts. Defaults to the currently-live + rows. Has no effect on the ``scores``/``counts`` namespaces, which are as-of-invariant. Returns _______ str The CSV string containing the variant data. """ - assert type(score_set.dataset_columns) is dict - namespaced_score_set_columns: dict[str, list[str]] = { - "core": ["accession", "hgvs_nt", "hgvs_splice", "hgvs_pro"], - "mavedb": [], - } - if include_post_mapped_hgvs: - namespaced_score_set_columns["mavedb"].append("post_mapped_hgvs_g") - namespaced_score_set_columns["mavedb"].append("post_mapped_hgvs_p") - namespaced_score_set_columns["mavedb"].append("post_mapped_hgvs_c") - namespaced_score_set_columns["mavedb"].append("post_mapped_hgvs_at_assay_level") - namespaced_score_set_columns["mavedb"].append("post_mapped_vrs_digest") - for namespace in namespaces: - namespaced_score_set_columns[namespace] = [] - - if include_custom_columns: - if "scores" in namespaced_score_set_columns: - namespaced_score_set_columns["scores"] = [ - col for col in [str(x) for x in list(score_set.dataset_columns.get("score_columns", []))] - ] - if "counts" in namespaced_score_set_columns: - namespaced_score_set_columns["counts"] = [ - col for col in [str(x) for x in list(score_set.dataset_columns.get("count_columns", []))] - ] - elif "scores" in namespaced_score_set_columns: - namespaced_score_set_columns["scores"].append(REQUIRED_SCORE_COLUMN) - if "vep" in namespaced_score_set_columns: - namespaced_score_set_columns["vep"].append("vep_functional_consequence") - if "gnomad" in namespaced_score_set_columns: - namespaced_score_set_columns["gnomad"].append("gnomad_af") - if "clingen" in namespaced_score_set_columns: - namespaced_score_set_columns["clingen"].append("clingen_allele_id") - - # Parse ClinVar-versioned namespaces of the form "clinvar.YEAR_MONTH". - # The corresponding db_version stored in clinical_controls is "MONTH_YEAR". - clinvar_namespaces: dict[str, str] = {} # namespace -> db_version (MONTH_YEAR) - for ns in namespaces: - m = CLINVAR_NS_PATTERN.match(ns) - if m: - year, month = m.group(1), m.group(2) - db_version = f"{month}_{year}" - clinvar_namespaces[ns] = db_version - namespaced_score_set_columns[ns] = ["clinical_significance", "clinical_review_status"] - - need_mappings = ( - include_post_mapped_hgvs - or "clingen" in namespaces - or "vep" in namespaces - or "gnomad" in namespaces - or bool(clinvar_namespaces) + columns, clinvar_namespaces = _plan_csv_columns( + score_set, + namespaces, + include_custom_columns=include_custom_columns, + include_post_mapped_hgvs=include_post_mapped_hgvs, + ) + need_mappings = bool( + include_post_mapped_hgvs or clinvar_namespaces or ({"clingen", "vep", "gnomad"} & set(namespaces)) ) need_gnomad = "gnomad" in namespaces - variants: list[Variant] = [] - mappings: Optional[list[Optional[MappedVariant]]] = [] if need_mappings else None - gnomad_data: Optional[list[Optional[GnomADVariant]]] = [] if need_gnomad else None - - select_columns: list[Any] = [Variant] - if need_mappings: - select_columns.append(MappedVariant) - if need_gnomad: - select_columns.append(GnomADVariant) - - query = ( - select(*select_columns) - .where(Variant.score_set_id == score_set.id) - .order_by(cast(func.split_part(Variant.urn, "#", 2), Integer)) + data = _fetch_csv_variant_data( + db, + score_set, + need_mappings=need_mappings, + need_gnomad=need_gnomad, + clinvar_namespaces=clinvar_namespaces, + start=start, + limit=limit, + as_of=as_of, ) - if need_mappings: - query = query.join( - MappedVariant, - and_(Variant.id == MappedVariant.variant_id, MappedVariant.current.is_(True)), - isouter=True, - ) - - if need_gnomad: - query = query.join( - MappedVariant.gnomad_variants.of_type(GnomADVariant), - isouter=True, - ).where( - or_( - and_(GnomADVariant.db_name == "gnomAD", GnomADVariant.db_version == "v4.1"), - GnomADVariant.id.is_(None), - ) - ) - - if start: - query = query.offset(start) - if limit: - query = query.limit(limit) - - result = db.execute(query).all() - - for row in result: - variant = row[0] - variants.append(variant) - - if need_mappings and mappings is not None: - mappings.append(row[1]) - - if need_gnomad and gnomad_data is not None: - idx = 2 if need_mappings else 1 - gnomad_data.append(row[idx]) - - # For each ClinVar namespace, fetch a mapping from mapped_variant_id to ClinvarControl. - clinvar_data_map: dict[str, dict[int, Optional[ClinvarControl]]] = {} - if clinvar_namespaces and mappings is not None: - mv_ids = [m.id for m in mappings if m is not None] - for ns, db_version in clinvar_namespaces.items(): - mv_to_cc: dict[int, Optional[ClinvarControl]] = {} - if mv_ids: - aliased_cc = aliased(ClinvarControl) - cc_query = ( - select( - mapped_variants_clinical_controls_association_table.c.mapped_variant_id, - aliased_cc, - ) - .join( - aliased_cc, - mapped_variants_clinical_controls_association_table.c.clinical_control_id == aliased_cc.id, - ) - .where( - and_( - mapped_variants_clinical_controls_association_table.c.mapped_variant_id.in_(mv_ids), - aliased_cc.db_name == "ClinVar", - aliased_cc.db_version == db_version, - ) - ) - ) - for mv_id, cc in db.execute(cc_query).all(): - mv_to_cc[mv_id] = cc - clinvar_data_map[ns] = mv_to_cc - - # Build per-variant ClinVar lookup (list indexed in parallel with variants). - clinvar_per_variant: Optional[list[Optional[dict[str, Optional[ClinvarControl]]]]] = None - if clinvar_namespaces and mappings is not None: - clinvar_per_variant = [] - for mapping in mappings: - row_clinvar: dict[str, Optional[ClinvarControl]] = {} - for ns, mv_to_cc in clinvar_data_map.items(): - if mapping is not None and mapping.id is not None: - row_clinvar[ns] = mv_to_cc.get(mapping.id) - else: - row_clinvar[ns] = None - clinvar_per_variant.append(row_clinvar) - rows_data = variants_to_csv_rows( - variants, - columns=namespaced_score_set_columns, + data.variants, + columns=columns, namespaced=namespaced, - mappings=mappings, - gnomad_data=gnomad_data, - clinvar_data_by_ns=clinvar_per_variant, - ) # type: ignore - - rows_columns = [] - for namespace, cols in namespaced_score_set_columns.items(): - for col in cols: - if CLINVAR_NS_PATTERN.match(namespace): - # ClinVar versioned namespaces always include the full namespace prefix - # to avoid column-name collisions when multiple versions are requested. - rows_columns.append(f"{namespace}.{col}") - elif namespaced and namespace not in ["core", "mavedb"]: - rows_columns.append(f"{namespace}.{col}") - elif namespaced and namespace == "mavedb": - rows_columns.append(f"mavedb.{col}") - else: - rows_columns.append(col) + mappings=data.mappings, + gnomad_data=data.gnomad_data, + clinvar_data_by_ns=data.clinvar_per_variant, + ) + rows_columns = _csv_header_columns(columns, namespaced=namespaced) if drop_na_columns: rows_data, rows_columns = drop_na_columns_from_csv_file_rows(rows_data, rows_columns) @@ -874,7 +1030,7 @@ def is_replaces_id_unique_violation(exc: IntegrityError) -> bool: def variant_to_csv_row( variant: Variant, columns: dict[str, list[str]], - mapping: Optional[MappedVariant] = None, + mapping: Optional["_CsvMappedRow"] = None, gnomad_data: Optional[GnomADVariant] = None, clinvar_data_by_ns: Optional[dict[str, Optional[ClinvarControl]]] = None, namespaced: Optional[bool] = None, @@ -891,8 +1047,8 @@ def variant_to_csv_row( Columns to serialize. namespaced: Optional[bool] = None Namespace the columns or not. - mapping : variant.models.MappedVariant, optional - Mapped variant corresponding to the variant. + mapping : _CsvMappedRow, optional + The variant's mapping-derived fields, resolved from its live mapping record's authoritative allele. gnomad_data : variant.models.GnomADVariant, optional gnomAD variant data corresponding to the variant. clinvar_data_by_ns : dict[str, Optional[ClinvarControl]], optional @@ -921,39 +1077,19 @@ def variant_to_csv_row( # export columns in the `core` namespace without a namespace row[column_key] = value for column_key in columns.get("mavedb", []): + # The canonical post-mapped HGVS at each level (and the assay-level string / VRS digest) are + # pre-resolved onto `mapping` from the authoritative allele + its projection group; no VRS-object + # fallback parse is needed here — the substrate stores the canonical string per level directly. if column_key == "post_mapped_hgvs_g": value = str(mapping.hgvs_g) if mapping and mapping.hgvs_g else na_rep - if value == na_rep: - fallback_hgvs = ( - get_hgvs_from_post_mapped(mapping.post_mapped, combine_cis=True) - if mapping and mapping.post_mapped - else None - ) - if fallback_hgvs is not None and is_hgvs_g(fallback_hgvs): - value = fallback_hgvs - else: - value = na_rep - elif column_key == "post_mapped_hgvs_p": value = str(mapping.hgvs_p) if mapping and mapping.hgvs_p else na_rep - if value == na_rep: - fallback_hgvs = ( - get_hgvs_from_post_mapped(mapping.post_mapped, combine_cis=True) - if mapping and mapping.post_mapped - else None - ) - if fallback_hgvs is not None and is_hgvs_p(fallback_hgvs): - value = fallback_hgvs - else: - value = na_rep - elif column_key == "post_mapped_hgvs_c": value = str(mapping.hgvs_c) if mapping and mapping.hgvs_c else na_rep elif column_key == "post_mapped_hgvs_at_assay_level": value = str(mapping.hgvs_assay_level) if mapping and mapping.hgvs_assay_level else na_rep elif column_key == "post_mapped_vrs_digest": - digest = get_digest_from_post_mapped(mapping.post_mapped) if mapping and mapping.post_mapped else None - value = digest if digest is not None else na_rep + value = mapping.vrs_digest if mapping and mapping.vrs_digest else na_rep if is_null(value): value = na_rep key = f"mavedb.{column_key}" if namespaced else column_key @@ -1022,7 +1158,7 @@ def variant_to_csv_row( def variants_to_csv_rows( variants: Sequence[Variant], columns: dict[str, list[str]], - mappings: Optional[Sequence[Optional[MappedVariant]]] = None, + mappings: Optional[Sequence[Optional["_CsvMappedRow"]]] = None, gnomad_data: Optional[Sequence[Optional[GnomADVariant]]] = None, clinvar_data_by_ns: Optional[Sequence[Optional[dict[str, Optional[ClinvarControl]]]]] = None, namespaced: Optional[bool] = None, @@ -1039,8 +1175,8 @@ def variants_to_csv_rows( Columns to serialize. namespaced: Optional[bool] = None Namespace the columns or not. - mappings : list[Optional[variant.models.MappedVariant]], optional - List of mapped variants corresponding to the variants. + mappings : list[Optional[_CsvMappedRow]], optional + Per-variant mapping-derived fields (parallel to ``variants``). gnomad_data : list[Optional[variant.models.GnomADVariant]], optional List of gnomAD variant data corresponding to the variants. clinvar_data_by_ns : list[Optional[dict[str, Optional[ClinvarControl]]]], optional @@ -1053,7 +1189,7 @@ def variants_to_csv_rows( list[dict[str, Any]] """ n = len(variants) - _mappings: Sequence[Optional[MappedVariant]] = mappings if mappings is not None else [None] * n + _mappings: Sequence[Optional["_CsvMappedRow"]] = mappings if mappings is not None else [None] * n _gnomad: Sequence[Optional[GnomADVariant]] = gnomad_data if gnomad_data is not None else [None] * n _clinvar: Sequence[Optional[dict[str, Optional[ClinvarControl]]]] = ( clinvar_data_by_ns if clinvar_data_by_ns is not None else [None] * n diff --git a/tests/helpers/util/score_set.py b/tests/helpers/util/score_set.py index 7e9022db7..35fd9e382 100644 --- a/tests/helpers/util/score_set.py +++ b/tests/helpers/util/score_set.py @@ -8,8 +8,6 @@ from fastapi.testclient import TestClient from sqlalchemy import select -from mavedb.models.clinical_control import ClinvarControl as ClinicalControlDbModel -from mavedb.models.gnomad_variant import GnomADVariant as GnomADVariantDbModel from mavedb.models.mapped_variant import MappedVariant as MappedVariantDbModel from mavedb.models.score_set import ScoreSet as ScoreSetDbModel from mavedb.models.variant import Variant as VariantDbModel @@ -205,7 +203,7 @@ def create_acc_score_set_with_variants( return score_set -def seed_annotation_substrate(db, score_set, *, skip_first=False): +def seed_annotation_substrate(db, score_set, *, skip_first=False, pre_mapped=None): """Give a score set's variants a live ``MappingRecord`` with one authoritative allele. The VA endpoints build a ``VariantAnnotationContext`` from the ``MappingRecord`` / ``Allele`` @@ -214,7 +212,8 @@ def seed_annotation_substrate(db, score_set, *, skip_first=False): mirrors what the mapping pipeline writes. (Global seeding from ``mock_worker_vrs_mapping`` is deferred to Slice 5.2/5.3, where it can be reconciled with the clinical-controls seeding.) - With ``skip_first=True`` the first variant is left un-mapped on the new substrate (for the + ``pre_mapped`` sets the assayed-level VRS on each record (the flat ``preMapped`` on the variant + detail). With ``skip_first=True`` the first variant is left un-mapped on the new substrate (for the "some variants were not mapped" cases). Returns the score set's variants in id order. """ variants = db.scalars( @@ -227,6 +226,7 @@ def seed_annotation_substrate(db, score_set, *, skip_first=False): seed_mapping_record( db, variant, + pre_mapped=pre_mapped, alleles=[ AlleleSpec( digest=f"va-allele-{variant.id}", @@ -239,6 +239,82 @@ def seed_annotation_substrate(db, score_set, *, skip_first=False): return variants +def seed_csv_substrate( + db, + score_set, + *, + assay_level="genomic", + hgvs_g=None, + hgvs_c=None, + hgvs_p=None, + vep_consequence=None, + clingen_allele_id=None, + gnomad_variant_ids=(), + clinvar_control_ids=(), + annotate="all", + valid_from=None, +): + """Seed every variant of a score set with a live mapping record on the allele substrate, shaped for the + CSV export reader (``get_score_set_variants_as_csv``). + + The authoritative allele sits at ``assay_level`` and carries the annotations the ``vep``/``clingen``/ + ``gnomad``/``clinvar`` namespaces read (VEP consequence, ClinGen id, gnomAD/ClinVar links). For a + nucleotide assay, a projection-group sibling at the other nucleotide level and a protein apex allele are + added so the ``post_mapped_hgvs_{g,c,p}`` triple reconstructs the same way the lean view does. ``hgvs_g``/ + ``hgvs_c``/``hgvs_p`` supply the canonical string at each level (the one at ``assay_level`` is also the + record's ``hgvs_assay_level``). ``annotate`` controls which variants get the authoritative-allele + annotations: ``"all"`` (every variant) or ``"first"`` (only the first, for "one linked control" cases). + """ + variants = db.scalars( + select(VariantDbModel) + .join(ScoreSetDbModel) + .where(ScoreSetDbModel.urn == score_set["urn"]) + .order_by(VariantDbModel.id) + ).all() + level_hgvs = {"genomic": hgvs_g, "cdna": hgvs_c, "protein": hgvs_p} + assay_hgvs = level_hgvs[assay_level] + + for index, variant in enumerate(variants): + annotated = annotate == "all" or index == 0 + specs = [ + AlleleSpec( + digest=f"csv-auth-{variant.id}", + level=assay_level, + is_authoritative=True, + projection_group=0, + hgvs_g=hgvs_g if assay_level == "genomic" else None, + hgvs_c=hgvs_c if assay_level == "cdna" else None, + hgvs_p=hgvs_p if assay_level == "protein" else None, + vep_consequence=vep_consequence if annotated else None, + clingen_allele_id=clingen_allele_id if annotated else None, + gnomad_variant_ids=tuple(gnomad_variant_ids) if annotated else (), + clinvar_control_ids=tuple(clinvar_control_ids) if annotated else (), + ) + ] + # Nucleotide assay: add the other-nucleotide projection sibling and the protein apex so the + # g/c/p triple reconstructs. A protein assay populates only the protein slot (no c/g fan-out). + if assay_level in ("genomic", "cdna"): + other_level = "cdna" if assay_level == "genomic" else "genomic" + if level_hgvs[other_level] is not None: + specs.append( + AlleleSpec( + digest=f"csv-sib-{variant.id}", + level=other_level, + projection_group=0, + hgvs_g=hgvs_g if other_level == "genomic" else None, + hgvs_c=hgvs_c if other_level == "cdna" else None, + ) + ) + if hgvs_p is not None: + specs.append(AlleleSpec(digest=f"csv-prot-{variant.id}", level="protein", hgvs_p=hgvs_p)) + + seed_mapping_record( + db, variant, assay_level=assay_level, hgvs_assay_level=assay_hgvs, alleles=specs, valid_from=valid_from + ) + + return variants + + def link_clinical_controls_to_alleles(db, score_set): """Seed the new-model annotation graph for a score set's first two variants and link the two seeded ClinVar controls to their alleles via ``ClinvarAlleleLink``. @@ -265,41 +341,6 @@ def link_clinical_controls_to_alleles(db, score_set): ) -def link_clinvar_control_to_mapped_variant(db, score_set): - """Link the seeded ClinVar clinical control (id=1) to the first mapped variant of a score set.""" - mapped_variants = db.scalars( - select(MappedVariantDbModel) - .join(VariantDbModel) - .join(ScoreSetDbModel) - .where(ScoreSetDbModel.urn == score_set["urn"]) - ).all() - - mapped_variants[0].clinical_controls.append( - db.scalar(select(ClinicalControlDbModel).where(ClinicalControlDbModel.id == 1)) - ) - - db.add(mapped_variants[0]) - db.commit() - - -def link_gnomad_variants_to_mapped_variants(db, score_set): - mapped_variants = db.scalars( - select(MappedVariantDbModel) - .join(VariantDbModel) - .join(ScoreSetDbModel) - .where(ScoreSetDbModel.urn == score_set["urn"]) - ).all() - - # The first mapped variant gets the gnomAD variant. - mapped_variants[0].gnomad_variants.append( - db.scalar(select(GnomADVariantDbModel).where(GnomADVariantDbModel.id == 1)) - ) - - db.add(mapped_variants[0]) - db.add(mapped_variants[1]) - db.commit() - - def mock_worker_vrs_mapping(client, db, score_set, alleles=True): # The mapping job is tested elsewhere, so insert mapped variants manually. variants = db.scalars( From d53c273f316a4d19a9a14a098c711e88519a3417 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Thu, 23 Jul 2026 11:07:51 -0700 Subject: [PATCH 103/252] feat(variants): add migration stubs for retired mapped-variant routes The mapped-variant endpoints removed earlier on this branch are still live on main, so merging is what actually retires them for public API consumers. Soften that landing per-route, based on whether the replacement is wire-compatible: - GET /score-sets/{urn}/mapped-variants now returns 410 Gone pointing at variant-details, since the streaming NDJSON shape isn't compatible with the old JSON array and a redirect would mislead callers - /mapped-variants/{urn} and its va/* and vrs/{identifier} siblings 301-redirect onto their /variants equivalents, since that move was a straight relocation with an unchanged response shape - POST /variants/clingen-allele-id-lookups stays fully removed with no stub; it was effectively an internal-only route - add a reusable 410 entry to routers/shared.py's BASE_RESPONSES --- src/mavedb/routers/mapped_variant.py | 96 ++++++++++++++++++++++++++++ src/mavedb/routers/score_sets.py | 25 ++++++++ src/mavedb/routers/shared.py | 2 + src/mavedb/server_main.py | 2 + tests/routers/test_mapped_variant.py | 53 +++++++++++++++ tests/routers/test_score_set.py | 13 ++++ 6 files changed, 191 insertions(+) create mode 100644 src/mavedb/routers/mapped_variant.py create mode 100644 tests/routers/test_mapped_variant.py diff --git a/src/mavedb/routers/mapped_variant.py b/src/mavedb/routers/mapped_variant.py new file mode 100644 index 000000000..a42fa2503 --- /dev/null +++ b/src/mavedb/routers/mapped_variant.py @@ -0,0 +1,96 @@ +import logging +from typing import Annotated +from urllib.parse import quote + +from fastapi import APIRouter, Path, Request +from fastapi.responses import RedirectResponse +from ga4gh.core.identifiers import GA4GH_IR_REGEXP + +from mavedb.lib.logging import LoggedRoute +from mavedb.routers.shared import ROUTER_BASE_PREFIX + +TAG_NAME = "Variants" + +logger = logging.getLogger(__name__) + +router = APIRouter( + prefix=f"{ROUTER_BASE_PREFIX}/mapped-variants", + tags=[TAG_NAME], + route_class=LoggedRoute, +) + + +def _redirect(request: Request, target: str) -> RedirectResponse: + url = f"{target}?{request.url.query}" if request.url.query else target + return RedirectResponse(url=url, status_code=301) + + +@router.get( + "/{urn}", + status_code=301, + deprecated=True, + summary="Moved to GET /variants/{urn}", +) +def redirect_mapped_variant(*, urn: str, request: Request) -> RedirectResponse: + """This resource has moved. Use ``GET /variants/{urn}`` instead.""" + return _redirect(request, f"{ROUTER_BASE_PREFIX}/variants/{quote(urn, safe='')}") + + +@router.get( + "/{urn}/va/study-result", + status_code=301, + deprecated=True, + summary="Moved to GET /variants/{urn}/va/study-result", +) +def redirect_mapped_variant_study_result(*, urn: str, request: Request) -> RedirectResponse: + """This resource has moved. Use ``GET /variants/{urn}/va/study-result`` instead.""" + return _redirect(request, f"{ROUTER_BASE_PREFIX}/variants/{quote(urn, safe='')}/va/study-result") + + +@router.get( + "/{urn}/va/functional-statement", + status_code=301, + deprecated=True, + summary="Moved to GET /variants/{urn}/va/functional-statement", +) +def redirect_mapped_variant_functional_impact_statement(*, urn: str, request: Request) -> RedirectResponse: + """This resource has moved. Use ``GET /variants/{urn}/va/functional-statement`` instead.""" + return _redirect(request, f"{ROUTER_BASE_PREFIX}/variants/{quote(urn, safe='')}/va/functional-statement") + + +@router.get( + "/{urn}/va/pathogenicity-statement", + status_code=301, + deprecated=True, + summary="Moved to GET /variants/{urn}/va/pathogenicity-statement", +) +def redirect_mapped_variant_acmg_evidence_line(*, urn: str, request: Request) -> RedirectResponse: + """This resource has moved. Use ``GET /variants/{urn}/va/pathogenicity-statement`` instead.""" + return _redirect(request, f"{ROUTER_BASE_PREFIX}/variants/{quote(urn, safe='')}/va/pathogenicity-statement") + + +@router.get( + "/vrs/{identifier}", + status_code=301, + deprecated=True, + summary="Moved to GET /variants/vrs/{identifier}", +) +def redirect_mapped_variants_by_identifier( + *, + identifier: Annotated[ + str, + Path( + description="String, a valid GA4GH digest based identifier.", + json_schema_extra={"example": "ga4gh:SQ.0123abcd"}, + regex=GA4GH_IR_REGEXP, + ), + ], + request: Request, +) -> RedirectResponse: + """This resource has moved. Use ``GET /variants/vrs/{identifier}`` instead. + + Note that the replacement's ``only_current`` boolean query parameter has been superseded by + ``as_of``; a caller relying on ``only_current=false`` should switch to passing an explicit + ``as_of`` timestamp rather than expecting it to carry over through this redirect. + """ + return _redirect(request, f"{ROUTER_BASE_PREFIX}/variants/vrs/{quote(identifier, safe='')}") diff --git a/src/mavedb/routers/score_sets.py b/src/mavedb/routers/score_sets.py index 6bbed79f8..9b3203726 100644 --- a/src/mavedb/routers/score_sets.py +++ b/src/mavedb/routers/score_sets.py @@ -88,6 +88,7 @@ ACCESS_CONTROL_ERROR_RESPONSES, BASE_400_RESPONSE, BASE_409_RESPONSE, + BASE_RESPONSES, GATEWAY_ERROR_RESPONSES, PUBLIC_ERROR_RESPONSES, ROUTER_BASE_PREFIX, @@ -972,6 +973,30 @@ async def get_score_set_variant_details( ) +@router.get( + "/score-sets/{urn}/mapped-variants", + status_code=410, + deprecated=True, + responses={410: BASE_RESPONSES[410]}, + summary="Removed; see GET /score-sets/{urn}/variant-details", +) +def get_score_set_mapped_variants_removed(*, urn: str) -> Any: + """This endpoint has been permanently removed. + + Its JSON-array response has been replaced by a streaming NDJSON payload with a different + field shape (flat ``preMapped``/``postMapped`` VRS pair rather than a ``MappedVariant``-keyed + record), so the two are not wire-compatible and this route does not redirect. Use + ``GET /score-sets/{urn}/variant-details`` instead. + """ + raise HTTPException( + status_code=410, + detail=( + f"GET /score-sets/{urn}/mapped-variants has been removed. " + f"Use GET /score-sets/{urn}/variant-details instead." + ), + ) + + @router.get( "/score-sets/{urn}/variants/data", status_code=200, diff --git a/src/mavedb/routers/shared.py b/src/mavedb/routers/shared.py index f98edb39c..2cddfd23d 100644 --- a/src/mavedb/routers/shared.py +++ b/src/mavedb/routers/shared.py @@ -8,6 +8,7 @@ 403: {"description": "Forbidden. Insufficient permissions."}, 404: {"description": "Resource not found."}, 409: {"description": "Conflict with current resource state."}, + 410: {"description": "Gone. This resource has been permanently removed."}, 416: {"description": "Requested range not satisfiable."}, 422: {"description": "Unprocessable entity. Validation failed."}, 429: {"description": "Too many requests. Rate limit exceeded."}, @@ -23,6 +24,7 @@ BASE_403_RESPONSE: Mapping[Union[int, str], dict[str, Any]] = {403: BASE_RESPONSES[403]} BASE_404_RESPONSE: Mapping[Union[int, str], dict[str, Any]] = {404: BASE_RESPONSES[404]} BASE_409_RESPONSE: Mapping[Union[int, str], dict[str, Any]] = {409: BASE_RESPONSES[409]} +BASE_410_RESPONSE: Mapping[Union[int, str], dict[str, Any]] = {410: BASE_RESPONSES[410]} BASE_416_RESPONSE: Mapping[Union[int, str], dict[str, Any]] = {416: BASE_RESPONSES[416]} BASE_422_RESPONSE: Mapping[Union[int, str], dict[str, Any]] = {422: BASE_RESPONSES[422]} BASE_429_RESPONSE: Mapping[Union[int, str], dict[str, Any]] = {429: BASE_RESPONSES[429]} diff --git a/src/mavedb/server_main.py b/src/mavedb/server_main.py index d3e8c4221..7109cedff 100644 --- a/src/mavedb/server_main.py +++ b/src/mavedb/server_main.py @@ -52,6 +52,7 @@ hgvs, job_runs, licenses, + mapped_variant, orcid, permissions, pipelines, @@ -106,6 +107,7 @@ app.include_router(job_runs.router) app.include_router(licenses.router) # app.include_router(log.router) +app.include_router(mapped_variant.router) app.include_router(orcid.router) app.include_router(permissions.router) app.include_router(pipelines.router) diff --git a/tests/routers/test_mapped_variant.py b/tests/routers/test_mapped_variant.py new file mode 100644 index 000000000..e118ba192 --- /dev/null +++ b/tests/routers/test_mapped_variant.py @@ -0,0 +1,53 @@ +# ruff: noqa: E402 +"""Router tests for the retired ``/mapped-variants`` surface. + +The router itself is gone (see ``tests/routers/test_variant_annotation.py``); what remains at this +prefix is a set of 301 redirects onto the routes' new home under ``/variants``, kept so external +callers hitting the old public paths get pointed at the replacement instead of a bare 404. +""" + +import pytest + +arq = pytest.importorskip("arq") +cdot = pytest.importorskip("cdot") +fastapi = pytest.importorskip("fastapi") + +from urllib.parse import quote, quote_plus + + +TEST_URN = "urn:mavedb:00000001-a-1#1" + + +def test_redirect_mapped_variant(client): + response = client.get(f"/api/v1/mapped-variants/{quote_plus(TEST_URN)}", follow_redirects=False) + + assert response.status_code == 301 + assert response.headers["location"] == f"/api/v1/variants/{quote_plus(TEST_URN)}" + + +@pytest.mark.parametrize( + "suffix", + ["va/study-result", "va/functional-statement", "va/pathogenicity-statement"], +) +def test_redirect_mapped_variant_va_routes(client, suffix): + response = client.get(f"/api/v1/mapped-variants/{quote_plus(TEST_URN)}/{suffix}", follow_redirects=False) + + assert response.status_code == 301 + assert response.headers["location"] == f"/api/v1/variants/{quote_plus(TEST_URN)}/{suffix}" + + +def test_redirect_mapped_variants_by_identifier(client): + identifier = "ga4gh:VA.0123456789abcdefghijklmnopqrstuv" + response = client.get( + f"/api/v1/mapped-variants/vrs/{identifier}?only_current=false", + follow_redirects=False, + ) + + assert response.status_code == 301 + assert response.headers["location"] == f"/api/v1/variants/vrs/{quote(identifier, safe='')}?only_current=false" + + +def test_redirect_mapped_variants_by_identifier_rejects_malformed_identifier(client): + response = client.get("/api/v1/mapped-variants/vrs/not-a-valid-identifier", follow_redirects=False) + + assert response.status_code == 422 diff --git a/tests/routers/test_score_set.py b/tests/routers/test_score_set.py index 2bc830db3..4ade659ba 100644 --- a/tests/routers/test_score_set.py +++ b/tests/routers/test_score_set.py @@ -4265,6 +4265,19 @@ def test_cannot_get_variant_details_for_nonexistent_score_set(client, setup_rout assert response.status_code == 404 +def test_score_set_mapped_variants_is_permanently_removed(client): + """The old JSON-array ``mapped-variants`` route is gone in favor of the streaming NDJSON + ``variant-details`` route; it returns 410 rather than redirecting since the two are not + wire-compatible.""" + urn = "urn:mavedb:00000001-a-1" + response = client.get(f"/api/v1/score-sets/{urn}/mapped-variants") + + assert response.status_code == 410 + assert response.json()["detail"] == ( + f"GET /score-sets/{urn}/mapped-variants has been removed. Use GET /score-sets/{urn}/variant-details instead." + ) + + def test_get_score_set_variant_details_honors_as_of(client, session, data_provider, data_files, setup_router_db): """``as_of`` time-travels the molecular layer: a far-future instant sees the freshly-seeded substrate as live (the full set), a far-past instant sees no live mapping (an empty stream). Either way 200 — From fbfd4b91d34e6d54179faa10b5c4057321beacfc Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Thu, 23 Jul 2026 22:23:41 -0700 Subject: [PATCH 104/252] fix(measurements): reach a CA's protein consequence unconditionally MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The include_nucleotide_siblings flag never actually gated siblings — reverse translation cross-links every record to its full synonymous nt set, so a sibling's record already links the anchor allele. Remove the flag (api + query param) and make the protein-apex fold-in unconditional for a CA query, so one behavior serves both the variant page and search: a CA reaches its protein consequence even when that protein assay hasn't been reverse-translated (its record links only the protein node). Resolve the apex once in _resolve_protein_apex, and instrument it: warn on a divergent apex (>1 PAID), and publish apex PAID/unregistered counts and the protein-consequence / apex-only-unresolved rates to the request log. Update tests to the one-behavior model. --- src/mavedb/lib/allele_measurements.py | 229 ++++++++++++++++---------- src/mavedb/routers/clingen_alleles.py | 12 -- tests/lib/test_allele_measurements.py | 182 +++++++++++++++----- tests/routers/test_clingen_allele.py | 22 +-- 4 files changed, 298 insertions(+), 147 deletions(-) diff --git a/src/mavedb/lib/allele_measurements.py b/src/mavedb/lib/allele_measurements.py index 6ca2a6d23..6cfb12797 100644 --- a/src/mavedb/lib/allele_measurements.py +++ b/src/mavedb/lib/allele_measurements.py @@ -1,29 +1,36 @@ -"""The ClinGen-allele-centric variant page's measurements list — ``GET /clingen-alleles/{caid}/measurements``. - -Given a ClinGen allele id (a nucleotide ``CA`` or a protein ``PA``), list every measurement whose -**cross-layer equivalence class** touches that allele — not just the exact allele. The equivalence -relation is co-membership in a ``MappingRecord``'s allele set (the mapping + reverse-translation graph, -:mod:`lib.alleles`): a ``CA`` anchors on its nucleotide alleles (genomic + coding share the CA), so the -list gathers the nt measurements of that change (**direct**) *and* the protein measurements of its -consequence (**related**); a ``PA`` anchors on the protein allele, gathering the protein measurements of -that change (**direct**) *and* every nt measurement that encodes it (**related**). The asymmetry falls -out of *which* allele anchors the co-membership — a sibling nt change that also encodes the same protein -is not pulled onto a CA page, because its record links the protein but not the anchor nt allele. -``include_nucleotide_siblings`` opts out of that asymmetry for discovery surfaces — see -:func:`get_allele_measurements`. - -Distinct from :func:`lib.variant_detail.get_variant_detail`, which is the *record-scoped*, all-levels -detail of one *selected* measurement. This list is the *cross-record* union — who else measured -something in the equivalence class. - -Authorization uses ``has_permission`` directly (matching ``lib/score_sets.py``): ``user_data`` is checked -per entity — score-set READ gates whether a measurement appears at all (a private score set's measurement -never leaks), and calibration READ gates the inline classification (withheld while the measurement still -shows). The two-gate split is deliberate: a readable measurement can carry an unreadable classification. -``as_of`` reconstructs the molecular layer (which records/links are live) at a past instant; scores are -immutable and calibrations carry no valid-time, so both are as-of-invariant. +"""The variant page's measurements list — ``GET /clingen-alleles/{caid}/measurements``. + +Given a ClinGen allele id — a nucleotide ``CA`` or a protein ``PA`` — return every measurement related to +it across sequence levels. Two measurements are related when their mapping records share an allele (in the +mapping + reverse-translation graph). A ``CA`` anchors on its nt alleles; a ``PA`` on the protein allele. +Each measurement is labelled by its own measured level relative to the query: ``direct`` (measured at the +query), ``protein_consequence``, or ``nucleotide_encoding``. + +Example. Reference codon ``TTT`` (Phe200): both ``c.598T>C`` and the sibling ``c.600T>A`` encode +``p.Phe200Leu``. Call them ``CA1``, ``CA2``, ``PA1`` — X assays ``c.598T>C`` (nt), Y assays +``p.Phe200Leu`` (protein), Z assays ``c.600T>A`` (nt): + + query CA1 → X direct, Y protein_consequence, Z nucleotide_encoding + query PA1 → Y direct, X and Z nucleotide_encoding + +A CA query reaches its protein consequence (Y) through the shared protein node ``PA1``. This is what makes +Y reachable when a protein assay has not been reverse-translated: its record then links only the protein +node, with no nt allele for the CA anchor to match, so the apex is the only path to it. The sibling Z is +reached without the apex — reverse translation links every record to its full synonymous nt set, so Z's +record already links ``CA1`` directly. There is no page/search distinction: every surface runs one query. + +The protein-consequence step is resolved once, in :func:`_resolve_protein_apex`, which also measures it: +a consequence resolving to more than one protein id (a data-integrity signal) is logged, and the number of +consequences reached with no reverse-translation data of their own is written to the request log. + +Authorization (``has_permission``, per ``lib/score_sets.py``): score-set READ gates whether a measurement +appears; calibration READ gates only its inline classification. ``as_of`` reconstructs which records and +links were live at a past instant; scores and calibrations are as-of-invariant. + +Distinct from :func:`lib.variant_detail.get_variant_detail` — the record-scoped detail of one measurement. """ +import logging from dataclasses import dataclass from datetime import date, datetime from enum import Enum @@ -32,6 +39,7 @@ from sqlalchemy import and_, select from sqlalchemy.orm import Session, aliased, joinedload +from mavedb.lib.logging.context import logging_context, save_to_logging_context from mavedb.lib.permissions import Action, has_permission from mavedb.lib.score_calibrations import calibration_preference_key, classification_evidence_strength from mavedb.lib.types.authentication import UserData @@ -47,31 +55,25 @@ ) from mavedb.models.variant import Variant +logger = logging.getLogger(__name__) + class MeasurementRelationship(str, Enum): - """The relationship of a measurement to the queried ClinGen allele. ``direct`` = the measurement was - assayed *at* this allele; the other two are the RT-related measurements, named for how they relate - to the query. - """ + """How a measurement relates to the queried ClinGen id, by the measurement's *measured* level.""" - # the measurement was assayed *at* this allele - direct = "direct" - # CA query, protein measurement of consequence(N) - protein_consequence = "protein_consequence" - # A nt measurement encoding the queried change's protein consequence: a PA query's encodings of P, or — - # only under ``include_nucleotide_siblings`` — a CA query's sibling nt changes that share consequence(N). + direct = "direct" # assayed at the queried allele + protein_consequence = "protein_consequence" # protein measurement of a CA's consequence + # an nt measurement of a PA's encoding, or a CA's sibling nt change (encodes the same protein) nucleotide_encoding = "nucleotide_encoding" @dataclass(frozen=True) class AlleleMeasurement: - """One measurement in the queried allele's equivalence class (transit; serialized by - ``view_models.allele_measurement``). + """One measurement in the query's equivalence class (transit; serialized by ``view_models``). - ``assay_level`` is the level at which *this* measurement was assayed (protein/cdna/genomic) — the - clinically load-bearing fact, always shown. ``relationship`` says how it relates to the queried - ClinGen id. ``preferred_classification`` is the readable functional classification the UI defaults to - (primary-first cascade, RUO excluded; ``None`` when none applies or the calibration is unreadable). + ``assay_level`` is where this measurement was assayed (always shown, the clinically load-bearing + fact). ``preferred_classification`` is the readable classification the UI defaults to (primary-first + cascade, RUO excluded; ``None`` when none applies or the calibration is unreadable). """ variant_urn: str @@ -87,22 +89,27 @@ class AlleleMeasurement: superseded_by_score_set: Optional[str] +@dataclass(frozen=True) +class _ProteinApex: + """Protein consequence(s) co-membered with a CA anchor's nt alleles — the shared node a CA query + reaches protein measurements through. One PAID is well-formed; ``len(caids) > 1`` is a data-quality + signal (a duplicate or divergent consequence). ``unregistered`` counts apex rows without a PAID yet. + """ + + allele_ids: list[int] + caids: list[str] + unregistered: int + + def _preferred_classification( db: Session, variant: Variant, *, user_data: Optional[UserData] ) -> Optional[ScoreCalibrationFunctionalClassification]: """The variant's preferred *readable* functional classification, or ``None``. - A variant falls into one classification per calibration (non-overlapping ranges); we surface the one - the UI would default to, mirroring its calibration preference cascade: ``primary`` first, then - ``investigator_provided``. Within a tier we take the strongest evidence (the VA-Spec posture — the - strongest calibration is the evidence calibration), then ``id`` only for determinism. - - This is a clinical surface, so research-use-only calibrations are excluded outright (not merely deprioritized - as in the shared cascade) — a RUO call is never the classification shown here. - - Only calibrations the caller can read (calibration READ) are considered, so a private calibration is - skipped rather than blanking a readable call. Kept in sync with the UI's `activeCalibrationOptions` - default selection. + Mirrors the UI's calibration cascade — ``primary`` then ``investigator_provided``, strongest evidence + within a tier, then ``id`` for determinism. RUO calibrations are excluded outright (never shown here), + and calibrations the caller can't read are skipped rather than blanking a readable call. Kept in sync + with the UI's ``activeCalibrationOptions`` default. """ candidates = [ (classification, calibration) @@ -134,13 +141,8 @@ def preference(pair: tuple[ScoreCalibrationFunctionalClassification, ScoreCalibr def _ordering_key(measurement: AlleleMeasurement, published_date: Optional[date]) -> tuple: - """Sort order: - 1. Current measurements first, then superseded (the latter only if ``include_superseded``) - 2. Direct measurements first, then related (protein_consequence / nucleotide_encoding) - 3. Within each, the strongest evidence first (pathogenic wins ties) - 4. Within each, the newest-published first - 5. Within each, the URN for a stable tiebreak - """ + """Sort: current before superseded; direct before related; strongest evidence (pathogenic wins ties) + first; newest-published first; then URN for a stable tiebreak.""" magnitude, direction = classification_evidence_strength(measurement.preferred_classification) return ( 0 if measurement.is_current else 1, @@ -153,22 +155,45 @@ def _ordering_key(measurement: AlleleMeasurement, published_date: Optional[date] ) +def _resolve_protein_apex(db: Session, anchor_ids: list[int], *, as_of: Optional[datetime]) -> _ProteinApex: + """The protein consequence(s) co-membered with the nt ``anchor_ids`` via a shared live record. + + The single site the apex is resolved, so its cardinality (see :class:`_ProteinApex`) is measured + rather than being a silent side effect of the union. Reads ``clingen_allele_id`` to count PAIDs. + """ + anchor_link = aliased(MappingRecordAllele) + rows = db.execute( + select(Allele.id, Allele.clingen_allele_id) + .join(MappingRecordAllele, MappingRecordAllele.allele_id == Allele.id) + .join(anchor_link, anchor_link.mapping_record_id == MappingRecordAllele.mapping_record_id) + .where(anchor_link.allele_id.in_(anchor_ids)) + .where(anchor_link.live_at(as_of)) + .where(MappingRecordAllele.live_at(as_of)) + .where(Allele.level == SequenceLevel.protein.value) + .distinct() + ).all() + + return _ProteinApex( + allele_ids=[row.id for row in rows], + caids=sorted({row.clingen_allele_id for row in rows if row.clingen_allele_id is not None}), + unregistered=sum(1 for row in rows if row.clingen_allele_id is None), + ) + + def get_allele_measurements( db: Session, clingen_allele_id: str, *, user_data: Optional[UserData], include_superseded: bool = False, - include_nucleotide_siblings: bool = False, as_of: Optional[datetime] = None, ) -> list[AlleleMeasurement]: - """List the measurements in ``clingen_allele_id``'s cross-layer equivalence class. Returns - ``[]`` when the id resolves to no allele or no live record. + """The measurements in ``clingen_allele_id``'s equivalence class, or ``[]`` if it resolves to no live + record. - ``include_nucleotide_siblings`` (a ``CA`` entry only; a no-op for ``PA``) widens the class through the - queried change's protein consequence to also surface the *sibling* nt changes — other nucleotide - variants that encode the same amino-acid change and were themselves assayed at the nucleotide level - (``relationship=nucleotide_encoding``). + A CA query returns ``direct`` measurements, its ``protein_consequence`` (reached through the protein + apex, see the module docstring), and the ``nucleotide_encoding`` siblings. A PA query returns its + ``direct`` protein measurements and their ``nucleotide_encoding`` encodings. """ anchor = db.execute(select(Allele.id, Allele.level).where(Allele.clingen_allele_id == clingen_allele_id)).all() if not anchor: @@ -177,22 +202,22 @@ def get_allele_measurements( anchor_ids = [row.id for row in anchor] entry_is_protein = any(row.level == SequenceLevel.protein.value for row in anchor) - # Sibling nucleotide changes (search discovery, CA only): fold the queried change's protein consequence - # — the protein alleles co-membered with the anchor nt alleles — into the anchor, so the single record - # union below also reaches every record encoding that consequence. a PA query already anchors on the protein, so - # this is exactly what makes a CA+siblings query behave like one. (This query would be redundant for a PA entry). - if include_nucleotide_siblings and not entry_is_protein: - anchor_link = aliased(MappingRecordAllele) - anchor_ids += db.scalars( - select(MappingRecordAllele.allele_id) - .join(anchor_link, anchor_link.mapping_record_id == MappingRecordAllele.mapping_record_id) - .join(Allele, Allele.id == MappingRecordAllele.allele_id) - .where(anchor_link.allele_id.in_(anchor_ids)) - .where(anchor_link.live_at(as_of)) - .where(MappingRecordAllele.live_at(as_of)) - .where(Allele.level == SequenceLevel.protein.value) - .distinct() - ).all() + # A CA query always includes its protein consequence, reached through the shared protein node, so it is + # reachable even when a protein assay isn't reverse-translated yet (its record then links only the + # protein node). Transitive, so it's measured: cardinality >1 logged, provenance tallied below. No-op + # for a PA (already anchored on the protein). + apex: Optional[_ProteinApex] = None + if not entry_is_protein: + apex = _resolve_protein_apex(db, anchor_ids, as_of=as_of) + anchor_ids += apex.allele_ids + if len(apex.caids) > 1: + logger.warning( + msg=( + f"ClinGen allele {clingen_allele_id} resolved to {len(apex.caids)} distinct protein " + f"apexes ({', '.join(apex.caids)}); measurements may conflate consequences." + ), + extra=logging_context(), + ) record_ids = db.scalars( select(MappingRecordAllele.mapping_record_id) @@ -203,6 +228,22 @@ def get_allele_measurements( if not record_ids: return [] + # Records that carry a derived (RT) link of their own. A protein consequence whose record has none was + # reached only through the apex of an already resolved protein consequence. + records_with_derived_links: set[int] = ( + set( + db.scalars( + select(MappingRecordAllele.mapping_record_id) + .where(MappingRecordAllele.mapping_record_id.in_(record_ids)) + .where(MappingRecordAllele.is_authoritative.is_(False)) + .where(MappingRecordAllele.live_at(as_of)) + .distinct() + ).all() + ) + if apex is not None + else set() + ) + # Each record's authoritative (measured) allele fixes the assayed level and the direct/related call. authoritative = aliased(MappingRecordAllele) rows = ( @@ -227,12 +268,14 @@ def get_allele_measurements( ) measurements: list[tuple[AlleleMeasurement, Optional[date]]] = [] + protein_consequence_count = 0 + apex_only_unresolved_count = 0 for record, variant, measured_allele in rows: score_set = variant.score_set if not has_permission(user_data, score_set, Action.READ).permitted: continue - # Don't leak unpermitted resources. + # Don't leak an unreadable superseding score set. superseding = score_set.superseding_score_set if superseding is not None and not has_permission(user_data, superseding, Action.READ).permitted: superseding = None @@ -241,9 +284,7 @@ def get_allele_measurements( if not is_current and not include_superseded: continue - # Label by the measured level, not the entry level: direct when the measured allele *is* the query, - # else protein→protein_consequence / nucleotide→nucleotide_encoding. Without siblings the sibling-nt - # branch is unreachable (a record links the anchor only via itself or its protein consequence). + # Label by the measured (authoritative) allele's level relative to the query. if measured_allele.clingen_allele_id == clingen_allele_id: relationship = MeasurementRelationship.direct elif measured_allele.level == SequenceLevel.protein.value: @@ -251,6 +292,13 @@ def get_allele_measurements( else: relationship = MeasurementRelationship.nucleotide_encoding + # Apex-only provenance: a protein consequence whose record has no derived link was reached purely + # through the apex. + if apex is not None and relationship == MeasurementRelationship.protein_consequence: + protein_consequence_count += 1 + if record.id not in records_with_derived_links: + apex_only_unresolved_count += 1 + assay_level = SequenceLevel(record.assay_level) if record.assay_level else None measurement = AlleleMeasurement( variant_urn=variant.urn or "", @@ -267,5 +315,16 @@ def get_allele_measurements( ) measurements.append((measurement, score_set.published_date)) + # Publish the apex provenance to the request log so the ambiguous-apex and apex-only-unresolved rates are observable. + if apex is not None: + save_to_logging_context( + { + "apex_paid_count": len(apex.caids), + "apex_unregistered_count": apex.unregistered, + "measurements_protein_consequence": protein_consequence_count, + "measurements_apex_only_unresolved": apex_only_unresolved_count, + } + ) + measurements.sort(key=lambda pair: _ordering_key(pair[0], pair[1])) return [measurement for measurement, _ in measurements] diff --git a/src/mavedb/routers/clingen_alleles.py b/src/mavedb/routers/clingen_alleles.py index 64c6c1544..b3b626c02 100644 --- a/src/mavedb/routers/clingen_alleles.py +++ b/src/mavedb/routers/clingen_alleles.py @@ -50,16 +50,6 @@ def get_clingen_allele_measurements( "measurements are a deliberate power-user / citation path, never surfaced by discovery." ), ), - include_nucleotide_siblings: bool = Query( - default=False, - description=( - "For a nucleotide (CA) query only: also return sibling nucleotide changes — other DNA " - "variants encoding the same protein consequence that were themselves assayed at the " - "nucleotide level (relationship 'nucleotide_encoding'). Default false: the variant page " - "anchors strictly on the queried allele. Discovery surfaces (search) set it to surface all " - "evidence bearing on the consequence. No-op for a protein (PA) query." - ), - ), as_of: Optional[datetime] = Query( default=None, description=( @@ -82,7 +72,6 @@ def get_clingen_allele_measurements( "requested_resource": clingen_allele_id, "as_of": as_of, "include_superseded": include_superseded, - "include_nucleotide_siblings": include_nucleotide_siblings, } ) response.headers["X-As-Of"] = as_of.isoformat() if as_of is not None else "current" @@ -92,6 +81,5 @@ def get_clingen_allele_measurements( clingen_allele_id, user_data=user_data, include_superseded=include_superseded, - include_nucleotide_siblings=include_nucleotide_siblings, as_of=as_of, ) diff --git a/tests/lib/test_allele_measurements.py b/tests/lib/test_allele_measurements.py index a4621c33b..319caea27 100644 --- a/tests/lib/test_allele_measurements.py +++ b/tests/lib/test_allele_measurements.py @@ -1,26 +1,31 @@ # ruff: noqa: E402 """Integration tests for the ClinGen-allele measurements list (``lib/allele_measurements.py``). -These pin the model that the first pass got wrong: measurements aggregate the **cross-layer equivalence -class** (co-membership in the mapping/RT link graph), not one exact allele, and *which* allele anchors it -determines the direct/related split (a CA anchors on the nt alleles → nt measurements are direct, the -protein consequence is related; a PA anchors on the protein → protein measurements are direct, the nt -encodings related). Also: the narrowness (a sibling nt change is not pulled onto a CA page), the two -authorization gates (score-set READ hides the measurement; calibration READ withholds only the inline -classification) exercised through the real ``has_permission``, superseded opt-in, and the default ordering -(direct-first, strongest evidence, pathogenic-biased). +These pin the model: measurements aggregate the **cross-layer equivalence class** (co-membership in the +mapping/RT link graph), not one exact allele, and *which* allele anchors it determines the direct/related +split (a CA anchors on the nt alleles → nt measurements are direct, the protein consequence and the sibling +nt encodings are related; a PA anchors on the protein → protein measurements are direct, the nt encodings +related). Also: the protein apex reaches a consequence whose assay isn't reverse-translated (with its +cardinality/provenance instrumentation), the two authorization gates (score-set READ hides the measurement; +calibration READ withholds only the inline classification) exercised through the real ``has_permission``, +superseded opt-in, and the default ordering (direct-first, strongest evidence, pathogenic-biased). """ +import logging + import pytest pytest.importorskip("psycopg2") from datetime import date +from starlette_context import context, request_cycle_context + from mavedb.lib.allele_measurements import ( AlleleMeasurement, MeasurementRelationship, _ordering_key, + _resolve_protein_apex, get_allele_measurements, ) from mavedb.lib.types.authentication import UserData @@ -204,26 +209,33 @@ def test_pa_entry_swaps_direct_and_related(session, setup_lib_db_with_score_set) @pytest.mark.integration -def test_sibling_nt_not_pulled_onto_ca_page(session, setup_lib_db_with_score_set): - """A sibling nt change that encodes the *same* protein is not pulled onto a CA page (its record links - the protein but not this nt allele) — but both nt changes show on the protein's PA page as encodings.""" +def test_sibling_nt_shown_on_ca_page(session, setup_lib_db_with_score_set): + """Reverse translation links every record to its full synonymous nt set, so a sibling nt change's record + links the anchor allele directly and the sibling shows on the CA page as ``nucleotide_encoding`` (after + the direct measurement). Both nt changes also show on the shared protein's PA page as encodings.""" score_set = setup_lib_db_with_score_set nt1 = _allele(session, "nt-1", level="cdna", clingen_allele_id="CA111") nt2 = _allele(session, "nt-2", level="cdna", clingen_allele_id="CA222") prot = _allele(session, "prot-P", level="protein", clingen_allele_id="PA9") + # Real RT linking: each record links its authoritative nt allele, the sibling nt allele (an RT + # candidate), and the shared protein consequence. a = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}) ra = _record(session, a, assay_level="cdna") _link(session, ra, nt1, is_authoritative=True) + _link(session, ra, nt2) _link(session, ra, prot) c = _variant(session, score_set, 2, data={"score_data": {"score": 1.0}}) rc = _record(session, c, assay_level="cdna") _link(session, rc, nt2, is_authoritative=True) + _link(session, rc, nt1) _link(session, rc, prot) ca_page = get_allele_measurements(session, "CA111", user_data=_user_data(session)) - assert {m.variant_urn for m in ca_page} == {a.urn} + assert {m.variant_urn: m.relationship for m in ca_page} == {a.urn: "direct", c.urn: "nucleotide_encoding"} + # Direct precedes the sibling encoding. + assert [m.variant_urn for m in ca_page] == [a.urn, c.urn] pa_page = get_allele_measurements(session, "PA9", user_data=_user_data(session)) assert {m.variant_urn for m in pa_page} == {a.urn, c.urn} @@ -231,36 +243,132 @@ def test_sibling_nt_not_pulled_onto_ca_page(session, setup_lib_db_with_score_set @pytest.mark.integration -def test_sibling_nt_pulled_onto_ca_page_with_flag(session, setup_lib_db_with_score_set): - """``include_nucleotide_siblings`` widens a CA page through the protein consequence: the sibling nt - change encoding the same protein is pulled in as a ``nucleotide_encoding`` while the direct measurement - is unchanged. A no-op for a PA query, which already returns every encoding.""" +def test_resolve_protein_apex_single(session, setup_lib_db_with_score_set): + """The resolver returns the one protein consequence co-membered with the anchor nt allele, its PAID, + and no unregistered rows.""" score_set = setup_lib_db_with_score_set - nt1 = _allele(session, "nt-1", level="cdna", clingen_allele_id="CA111") - nt2 = _allele(session, "nt-2", level="cdna", clingen_allele_id="CA222") + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") prot = _allele(session, "prot-P", level="protein", clingen_allele_id="PA9") + v = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}) + r = _record(session, v, assay_level="cdna") + _link(session, r, nt, is_authoritative=True) + _link(session, r, prot) - a = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}) - ra = _record(session, a, assay_level="cdna") - _link(session, ra, nt1, is_authoritative=True) - _link(session, ra, prot) + apex = _resolve_protein_apex(session, [nt.id], as_of=None) + assert apex.allele_ids == [prot.id] + assert apex.caids == ["PA9"] + assert apex.unregistered == 0 - c = _variant(session, score_set, 2, data={"score_data": {"score": 1.0}}) - rc = _record(session, c, assay_level="cdna") - _link(session, rc, nt2, is_authoritative=True) - _link(session, rc, prot) - result = get_allele_measurements(session, "CA111", user_data=_user_data(session), include_nucleotide_siblings=True) - by_urn = {m.variant_urn: m for m in result} - assert set(by_urn) == {a.urn, c.urn} - assert by_urn[a.urn].relationship == "direct" - assert by_urn[c.urn].relationship == "nucleotide_encoding" - # Display order: the directly-measured change precedes its sibling nucleotide encoding. - assert [m.variant_urn for m in result] == [a.urn, c.urn] - - # No-op for a protein anchor — it already returns every nt encoding. - pa = get_allele_measurements(session, "PA9", user_data=_user_data(session), include_nucleotide_siblings=True) - assert {m.variant_urn for m in pa} == {a.urn, c.urn} +@pytest.mark.integration +def test_resolve_protein_apex_unregistered(session, setup_lib_db_with_score_set): + """An apex protein allele with no CAID yet counts toward ``unregistered`` and contributes no PAID.""" + score_set = setup_lib_db_with_score_set + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + prot = _allele(session, "prot-P", level="protein", clingen_allele_id=None) + v = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}) + r = _record(session, v, assay_level="cdna") + _link(session, r, nt, is_authoritative=True) + _link(session, r, prot) + + apex = _resolve_protein_apex(session, [nt.id], as_of=None) + assert apex.allele_ids == [prot.id] + assert apex.caids == [] + assert apex.unregistered == 1 + + +@pytest.mark.integration +def test_widening_ambiguous_apex_is_loud(session, setup_lib_db_with_score_set, caplog): + """An anchor whose nt change is co-membered with two *different* protein apexes (a divergent walk) is + surfaced loudly: a warning naming the conflicting PAIDs, and ``apex_paid_count == 2`` in the request + log.""" + score_set = setup_lib_db_with_score_set + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + prot1 = _allele(session, "prot-1", level="protein", clingen_allele_id="PA1") + prot2 = _allele(session, "prot-2", level="protein", clingen_allele_id="PA2") + + # nt is the measured allele of r1 (→ PA1) and a derived member of r2 (→ PA2): the same nt change + # resolving to two protein consequences. + v1 = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}) + r1 = _record(session, v1, assay_level="cdna") + _link(session, r1, nt, is_authoritative=True) + _link(session, r1, prot1) + + v2 = _variant(session, score_set, 2, data={"score_data": {"score": 1.0}}, hgvs_pro="p.Met1Leu") + r2 = _record(session, v2, assay_level="protein") + _link(session, r2, prot2, is_authoritative=True) + _link(session, r2, nt) + + with request_cycle_context({}): + with caplog.at_level(logging.WARNING, logger="mavedb.lib.allele_measurements"): + get_allele_measurements(session, "CA123", user_data=_user_data(session)) + assert context["apex_paid_count"] == 2 + + assert any("PA1" in rec.message and "PA2" in rec.message for rec in caplog.records) + + +@pytest.mark.integration +def test_widening_provenance_apex_only_unresolved(session, setup_lib_db_with_score_set): + """A protein measurement reached only through the apex, whose own record carries no derived link, is + counted apex-only-unresolved; giving that record a derived link of its own drops the count.""" + score_set = setup_lib_db_with_score_set + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + prot = _allele(session, "prot-P", level="protein", clingen_allele_id="PA9") + + # nt measurement: RT ran — its record links nt (authoritative) and the protein consequence. + nt_measured = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}, hgvs_nt="c.1A>T") + nt_record = _record(session, nt_measured, assay_level="cdna") + _link(session, nt_record, nt, is_authoritative=True) + _link(session, nt_record, prot) + + # protein measurement: RT has NOT run — its record links only its authoritative protein allele. + protein_measured = _variant(session, score_set, 2, data={"score_data": {"score": 1.0}}, hgvs_pro="p.Met1Leu") + protein_record = _record(session, protein_measured, assay_level="protein") + _link(session, protein_record, prot, is_authoritative=True) + + with request_cycle_context({}): + result = get_allele_measurements(session, "CA123", user_data=_user_data(session)) + # Both surface via the apex; the protein measurement is the apex-only-unresolved one. + assert {m.variant_urn for m in result} == {nt_measured.urn, protein_measured.urn} + assert context["measurements_protein_consequence"] == 1 + assert context["measurements_apex_only_unresolved"] == 1 + + # Once the protein record gains a derived (non-authoritative) link of its own, it is resolved. + _link(session, protein_record, nt) + with request_cycle_context({}): + get_allele_measurements(session, "CA123", user_data=_user_data(session)) + assert context["measurements_apex_only_unresolved"] == 0 + + +@pytest.mark.integration +def test_ca_page_reaches_apex_only_protein_consequence(session, setup_lib_db_with_score_set): + """The variant page (no siblings) reaches a protein measurement of the anchor's consequence even when + that measurement's record has no nt link of its own (RT never ran on its score set) — through the + shared apex — and publishes the apex provenance to the request log.""" + score_set = setup_lib_db_with_score_set + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + prot = _allele(session, "prot-P", level="protein", clingen_allele_id="PA9") + + # nt measurement: RT ran — its record links nt (authoritative) and the protein consequence. + nt_measured = _variant(session, score_set, 1, data={"score_data": {"score": -2.0}}, hgvs_nt="c.1A>T") + nt_record = _record(session, nt_measured, assay_level="cdna") + _link(session, nt_record, nt, is_authoritative=True) + _link(session, nt_record, prot) + + # protein measurement: RT has NOT run — its record links only its authoritative protein allele. + protein_measured = _variant(session, score_set, 2, data={"score_data": {"score": 1.0}}, hgvs_pro="p.Met1Leu") + protein_record = _record(session, protein_measured, assay_level="protein") + _link(session, protein_record, prot, is_authoritative=True) + + with request_cycle_context({}): + result = get_allele_measurements(session, "CA123", user_data=_user_data(session)) + by_urn = {m.variant_urn: m for m in result} + # Reached through the apex despite no nt link on the protein measurement's record. + assert set(by_urn) == {nt_measured.urn, protein_measured.urn} + assert by_urn[protein_measured.urn].relationship == "protein_consequence" + # The page path now widens, so it emits the apex instrumentation. + assert context["apex_paid_count"] == 1 + assert context["measurements_apex_only_unresolved"] == 1 @pytest.mark.integration diff --git a/tests/routers/test_clingen_allele.py b/tests/routers/test_clingen_allele.py index 8c01ec8ef..7ac06e709 100644 --- a/tests/routers/test_clingen_allele.py +++ b/tests/routers/test_clingen_allele.py @@ -108,10 +108,9 @@ def test_superseded_measurement_is_opt_in(client, session, data_provider, data_f assert body[0]["supersededByScoreSet"] == newer["urn"] -def test_nucleotide_siblings_are_opt_in(client, session, data_provider, data_files, setup_router_db): - """The sibling nt bucket is a discovery opt-in: a different DNA variant encoding the same protein - consequence is absent by default and pulled in as a ``nucleotide_encoding`` under - ``include_nucleotide_siblings``.""" +def test_nucleotide_siblings_shown(client, session, data_provider, data_files, setup_router_db): + """A different DNA variant encoding the same protein consequence surfaces on a CA page as a + ``nucleotide_encoding``, reached through the shared protein consequence.""" experiment = create_experiment(client) score_set = create_seq_score_set_with_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" @@ -133,20 +132,17 @@ def test_nucleotide_siblings_are_opt_in(client, session, data_provider, data_fil session.add(MappingRecordAllele(mapping_record_id=record.id, allele_id=prot.id)) session.commit() - default = client.get("/api/v1/clingen-alleles/CA111/measurements") - assert {m["variantUrn"] for m in default.json()} == {f"{score_set['urn']}#1"} - - widened = client.get("/api/v1/clingen-alleles/CA111/measurements", params={"include_nucleotide_siblings": True}) - assert widened.status_code == 200 - by_urn = {m["variantUrn"]: m for m in widened.json()} + response = client.get("/api/v1/clingen-alleles/CA111/measurements") + assert response.status_code == 200 + by_urn = {m["variantUrn"]: m for m in response.json()} assert set(by_urn) == {f"{score_set['urn']}#1", f"{score_set['urn']}#2"} assert by_urn[f"{score_set['urn']}#1"]["relationship"] == "direct" assert by_urn[f"{score_set['urn']}#2"]["relationship"] == "nucleotide_encoding" def test_direct_measurements_sort_before_related(client, session, data_provider, data_files, setup_router_db): - """The list is ordered, and direct measurements precede related ones in the body: a CA+siblings query - returns the directly-assayed change first, then the sibling nucleotide encoding.""" + """The list is ordered, and direct measurements precede related ones in the body: a CA query returns + the directly-assayed change first, then the sibling nucleotide encoding.""" experiment = create_experiment(client) score_set = create_seq_score_set_with_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" @@ -167,7 +163,7 @@ def test_direct_measurements_sort_before_related(client, session, data_provider, session.add(MappingRecordAllele(mapping_record_id=record.id, allele_id=prot.id)) session.commit() - response = client.get("/api/v1/clingen-alleles/CA111/measurements", params={"include_nucleotide_siblings": True}) + response = client.get("/api/v1/clingen-alleles/CA111/measurements") assert response.status_code == 200 body = response.json() From a9c2f404f672261f394e97ced7063e5823f20189 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Fri, 24 Jul 2026 08:00:28 -0700 Subject: [PATCH 105/252] feat(cleanup): extend job timeouts Even with higher timeout settings, some VEP jobs were still hitting against the timeout window. Given the new checkpoint features, increasing the coarse timeout is lower risk and, in the aggregate, will help more jobs run to completion. --- src/mavedb/worker/jobs/system/cleanup.py | 6 +++--- src/mavedb/worker/settings/worker.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/mavedb/worker/jobs/system/cleanup.py b/src/mavedb/worker/jobs/system/cleanup.py index 4a2dc975e..40e636be6 100644 --- a/src/mavedb/worker/jobs/system/cleanup.py +++ b/src/mavedb/worker/jobs/system/cleanup.py @@ -46,11 +46,11 @@ # how long it has legitimately been running. This is what lets a multi-hour VEP fan-out run to # completion without being killed, while still catching a job that has genuinely hung. # 2. RUNNING_TIMEOUT_MINUTES — a wall-clock backstop for jobs that never emit progress (or whose -# heartbeat column is somehow stuck). Sits ~30 min under ArqWorkerSettings.job_timeout (8h) so +# heartbeat column is somehow stuck). Sits ~30 min under ArqWorkerSettings.job_timeout (24h) so # the DB-driven sweeper, not ARQ's hard kill, is what terminates a job — keeping DB state and # actual work in sync. -PROGRESS_STALL_MINUTES = 20 # RUNNING jobs should checkpoint progress at least this often -RUNNING_TIMEOUT_MINUTES = 450 # Wall-clock backstop (7.5h), 30 min under the 8h ARQ ceiling +PROGRESS_STALL_MINUTES = 30 # RUNNING jobs should checkpoint progress at least this often +RUNNING_TIMEOUT_MINUTES = 1410 # Wall-clock backstop (23.5h), 30 min under the 24h ARQ ceiling PENDING_TIMEOUT_MINUTES = 5 # PENDING jobs which are actionable within pipelines should be enqueued within 5 minutes PIPELINE_STUCK_TIMEOUT_MINUTES = ( 5 # Pipelines in non-terminal states with no active jobs should resolve within 5 minutes diff --git a/src/mavedb/worker/settings/worker.py b/src/mavedb/worker/settings/worker.py index 3b0ab4e89..f205fae7e 100644 --- a/src/mavedb/worker/settings/worker.py +++ b/src/mavedb/worker/settings/worker.py @@ -25,13 +25,13 @@ # driver, enabling incremental migration of job functions without touching # the FastAPI layer. Once all jobs use async sessions, raise MAX_JOBS to 10+. MAX_JOBS = 2 -# ARQ's hard coroutine-kill ceiling. Kept deliberately high (8h) so ARQ is the *last* resort: +# ARQ's hard coroutine-kill ceiling. Kept deliberately high (24h) so ARQ is the *last* resort: # our own DB-driven sweeper (cleanup_stalled_jobs) should detect and recover stalls long before # this fires, keeping the JobRun state machine in sync with reality. VEP fan-out over large allele # sets can legitimately run for hours, so a low ceiling would kill healthy work. # cleanup's RUNNING_TIMEOUT_MINUTES backstop sits ~30 min under this to try and sweep these jobs before # ARQ's hard timeout fires. -JOB_TIMEOUT_SECONDS = 8 * 60 * 60 # 8 hours +JOB_TIMEOUT_SECONDS = 24 * 60 * 60 # 24 hours class ArqWorkerSettings: From 7ac06672aaa913a8dbbc22a84e3eba538bf15937 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Mon, 17 Aug 2026 11:05:12 -0700 Subject: [PATCH 106/252] feat(cat-vrs): emit spec-alignment ConceptMappings for relation codes MaveDB's Cat-VRS relation vocabulary is member->defining framed and cannot be expressed as a subclass of the spec's own Relation enum, so interop was previously implicit (matching code strings only). Codes that are an exact match for a published Cat-VRS Relation term now carry an explicit ConceptMapping to it, so downstream consumers can resolve equivalence without guessing from naming. - extract lib/term_systems.py: a shared (system URI, prefix) -> Coding builder, replacing the single ad hoc _RELATION_SYSTEM constant so id/system pairs can't drift apart - add _SPEC_TERM_SYSTEM / _SPEC_EQUIVALENT tables in lib/cat_vrs.py mapping MaveDB relation codes to their Cat-VRS spec equivalents where one exists - add drift-guard tests asserting every MaveDB relation is either mapped or a declared gap, and that newly published spec terms get triaged --- src/mavedb/lib/cat_vrs.py | 126 +++++++++++++++++++++------------ src/mavedb/lib/term_systems.py | 36 ++++++++++ tests/lib/test_cat_vrs.py | 78 +++++++++++++++++++- tests/lib/test_term_systems.py | 14 ++++ 4 files changed, 208 insertions(+), 46 deletions(-) create mode 100644 src/mavedb/lib/term_systems.py create mode 100644 tests/lib/test_term_systems.py diff --git a/src/mavedb/lib/cat_vrs.py b/src/mavedb/lib/cat_vrs.py index 02e1d1d03..b551472a2 100644 --- a/src/mavedb/lib/cat_vrs.py +++ b/src/mavedb/lib/cat_vrs.py @@ -1,18 +1,7 @@ """Cat-VRS transit, built on the fly from a variant's live allele links. -The Categorical Variant served on ``/variants/{urn}`` is **not** materialized — it is assembled -per request from the variant's live ``MappingRecordAllele`` links (see -``lib/alleles.py::get_live_record_allele_links``). A stored skeleton would be a denormalized cache -of that link graph with a real sync cost and no benefit, since the only consumer is this bounded -single-variant build (full-scan paths never assemble Cat-VRS). The output is response-only and -HTTP-cacheable. - -This targets Cat-VRS spec **1.0.0** (``ga4gh.cat_vrs`` package 0.7.2). In that schema a -``CategoricalVariant``'s ``members`` are bare VRS variations with no per-member relation field; -relations ride on the ``DefiningAlleleConstraint`` as a list of ``MappableConcept``. The MaveDB -relation codes (read member -> defining) therefore cannot live on individual members, so the -per-member mapping is returned alongside, keyed by VRS digest, for the MaveDB layer that rides -beside the spec-pure object. +The Categorical Variant served on ``/variants/{urn}`` is assembled per request from the variant's +live ``MappingRecordAllele`` links (see ``lib/alleles.py::get_live_record_allele_links``). """ import logging @@ -21,14 +10,17 @@ from enum import Enum from typing import Optional -from ga4gh.cat_vrs.models import CategoricalVariant, DefiningAlleleConstraint, MappableConcept -from ga4gh.core.models import Coding, iriReference +from ga4gh.cat_vrs.models import CategoricalVariant, DefiningAlleleConstraint, MappableConcept, Relation +from ga4gh.core.models import ConceptMapping, iriReference +from ga4gh.core.models import Relation as MappingRelation from ga4gh.vrs.models import Allele as VrsAllele from ga4gh.vrs.models import CisPhasedBlock from sqlalchemy.orm import Session from mavedb.lib.alleles import get_live_record_allele_links from mavedb.lib.logging.context import logging_context +from mavedb.lib.term_systems import GKS_ALLELE_RELATION, MAVEDB_CAT_VRS_RELATION, SEQUENCE_ONTOLOGY, TermSystem +from mavedb.lib.term_systems import coding as term_coding from mavedb.lib.vrs import vrs_object_from_mapped_variant from mavedb.models.allele import Allele from mavedb.models.enums.sequence_level import NUCLEOTIDE_LEVELS, SequenceLevel @@ -36,14 +28,29 @@ logger = logging.getLogger(__name__) -# MaveDB-namespaced relation codes, read as `member -> defining`. Namespaced for MaveDB now, -# TODO: Harmonize with GA4GH Cat-VRS group for standardization. -_RELATION_SYSTEM = "https://mavedb.org/cat-vrs/relations" +# Cat-VRS does not attribute its own Relation values to one uniform system: translation_of and +# transcribed_to are Sequence Ontology terms, while liftover_to has no ontology equivalent and is +# carried on Cat-VRS's own internal vocabulary instead. Per-term, not a single constant, so that +# extending `_SPEC_EQUIVALENT` can't silently mislabel a future mapping with the wrong system. +# +# Sourced from the spec's published examples/schema (ga4gh/cat-vrs recipes-source.yaml and +# examples/json/proteinSequenceConsequence-ex2.json). +_SPEC_TERM_SYSTEM: dict[Relation, TermSystem] = { + Relation.TRANSLATION_OF: SEQUENCE_ONTOLOGY, + Relation.TRANSCRIBED_TO: SEQUENCE_ONTOLOGY, + Relation.LIFTOVER_TO: GKS_ALLELE_RELATION, +} -# TODO: Verify Cat-VRS relation names with GA4GH Cat-VRS group; these are the MaveDB-namespaced codes we raised for standardization. class CatVrsRelation(str, Enum): - """Relation of a member allele to the single defining (measured) allele.""" + """Relation of a member allele to the single defining (measured) allele. + + Deliberately **not** a subclass of Cat-VRS's ``Relation``: Python forbids extending an enum that + already has members, and inheriting would buy nothing anyway. Interop here is carried by + ``Coding.system`` on the emitted concept, not by Python enum identity — see + :func:`_relation_concept`, which maps the codes that have a spec equivalent onto it. Keeping this a + plain enum keeps the members statically visible to mypy and greppable. + """ # defining is protein; member is an nt allele (coding or genomic) that encodes it. Implied. ENCODES = "encodes" @@ -51,11 +58,17 @@ class CatVrsRelation(str, Enum): COORDINATE_REPRESENTATION_OF = "coordinate_representation_of" # defining is nt; member is the protein consequence. Consequence, no independent score. TRANSLATION_OF = "translation_of" - # defining is nt; member is a *synonymous cousin* — an nt allele in a different projection group that + # defining is nt; member is a *convergent encoding* — an nt allele in a different projection group that # encodes the same protein consequence as the measured change. CO_ENCODES = "co_encodes" +# MaveDB code -> the Cat-VRS term it is an exact match for. A code with no entry has no spec equivalent. +_SPEC_EQUIVALENT: dict[CatVrsRelation, Relation] = { + CatVrsRelation.TRANSLATION_OF: Relation.TRANSLATION_OF, +} + + class CatVrsMode(str, Enum): """The score-collapse semantics of the categorical variant.""" @@ -69,14 +82,16 @@ class CategoricalVariantTransit: categorical_variant: CategoricalVariant mode: CatVrsMode - # member vrs_digest -> relation (member -> defining). Excludes the defining allele itself. member_relations: dict[str, CatVrsRelation] + """vrs_digest keyed dict encoding digest: relation for the member -> defining relationship. + Excludes the defining allele itself. + """ -def is_convergent_cousin( +def is_convergent_encoding( member_level: Optional[str], member_group: Optional[int], *, defining_group: Optional[int] ) -> bool: - """Whether a projection-mode member is a *synonymous cousin* of the measured change. + """Whether a projection-mode member is a *convergent encoding* of the measured change. Concretely: - The defining (measured) allele is nt (genomic or cdna). @@ -101,8 +116,9 @@ def _relation_for( """Derive the member -> defining relation from the two levels, or None for the defining itself. Star model: every member relates to the single defining allele, so the relation depends only on the - (defining, member) level pair — plus, in projection mode, on whether the nt member shares the measured - change's ``projection_group`` (its coordinate partner) or lives in another group (a synonymous cousin). + (defining, member) level pair. Note that in projection mode relation also depends on whether the nt + member shares the measured change's ``projection_group`` (its projection) or lives in another group + (a convergent encoding). """ # Protein measured. Every nt member encodes it, the protein member is the defining. Grouping is irrelevant. if defining_level == SequenceLevel.protein.value: @@ -112,11 +128,12 @@ def _relation_for( if member_level == SequenceLevel.protein.value: return CatVrsRelation.TRANSLATION_OF if member_level in NUCLEOTIDE_LEVELS: - if is_convergent_cousin(member_level, member_group, defining_group=defining_group): + if is_convergent_encoding(member_level, member_group, defining_group=defining_group): return CatVrsRelation.CO_ENCODES return CatVrsRelation.COORDINATE_REPRESENTATION_OF - return None # pragma: no cover -- the levels are constrained by the DB and the mapping job + # Unreachable in practice. Levels are constrained by the DB and the mapping job. + return None # pragma: no cover def _is_precise_projection_member( @@ -125,20 +142,39 @@ def _is_precise_projection_member( """In projection mode, whether a member is a *precise* representation of the measured change. True for the protein consequence (the apex, shared by the whole equivalence class) and the measured - change's own coordinate partner — the other member of its ``projection_group`` (a c↔g pair). False for - the synonymous cousins in other projection groups. + change's own projection. False for convergent encodings in other projection groups. """ if member_level == SequenceLevel.protein.value: return True - return not is_convergent_cousin(member_level, member_group, defining_group=defining_group) + return not is_convergent_encoding(member_level, member_group, defining_group=defining_group) def _relation_concept(relation: CatVrsRelation) -> MappableConcept: - """Wrap a MaveDB relation code as a Cat-VRS ``MappableConcept`` for the constraint.""" + """Wrap a MaveDB relation code as a Cat-VRS ``MappableConcept``. + + The MaveDB code is always the ``primaryCoding``. Codes are read *member -> defining*, which is + MaveDB's own framing, so claiming the spec's system for them outright would misrepresent their + provenance. Where a code *is* the same concept as a published Cat-VRS relation, an ``exactMatch`` + :class:`ConceptMapping` to that term is attached via ``MappableConcept.mappings``. + + A code with no spec equivalent carries no mapping. See :data:`_SPEC_EQUIVALENT` for which spec + gaps exist and why. + """ + spec_term = _SPEC_EQUIVALENT.get(relation) return MappableConcept( name=relation.value, - primaryCoding=Coding(id=f"mavedb:{relation.value}", code=relation.value, system=_RELATION_SYSTEM), + primaryCoding=term_coding(MAVEDB_CAT_VRS_RELATION, relation.value), + mappings=( + [ + ConceptMapping( + coding=term_coding(_SPEC_TERM_SYSTEM[spec_term], spec_term.value), + relation=MappingRelation.EXACT_MATCH, + ) + ] + if spec_term is not None + else None + ), ) @@ -162,25 +198,25 @@ def build_categorical_variant( ) -> Optional[CategoricalVariantTransit]: """Assemble a Cat-VRS ``CategoricalVariant`` from a variant's live allele links. - ``links`` is the record-scoped live link set from ``get_live_record_allele_links`` — exactly one + ``links`` is the record-scoped live link set from ``get_live_record_allele_links``. Exactly one is authoritative (the measured/defining allele), the rest are derived members. Returns ``None`` when there is no authoritative, hydratable link to anchor on (e.g. an unmapped variant). **The categorical variant always anchors on the measured allele**. Member selection therefore differs by mode: - **Reverse translation** (protein measured): the defining allele is the protein change and every - nt allele in the record ``encodes`` it, so the **full equivalence class** is unfurled — that class + nt allele in the record ``encodes`` it, so the **full equivalence class** is unfurled. That class is exactly what the protein measurement's claim ranges over. ``include_convergent`` is inert here. - - **Projection** (nt measured): the record also carries the reverse-translation fan — the synonymous - *cousins* (other nt encoders of the same protein consequence, in different projection groups), which - are distinct, unmeasured variants rather than representations of the measured change. - ``include_convergent`` selects how they are handled: + - **Projection** (nt measured): the record also carries the reverse-translation fan out. This includes + all synonymous *convergent encodings* (other nt alleles encoding the same protein consequence, in + different projection groups), which are distinct, unmeasured variants rather than representations + of the measured change. ``include_convergent`` selects how they are handled: - - ``True`` (default; the detail envelope): the cousins are kept as members wearing the + - ``True`` (default; the detail envelope): the encodings are kept as members wearing the :attr:`CatVrsRelation.CO_ENCODES` relation, so the object is the full closure. The measured change's - coordinate partner and protein consequence stay ``coordinate_representation_of`` / ``translation_of``. - - ``False`` (the VA-Spec subject): the cousins are dropped (see :func:`_is_precise_projection_member`) - and only the measured change's precise coordinate partner and protein consequence remain. + projection and protein consequence stay ``coordinate_representation_of`` / ``translation_of``. + - ``False`` (the VA-Spec subject): the encodings are dropped (see :func:`_is_precise_projection_member`) + and only the measured change's precise projection and protein consequence remain. """ defining_link = next((link for link in links if link.is_authoritative), None) if defining_link is None: @@ -213,9 +249,9 @@ def build_categorical_variant( allele = link.allele - # Projection mode, narrow object (include_convergent=False): drop any synonymous cousins the + # Projection mode, narrow object (include_convergent=False): drop any convergent encodings the # reverse-translation fan left on the record, keeping only the measured change's precise coordinate - # partner and its protein consequence. + # projection and its protein consequence. if ( mode is CatVrsMode.PROJECTION and not include_convergent diff --git a/src/mavedb/lib/term_systems.py b/src/mavedb/lib/term_systems.py new file mode 100644 index 000000000..b9998433a --- /dev/null +++ b/src/mavedb/lib/term_systems.py @@ -0,0 +1,36 @@ +"""Shared building block for GA4GH ``Coding`` objects drawn from a named term system. + +A :data:`TermSystem` pairs a system URI/CURIE with the short prefix used to build a ``Coding.id`` for +codes drawn from it, kept together so a term's id and system can't drift apart the way two independent +lookup tables could. +""" + +from ga4gh.core.models import Coding + +TermSystem = tuple[str, str] + + +### MaveDB Term Systems + +# MaveDB's own Cat-VRS relation codes (member -> defining), in cat_vrs.py's own framing rather than the +# spec's. +MAVEDB_CAT_VRS_RELATION: TermSystem = ("https://mavedb.org/cat-vrs/relations", "mavedb") + + +### External Term Systems + +# Sequence Ontology. Used by several Cat-VRS Relation terms (translation_of, transcribed_to); see +# ga4gh/cat-vrs examples/json/proteinSequenceConsequence-ex2.json. +SEQUENCE_ONTOLOGY: TermSystem = ("http://www.sequenceontology.org", "so") + +# Cat-VRS's own internally controlled vocabulary for the allele-relation terms that have no external +# ontology equivalent (currently just liftover_to). See ga4gh/cat-vrs recipes-source.yaml. Other +# ga4gh-gks-term categories (e.g. experimental-var-func-impact-classification) are separate term +# systems under the same vocabulary owner, not this one. +GKS_ALLELE_RELATION: TermSystem = ("ga4gh-gks-term:allele-relation", "ga4gh-gks-term") + + +def coding(term_system: TermSystem, code: str) -> Coding: + """Build a ``Coding`` for `code`, drawn from `term_system`.""" + uri, prefix = term_system + return Coding(id=f"{prefix}:{code}", code=code, system=uri) diff --git a/tests/lib/test_cat_vrs.py b/tests/lib/test_cat_vrs.py index 47eb6ac64..1ee2b9ba0 100644 --- a/tests/lib/test_cat_vrs.py +++ b/tests/lib/test_cat_vrs.py @@ -13,9 +13,12 @@ pytest.importorskip("psycopg2") -from ga4gh.cat_vrs.models import CategoricalVariant, DefiningAlleleConstraint +from ga4gh.cat_vrs.models import CategoricalVariant, DefiningAlleleConstraint, Relation +from ga4gh.core.models import Relation as MappingRelation from mavedb.lib.cat_vrs import ( + _SPEC_EQUIVALENT, + _relation_concept, CatVrsMode, CatVrsRelation, build_categorical_variant, @@ -343,3 +346,76 @@ def test_wrapper_threads_as_of_to_the_historical_record(session, setup_lib_db_wi assert past is not None and past.mode == CatVrsMode.REVERSE_TRANSLATION assert current is not None and current.mode == CatVrsMode.PROJECTION + + +# --------------------------------------------------------------------------- +# Spec alignment +# +# MaveDB maintains its own relation vocabulary (Cat-VRS's `Relation` is an enum with members, so it +# cannot be subclassed, and inheriting would buy nothing — interop rides on `Coding.system`, not on +# Python enum identity). These tests are the drift guard that keeps the vocabulary honest: they fail +# when the spec gains a term MaveDB has not triaged, or when MaveDB coins a term without recording +# whether the spec already covers it. +# --------------------------------------------------------------------------- + + +def test_mapped_relations_emit_an_exact_match_to_the_spec_term(): + """A code with a spec equivalent must carry a machine-readable mapping to it. + + A matching code *string* is not enough: the two codings sit in different systems, so nothing but an + explicit ConceptMapping tells a consumer they are the same concept. + """ + concept = _relation_concept(CatVrsRelation.TRANSLATION_OF) + + assert concept.primaryCoding is not None + assert concept.primaryCoding.system == "https://mavedb.org/cat-vrs/relations" + assert concept.mappings is not None and len(concept.mappings) == 1 + + mapping = concept.mappings[0] + assert mapping.relation == MappingRelation.EXACT_MATCH + assert mapping.coding.code.root == Relation.TRANSLATION_OF.value + assert mapping.coding.system != concept.primaryCoding.system + + +@pytest.mark.parametrize( + "relation", + [CatVrsRelation.ENCODES, CatVrsRelation.CO_ENCODES, CatVrsRelation.COORDINATE_REPRESENTATION_OF], +) +def test_unmapped_relations_carry_no_mapping(relation): + """The absence of a mapping is the interoperable statement that no spec term covers this code. + + Emitting a `closeMatch` to something approximate would be worse than silence — a consumer would + resolve it and be wrong. See `_SPEC_EQUIVALENT` for why each of these three has no equivalent. + """ + concept = _relation_concept(relation) + + assert concept.primaryCoding is not None and concept.primaryCoding.code.root == relation.value + assert concept.mappings is None + + +def test_every_relation_is_either_mapped_or_a_declared_gap(): + """No MaveDB relation may exist without a triage decision recorded in `_SPEC_EQUIVALENT`. + + Coining a new code is fine; coining one *without deciding whether the spec already covers it* is the + drift this guards against. A new member fails here until it is either mapped or listed below. + """ + declared_gaps = { + CatVrsRelation.ENCODES, + CatVrsRelation.CO_ENCODES, + CatVrsRelation.COORDINATE_REPRESENTATION_OF, + } + + assert set(CatVrsRelation) == set(_SPEC_EQUIVALENT) | declared_gaps + + +def test_spec_relations_maveDB_deliberately_never_emits(): + """Fails when Cat-VRS publishes a relation MaveDB has not triaged. + + `liftover_to` is assembly-to-assembly, which MaveDB does not do. `transcribed_to` is genomic-> + transcript only and asserts transcription; MaveDB's c<->g relation is direction-neutral coordinate + equivalence that also holds for intronic/UTR-offset positions, so it is deliberately unmapped. + """ + emitted = {spec_term.value for spec_term in _SPEC_EQUIVALENT.values()} + never_emitted = {"liftover_to", "transcribed_to"} + + assert {relation.value for relation in Relation} == emitted | never_emitted diff --git a/tests/lib/test_term_systems.py b/tests/lib/test_term_systems.py new file mode 100644 index 000000000..962ce324e --- /dev/null +++ b/tests/lib/test_term_systems.py @@ -0,0 +1,14 @@ +"""Tests for the shared (system, prefix) -> Coding helper.""" + +import pytest + +from mavedb.lib.term_systems import coding + + +@pytest.mark.unit +def test_coding_builds_id_from_prefix_and_code(): + result = coding(("https://example.org/system", "ex"), "some_code") + + assert result.id == "ex:some_code" + assert result.code.root == "some_code" + assert result.system == "https://example.org/system" From 48b843779bf5bef5db7d47832bda57b827d0b343 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Mon, 17 Aug 2026 11:05:34 -0700 Subject: [PATCH 107/252] refactor(alleles): rename sibling/cousin terminology to projection/encoding "Sibling" and "synonymous cousin" described two unrelated relationships with confusingly similar names: a projection_group partner (the c<->g pair of one measured change) versus a convergent but distinct nt allele that happens to encode the same protein consequence. Rename both throughout code and identifiers to the terms the domain docs already use elsewhere: - projection_group partner: sibling_* -> projection_* (variables, columns aliases, function params) in score_set_variants.py, score_sets.py, and lib/allele_detail.py - distinct nt allele sharing a protein consequence: cousin -> convergent encoding, including is_convergent_cousin -> is_convergent_encoding in lib/cat_vrs.py and its call sites in variant_detail.py - update the worker-job error strings that referenced "nucleotide siblings" to match Behavior is unchanged; this is a pure rename. --- src/mavedb/lib/allele_detail.py | 126 +++++----- src/mavedb/lib/score_set_variants.py | 218 ++++++++---------- src/mavedb/lib/score_sets.py | 28 +-- src/mavedb/lib/variant_detail.py | 148 ++++++------ .../worker/jobs/external_services/clinvar.py | 2 +- .../worker/jobs/external_services/gnomad.py | 2 +- 6 files changed, 254 insertions(+), 270 deletions(-) diff --git a/src/mavedb/lib/allele_detail.py b/src/mavedb/lib/allele_detail.py index b81b0db91..109703ede 100644 --- a/src/mavedb/lib/allele_detail.py +++ b/src/mavedb/lib/allele_detail.py @@ -1,13 +1,13 @@ """The allele-detail view backing ``GET /alleles/{digest|CAID}``. -The allele-grain sibling of :mod:`lib.variant_detail`. Where the variant view anchors on a +The allele-grain counterpart of :mod:`lib.variant_detail`. Where the variant view anchors on a *measurement*, this anchors on an *allele* (a VRS digest, or a CAID naming the nt-canonical change) and serves that allele's identity, its full cross-layer **equivalence class**, and the digest-keyed annotations that ride alongside. It is measurement-agnostic, and that shapes what it does *not* carry: -- No score, counts, classification, or version standing — those are properties of a measured variant, +- No score, counts, classification, or version standing. Those are properties of a measured variant, not of an allele (an allele is deduplicated by digest and shared across every score set mapping to it). - No spec-pure Cat-VRS. The ``CategoricalVariant`` / ``DefiningAlleleConstraint`` is emitted only by the measurement view, rooted at the measured allele. Here the members are labelled on a flat map @@ -15,14 +15,12 @@ **The equivalence class is the full cross-record union** (:func:`lib.alleles.get_allele_translations`): every allele co-linked to any live record touching the focus. That is the discoverable "everything -related to this change" set. Each member is then labelled relative to the focus — which the graph -determines even without a measurement, because candidate-ness is *directional* (protein→nt is -many-to-one ⇒ candidate; nt→protein is deterministic ⇒ consequence; nt↔nt is a faithful coordinate -transform). "Less power than a measurement" only means we cannot say which nt was *measured* — but we -can still say faithful vs. candidate vs. convergent from the graph and the focus. - -``as_of`` reconstructs the molecular layer — class membership + annotations — at the past instant. The -focus allele's own identity is immutable and content-addressed. ``as_of`` defaults to currently-live rows. +related to this change" set. Each member is then labelled relative to the focus allele. Even though this +level is unable to determine which allele was actually measured, we can still say how a given allele relates +to the focus allele (e.g., faithful, candidate, convergent) based on the graph structure. + +``as_of`` reconstructs the molecular layer, class membership + annotations, at the past instant. The +focus allele's own identity is immutable and content-addressed. ``as_of`` defaults to this instant. """ from dataclasses import dataclass @@ -46,35 +44,44 @@ class AlleleDetail: """The assembled allele-detail envelope (transit; serialized by ``view_models.allele_detail``).""" - # Flat anchor identity — the queried allele's own molecular facts (for a CAID fetch, its - # representative allele; ``clingenAlleleId`` carries the CAID either way). + # Own identity (the focus allele itself). digest: str level: Optional[str] - hgvs: Optional[str] # reference-frame HGVS (exactly one of hgvs_g/c/p, depending on level) + """The sequence level of the allele (e.g., nucleotide or amino acid).""" + hgvs: Optional[str] + """The reference-frame HGVS string (exactly one of hgvs_g/c/p, depending on level).""" clingen_allele_id: Optional[str] - vrs: Optional[dict[str, Any]] # spec-pure GA4GH VRS variation from post_mapped + vrs: Optional[dict[str, Any]] - # The full cross-layer equivalence class, keyed by VRS digest and sharing keys with `annotations`. - # Each member is an `AlleleIdentity` labelled relative to the focus. The focus allele(s) carry - # `is_focus=True`. + # Cross-layer equivalence class (all alleles equivalent to the focus). alleles: dict[str, AlleleIdentity] + """The full cross-layer equivalence class, keyed by VRS digest and sharing keys with `annotations`. + Each member is an `AlleleIdentity` labelled relative to the focus. The focus allele(s) carry + `is_focus=True`.""" - # External annotations, keyed by VRS digest, for every member of the class. + # External annotations. annotations: dict[str, AlleleAnnotations] + """External annotations for each member of the cross-layer equivalence class, keyed by VRS digest.""" def _focus_projection_pairs(db: Session, focus_alleles: list[Allele], *, as_of: Optional[datetime]) -> dict[str, str]: - """A symmetric ``digest -> sibling digest`` map for each focus allele's c↔g coordinate partner. + """A symmetric ``digest -> digest`` map pairing each focus allele with its c↔g projection. + + A ``projection_group`` is one c↔g pair, local to a single mapping record. The projection is the ≤1 + *other* allele sharing the same ``projection_group`` as the focus allele. Reading any one live link + is safe: the focus holds one live link per record it appears in, and each record re-encodes the same + change onto the same content-addressed digests, so the pairing is record-independent (assuming those + records agree on genome assembly). - Read from any one live record of the focus: the partner is the ≤1 *other* allele sharing the focus - allele's ``projection_group`` (a projection group is a c↔g pair). This is the one place the - focus/partner-vs-convergent-cousin distinction needs the link-level ``projection_group`` — every - other member is classified by level alone. + This is the one place the projection-vs-convergent-encoding distinction needs the link-level + ``projection_group`` — every other member is classified by level alone. """ pairs: dict[str, str] = {} for focus in focus_alleles: if focus.vrs_digest is None: continue + + # limit(1) rather than one_or_none(), which would raise for any allele measured more than once. link = db.scalar( select(MappingRecordAllele) .where(MappingRecordAllele.allele_id == focus.id) @@ -84,7 +91,8 @@ def _focus_projection_pairs(db: Session, focus_alleles: list[Allele], *, as_of: ) if link is None: continue - sibling_digests = db.scalars( + + projection_digests = db.scalars( select(Allele.vrs_digest) .select_from(MappingRecordAllele) .join(Allele, Allele.id == MappingRecordAllele.allele_id) @@ -93,60 +101,63 @@ def _focus_projection_pairs(db: Session, focus_alleles: list[Allele], *, as_of: .where(MappingRecordAllele.allele_id != focus.id) .where(MappingRecordAllele.live_at(as_of)) ).all() - for sibling in sibling_digests: - if sibling is not None: - pairs[focus.vrs_digest] = sibling - pairs.setdefault(sibling, focus.vrs_digest) + for projection in projection_digests: + if projection is not None: + pairs[focus.vrs_digest] = projection + pairs.setdefault(projection, focus.vrs_digest) + return pairs def _member_label( - *, focus_is_protein: bool, member_level: Optional[str], is_coordinate_partner: bool + *, focus_is_protein: bool, member_level: Optional[str], is_projection: bool ) -> tuple[Optional[CatVrsRelation], Optional[AlleleDerivation]]: """The (relation, derivation) for a non-focus member, read *relative to the focus*. - **protein focus** → every nt member is a reverse-translation ``candidate`` that ``encodes`` it. - - **nt focus** → the protein consequence is ``translation_of`` (a deterministic ``projection``); the - focus's coordinate partner is ``coordinate_representation_of`` (also ``projection``); any other nt - is a synonymous cousin — ``co_encodes`` / ``convergent`` (a distinct change sharing the consequence). + - **nt focus** → The protein consequence is ``translation_of`` (a deterministic ``projection`` of the focus). + The focus's projection partner is ``coordinate_representation_of`` (also a deterministic ``projection``). + Any other nt is a convergent encoding: ``co_encodes`` and ``convergent`` (a distinct change which shares the + ultimate protein consequence of the focus). """ if focus_is_protein: if member_level in NUCLEOTIDE_LEVELS: return CatVrsRelation.ENCODES, AlleleDerivation.CANDIDATE + return None, None if member_level == SequenceLevel.protein.value: return CatVrsRelation.TRANSLATION_OF, AlleleDerivation.PROJECTION if member_level in NUCLEOTIDE_LEVELS: - if is_coordinate_partner: + if is_projection: return CatVrsRelation.COORDINATE_REPRESENTATION_OF, AlleleDerivation.PROJECTION + return CatVrsRelation.CO_ENCODES, AlleleDerivation.CONVERGENT + return None, None def get_allele_detail( - db: Session, anchor: Allele, *, focus_digests: set[str], as_of: Optional[datetime] = None + db: Session, focus: Allele, *, focus_digests: set[str], as_of: Optional[datetime] = None ) -> AlleleDetail: - """Assemble the allele-detail envelope anchored on ``anchor``. + """Assemble the allele-detail envelope anchored on ``focus``. - ``focus_digests`` is the set of VRS digests that are the *focus* of this view — a single digest for + ``focus_digests`` is the set of VRS digests that are the *focus* of this view. A single digest for a by-digest fetch, or the (genomic + coding) representations of a CAID for a by-CAID fetch. The - equivalence class is the record-agnostic union co-linked to any live record touching ``anchor`` - (:func:`get_allele_translations`, which includes ``anchor``); an orphan allele with no live links - falls back to itself. Every member is labelled relative to the focus (see :func:`_member_label`). - ``as_of`` reconstructs the molecular layer; see the module docstring. + equivalence class is every allele co-linked to a live record touching the ``focus`` + (:func:`get_allele_translations`). An orphan allele falls back to ``focus`` alone. Non-focus members + are labelled by :func:`_member_label`. """ - equivalence_class = get_allele_translations(db, anchor.id, as_of=as_of) or [anchor] - + equivalence_class = get_allele_translations(db, focus.id, as_of=as_of) or [focus] annotations = get_allele_annotations(db, equivalence_class, as_of=as_of) - focus_alleles = [a for a in equivalence_class if a.vrs_digest in focus_digests] or [anchor] + focus_alleles = [a for a in equivalence_class if a.vrs_digest in focus_digests] or [focus] focus_is_protein = any(a.level == SequenceLevel.protein.value for a in focus_alleles) pairs = _focus_projection_pairs(db, focus_alleles, as_of=as_of) - # The focus's coordinate partner(s) that are not themselves focus (for a CAID fetch the c↔g pair is - # jointly the focus, so this is empty and the partners are simply flagged is_focus instead). - coordinate_partner_digests = {pairs[d] for d in focus_digests if d in pairs} - focus_digests + # The focus's projection(s) that are not themselves focus (for a CAID fetch the projection pair is + # jointly the focus, so this is empty and they are simply flagged is_focus instead). + projection_digests = {pairs[d] for d in focus_digests if d in pairs} - focus_digests alleles: dict[str, AlleleIdentity] = {} for member in equivalence_class: @@ -154,16 +165,17 @@ def get_allele_detail( if digest is None: continue - if digest in focus_digests: - relation, derivation = None, None - else: + # Determine the relation and derivation for non-focus members. Focus + # members have no relation or derivation relative to themselves. + relation, derivation = None, None + if digest not in focus_digests: rel, der = _member_label( focus_is_protein=focus_is_protein, member_level=member.level, - is_coordinate_partner=digest in coordinate_partner_digests, + is_projection=digest in projection_digests, ) relation = rel.value if rel is not None else None - derivation = der.value if der is not None else None + derivation = der alleles[digest] = AlleleIdentity( level=member.level, @@ -176,12 +188,12 @@ def get_allele_detail( ) return AlleleDetail( - digest=anchor.vrs_digest or "", - level=anchor.level, - hgvs=anchor.hgvs_g or anchor.hgvs_c or anchor.hgvs_p, - clingen_allele_id=anchor.clingen_allele_id, - vrs=vrs_object_from_mapped_variant(anchor.post_mapped).model_dump(mode="json", exclude_none=True) - if anchor.post_mapped is not None + digest=focus.vrs_digest or "", + level=focus.level, + hgvs=focus.hgvs_g or focus.hgvs_c or focus.hgvs_p, + clingen_allele_id=focus.clingen_allele_id, + vrs=vrs_object_from_mapped_variant(focus.post_mapped).model_dump(mode="json", exclude_none=True) + if focus.post_mapped is not None else None, alleles=alleles, annotations=annotations, diff --git a/src/mavedb/lib/score_set_variants.py b/src/mavedb/lib/score_set_variants.py index c1f45c545..ab6388cb7 100644 --- a/src/mavedb/lib/score_set_variants.py +++ b/src/mavedb/lib/score_set_variants.py @@ -1,30 +1,22 @@ -"""The lean whole-set view backing the score-set page. - -The score-set table, heatmap, and score/effect histograms all compose from one pre-chewed per-variant -dataset. This module assembles that dataset for an entire score set in two O(N) bulk queries — a base -per-variant projection plus a standalone protein-HGVS projection stitched back in Python — and returns -one record per variant. (The protein projection is deliberately *not* a join: folded in, its opaque -cardinality drives the SQL planner into an O(N^2) per-variant rescan; see ``get_lean_score_set_variants``.) - -Each record carries the **submitted** HGVS the heatmap's frame toggle needs (``hgvs_nt``/``hgvs_pro``/ -``hgvs_splice``, the depositor's target frame) plus the **mapped** (reference frame) representation as a -``MappedTriple`` — one canonical HGVS per level (``genomic`` / ``cdna`` / ``protein``) and an -``assay_level`` pointer naming which slot is the measured one. ``mapped[assay_level]`` is the measured -representation. Only the *canonical* projection is served here — never the reverse-translation -fan-out (that lives in the detail endpoint), so a nucleotide assay fills all three slots while a protein -assay fills only ``protein`` (``cdna``/``genomic`` stay ``None`` — the c/g fan-out is ambiguous, so no -pick is fabricated). Each HGVS rides as an ``HgvsField``: the string (canonical, lossless) plus a parsed -``position``/``ref``/``alt`` block when it is a placeable simple substitution (the heatmap grid; ``None`` -for splice/indels/multivariants). - -The canonical nucleotide pair comes from the **authoritative** (measured) allele — its digest, ClinGen -id, and VEP consequence — plus one indexed join to its ``projection_group`` sibling at the other -nucleotide level (still **one row per variant**: a group has ≤2 members, so the authoritative has ≤1 -sibling). The protein slot is the separate ``DISTINCT ON`` subquery. Where ``projection_group`` is -unpopulated (pre-reverse-translation data) the sibling join returns null → the other nucleotide slot -stays ``None`` and improves as data is re-processed. This view does not assemble Cat-VRS. ``as_of`` -reconstructs the annotation layer at a past instant over the variant's immutable scores/submitted HGVS; -it defaults to the currently-live rows. +"""Lean whole-set view backing the score-set page (table, heatmap, score/effect histograms). + +Assembles one record per variant in two O(N) bulk queries: a base per-variant projection, plus a +separate protein-HGVS query stitched back in Python (see :func:`get_protein_hgvs_by_record` for why +it's split out). + +Each record carries the **submitted** HGVS (``hgvs_nt``/``hgvs_pro``/``hgvs_splice``, the depositor's +target frame) and the **mapped** (reference-frame) HGVS as a ``MappedTriple`` — one canonical HGVS per +level (``genomic``/``cdna``/``protein``), with ``assay_level`` naming the measured slot. A nucleotide +assay fills all three slots; a protein assay fills only ``protein`` (the c/g fan-out is ambiguous, so +no pick is fabricated). This is the canonical projection only, never the reverse-translation fan-out. + +The nucleotide pair comes from the authoritative (measured) allele plus its ``projection_group`` +partner at the other nucleotide level (a group has ≤2 members, so this stays one row per variant). +Where ``projection_group`` is unpopulated (pre-reverse-translation data), that slot is ``None`` until +reprocessed. Does not assemble Cat-VRS. + +``as_of`` reconstructs the annotation layer at a past instant; submitted HGVS and scores are immutable +and unaffected. Defaults to currently-live rows. """ from dataclasses import dataclass @@ -47,11 +39,10 @@ @dataclass(frozen=True) class HgvsField: - """An HGVS expression plus its parsed simple-substitution block when representable. + """An HGVS string plus its parsed position/ref/alt when it's a placeable simple substitution. - ``hgvs`` is always present (canonical, lossless). ``position``/``ref``/``alt`` are populated only - when the expression is a single-locus substitution the heatmap can place; they stay ``None`` for - splice/intronic positions, indels, frameshifts, and multivariants, which carry the string alone. + ``position``/``ref``/``alt`` are ``None`` for splice/intronic, indels, frameshifts, and + multivariants — those carry ``hgvs`` alone. """ hgvs: str @@ -62,16 +53,14 @@ class HgvsField: @dataclass(frozen=True) class MappedTriple: - """The mapped (reference-frame) HGVS at each level — the canonical projection of the measured change. + """The canonical mapped (reference-frame) HGVS at each level, or ``None`` where absent. - One slot per level, each an :class:`HgvsField` or ``None``. A **nucleotide** assay populates all - three (``mapped[assay_level]`` is the measured slot, the other nucleotide slot is the authoritative - allele's ``projection_group`` sibling, ``protein`` is the apex). A **protein** assay populates only - ``protein`` — the c/g fan-out is ambiguous, so ``cdna``/``genomic`` stay ``None`` rather than - fabricate a canonical pick. A slot is also ``None`` when its source is simply absent (an unmapped - variant; a projection that failed; pre-reverse-translation data with no sibling recorded). + A nucleotide assay populates all three slots (measured + the authoritative allele's + ``projection_group`` partner + protein apex); a protein assay populates only ``protein`` — the + c/g fan-out is ambiguous, so no pick is fabricated for ``cdna``/``genomic``. - ``cdna`` is the level-invariant search key — populated even when ``assay_level`` is ``genomic``. + ``cdna`` is the level-invariant field: populated whether the assay was measured at ``cdna`` or + ``genomic``, so it's the field to key cross-assay searches on. """ genomic: Optional[HgvsField] = None @@ -81,13 +70,13 @@ class MappedTriple: @dataclass(frozen=True) class LeanVariantRecord: - """One pre-chewed per-variant record: the selection key (``variant_urn``), the baseline ``score``, - a representative (lossy) VEP ``consequence``, the bridge identifiers into the annotation dimensions - (``clingen_allele_id``, ``assay_level_digest``), the submitted HGVS at each level, and the mapped - (reference-frame) representation as an ``assay_level`` pointer (an ``SequenceLevel`` value naming - the measured/canonical slot) plus a ``mapped`` :class:`MappedTriple`. ``mapped[assay_level]`` is the - measured representation; ``mapped.cdna`` is the level-invariant search key. Any field is ``None`` - when its source is absent (unmapped variant → null ``assay_level`` + empty triple).""" + """One pre-chewed per-variant record, keyed by ``variant_urn``. + + ``consequence`` is a representative (most severe) VEP call. ``clingen_allele_id``/``assay_level_digest`` + bridge into the annotation dimensions. ``mapped`` is the canonical :class:`MappedTriple`; + ``assay_level`` names which of its slots is the measured one. Any field is ``None`` when its + source is absent (an unmapped variant → null ``assay_level`` and an empty triple). + """ variant_urn: str score: Optional[float] @@ -102,8 +91,7 @@ class LeanVariantRecord: def _level(value: Optional[str]) -> Optional[SequenceLevel]: - """Coerce a stored assay-level string (the ``mapping_records.assay_level`` column) into the closed - :class:`SequenceLevel` set. ``None`` for an absent value.""" + """Coerce a stored assay-level string into :class:`SequenceLevel`, or ``None`` if absent.""" return SequenceLevel(value) if value else None @@ -112,9 +100,11 @@ def _hgvs_field_for_str(hgvs: Optional[str]) -> Optional[HgvsField]: simple substitution. ``None`` for an absent string.""" if not hgvs: return None + block = parse_simple_substitution(hgvs) if block is None: return HgvsField(hgvs=hgvs) + return HgvsField(hgvs=hgvs, position=block.position, ref=block.ref, alt=block.alt) @@ -122,21 +112,19 @@ def mapped_hgvs_by_level( *, assay_level: Optional[SequenceLevel], assay_level_hgvs: Optional[str], - sibling_level: Optional[str], - sibling_hgvs: Optional[str], + projection_level: Optional[str], + projection_hgvs: Optional[str], protein_hgvs: Optional[str], ) -> dict[str, Optional[str]]: - """The canonical mapped HGVS **string** at each populated level slot, keyed by ``SequenceLevel`` value. + """The canonical mapped HGVS string at each populated level, keyed by ``SequenceLevel`` value. - The canonical projection of a measured allele onto various levels relative to its reference frame: - - Measured slot (``[assay_level]``) ← ``assay_level_hgvs`` (the record's mapped assay HGVS). - - Other nucleotide slot ← the authoritative link's ``projection_group`` sibling (``sibling_level`` / - ``sibling_hgvs``), populated only on a nucleotide assay and only where the pairing is recorded. + - Measured slot ← ``assay_level_hgvs``. + - Other nucleotide slot ← the authoritative link's ``projection_group`` partner (nucleotide assays + only, and only where the pairing is recorded). - ``protein`` slot ← the separate protein-apex subquery. - A **protein** assay fills only ``protein`` (the measured slot is protein itself); its nucleotide - fan-out is ambiguous, so ``cdna``/``genomic`` are deliberately absent. An unmapped variant - (``assay_level is None``) yields an empty mapping. + A protein assay fills only ``protein``. An unmapped variant (``assay_level is None``) yields an + empty mapping. """ slots: dict[str, Optional[str]] = {} if assay_level == SequenceLevel.protein: @@ -144,9 +132,9 @@ def mapped_hgvs_by_level( elif assay_level in (SequenceLevel.cdna, SequenceLevel.genomic): slots[assay_level.value] = assay_level_hgvs slots[SequenceLevel.protein.value] = protein_hgvs - # The other nucleotide level, from the projection_group sibling (null where unpopulated). - if sibling_level is not None: - slots[sibling_level] = sibling_hgvs + # The other nucleotide level, from the projection_group partner (null where unpopulated). + if projection_level is not None: + slots[projection_level] = projection_hgvs return slots @@ -155,8 +143,8 @@ def _mapped_triple( *, assay_level: Optional[SequenceLevel], assay_level_hgvs: Optional[str], - sibling_level: Optional[str], - sibling_hgvs: Optional[str], + projection_level: Optional[str], + projection_hgvs: Optional[str], protein_hgvs: Optional[str], ) -> MappedTriple: """Assemble the canonical :class:`MappedTriple` for one variant, wrapping each slot from @@ -164,8 +152,8 @@ def _mapped_triple( slots = mapped_hgvs_by_level( assay_level=assay_level, assay_level_hgvs=assay_level_hgvs, - sibling_level=sibling_level, - sibling_hgvs=sibling_hgvs, + projection_level=projection_level, + projection_hgvs=projection_hgvs, protein_hgvs=protein_hgvs, ) return MappedTriple( @@ -176,14 +164,14 @@ def _mapped_triple( def get_protein_hgvs_by_record(db: Session, score_set_id: int, *, as_of: Optional[datetime] = None) -> dict[int, str]: - """The canonical protein-level mapped HGVS for each of a score set's live mapping records, keyed by - ``mapping_record.id``. Records with no protein projection (UTR/intronic) are simply absent from the - map. Shared by the lean whole-set view and the CSV export so both resolve the protein slot the same way. - - This runs as its **own** query, never a join into a per-variant projection. Joined in, it forces an - O(N^2) plan: the ``DISTINCT ON`` cardinality is opaque to the planner, so it lands on the inner side - of a nested loop and is rescanned in full once per variant — historically ~97% of the lean endpoint's - total time. Pulling it out and stitching by ``mapping_record_id`` in Python keeps both queries O(N). + """The canonical protein-level mapped HGVS for a score set's live mapping records, keyed by + ``mapping_record.id``. Records with no protein projection (UTR/intronic) are absent from the map. + Shared by the lean whole-set view and the CSV export. + + Runs as its own query rather than a join: the ``DISTINCT ON`` cardinality is opaque to the planner, + so a join lands it on the inner side of a nested loop and rescans it once per variant — O(N^2), and + historically ~97% of the lean endpoint's runtime. Stitching by ``mapping_record_id`` in Python keeps + both queries O(N). """ prot_link = aliased(MappingRecordAllele) prot_allele = aliased(Allele) @@ -203,13 +191,13 @@ def get_protein_hgvs_by_record(db: Session, score_set_id: int, *, as_of: Optiona .join(prot_variant, prot_variant.id == prot_record.variant_id) # Scope to this score set so the cost scales with the response. .where(prot_variant.score_set_id == score_set_id) - # Live links only. A live link implies a live parent record (ValidTime invariant), so this also - # covers record-liveness — no prot_record.live_at needed; the MR join is just the scoping bridge. + # Live links only; a live link implies a live parent record (ValidTime invariant), so no + # separate prot_record.live_at check is needed. .where(prot_link.live_at(as_of)) # Only the protein-level allele. .where(prot_allele.level == SequenceLevel.protein.value) - # DISTINCT ON requires ORDER BY to lead with its column; allele.id then picks which row survives - # per record — inert today (only one protein allele exists). + # DISTINCT ON needs ORDER BY to lead with its column; allele.id breaks ties (inert today — + # only one protein allele exists per record). .order_by(prot_link.mapping_record_id, prot_allele.id) ) # Keyed by record id (a globally-unique PK -> the right protein row, no cross-set risk). @@ -219,20 +207,17 @@ def get_protein_hgvs_by_record(db: Session, score_set_id: int, *, as_of: Optiona def get_lean_score_set_variants( db: Session, score_set: ScoreSet, *, as_of: Optional[datetime] = None ) -> list[LeanVariantRecord]: - """Assemble the lean whole-set view for ``score_set`` — one record per variant, ordered by variant - number. Unmapped variants are retained (left joins) with null mapped fields so the table and score - histogram still see them. ``as_of`` reconstructs the annotation layer at a past instant (submitted - HGVS and scores are immutable and unaffected); it defaults to the currently-live rows. + """Assemble the lean whole-set view for ``score_set``, one record per variant, ordered by variant + number. Unmapped variants are retained (left joins, null mapped fields) so the table and score + histogram still see them. ``as_of`` defaults to currently-live rows. """ - # The other-nucleotide sibling of the authoritative link: same mapping record + same - # projection_group, but NOT authoritative (so it is the *other* member of the ≤2-member group). Its - # allele carries the canonical HGVS at the level the assay was not measured at. See the join below. - sibling_link = aliased(MappingRecordAllele) - sibling_allele = aliased(Allele) - - # The base per-variant projection: 1:1 annotation joins, so this whole query is a stream of O(N) - # index-scan lookups. MappingRecord.id rides along as the key the protein projection (a *separate* - # query, below) is stitched back on. + # The authoritative link's *other* projection_group member — same record, same group, not + # authoritative — carries the canonical HGVS at the unmeasured nucleotide level. See join below. + projection_link = aliased(MappingRecordAllele) + projection_allele = aliased(Allele) + + # 1:1 annotation joins, so this is a stream of O(N) index-scan lookups. MappingRecord.id rides + # along as the key the protein projection (separate query, below) is stitched back on. base_statement = ( select( Variant.urn, @@ -246,18 +231,16 @@ def get_lean_score_set_variants( Allele.vrs_digest, Allele.clingen_allele_id, VepAlleleConsequence.functional_consequence, - sibling_allele.level.label("sibling_level"), - # Exactly one of the sibling's hgvs_g/hgvs_c is populated (it is a nucleotide allele), so - # coalesce yields its canonical string regardless of which nucleotide level it sits at. - func.coalesce(sibling_allele.hgvs_g, sibling_allele.hgvs_c).label("sibling_hgvs"), + projection_allele.level.label("projection_level"), + # Exactly one of hgvs_g/hgvs_c is populated (it's a nucleotide allele); coalesce picks it. + func.coalesce(projection_allele.hgvs_g, projection_allele.hgvs_c).label("projection_hgvs"), ) - # Variant -> its live mapping record. LEFT join so unmapped variants stay (with null mapped fields) - # for the table + score histogram. live_at rides in the ON clause, never a WHERE: a WHERE would - # reject the null-record row an outer join makes for an unmapped variant, silently collapsing the - # LEFT join to an inner one. Same pattern on every join below. + # LEFT join so unmapped variants stay (null mapped fields) for the table + score histogram. + # live_at rides in the ON clause, not WHERE. a WHERE would reject the outer join's null row, + # silently turning this into an INNER join. Same pattern on every join below. .outerjoin(MappingRecord, and_(MappingRecord.variant_id == Variant.id, MappingRecord.live_at(as_of))) - # The one authoritative (measured) allele. is_authoritative keeps this 1:1 with the variant - # (one authoritative link per live record — the invariant the mapping job upholds). + # The one authoritative (measured) allele; is_authoritative keeps this 1:1 (one authoritative + # link per live record, an invariant the mapping job upholds). .outerjoin( MappingRecordAllele, and_( @@ -266,39 +249,34 @@ def get_lean_score_set_variants( MappingRecordAllele.live_at(as_of), ), ) - # The authoritative allele row itself (digest, ClinGen id). No live_at: alleles are immutable and - # deduplicated, not ValidTime — the live link already establishes what applies. + # The allele row itself (digest, ClinGen id). No live_at: alleles are immutable/deduplicated, + # not ValidTime — the live link already establishes what applies. .outerjoin(Allele, Allele.id == MappingRecordAllele.allele_id) # Its VEP consequence. .outerjoin( VepAlleleConsequence, and_(VepAlleleConsequence.allele_id == Allele.id, VepAlleleConsequence.live_at(as_of)) ) - # The authoritative link's projection_group sibling — the canonical projection at the *other* - # nucleotide level. UNLIKE the protein subquery, this stays a direct 1:1 outer join: a group has - # ≤2 members and the authoritative link is one of them, so at most one non-authoritative sibling - # shares its (mapping_record_id, projection_group). The mapping_record_id predicate rides the - # existing ix_mapping_record_alleles_mapping_record_id index; within a record only a handful of - # links are scanned, so no O(N^2) rescan (contrast the opaque-cardinality protein subquery below). - # A NULL authoritative group (protein assay, or pre-reverse-translation data) matches nothing — - # SQL equality on NULL is never true — so the sibling stays null and the slot degrades gracefully. + # The authoritative link's projection_group partner (other nucleotide level). Unlike the protein + # subquery, this is safe as a direct join: a group has ≤2 members, so at most one match, and the + # mapping_record_id predicate uses an existing index — no O(N^2) rescan. A NULL group (protein + # assay, or pre-reverse-translation data) matches nothing, so the slot degrades to None. .outerjoin( - sibling_link, + projection_link, and_( - sibling_link.mapping_record_id == MappingRecord.id, - sibling_link.projection_group == MappingRecordAllele.projection_group, - sibling_link.is_authoritative.is_(False), - sibling_link.live_at(as_of), + projection_link.mapping_record_id == MappingRecord.id, + projection_link.projection_group == MappingRecordAllele.projection_group, + projection_link.is_authoritative.is_(False), + projection_link.live_at(as_of), ), ) - .outerjoin(sibling_allele, sibling_allele.id == sibling_link.allele_id) + .outerjoin(projection_allele, projection_allele.id == projection_link.allele_id) # The anchor: just this score set's variants. .where(Variant.score_set_id == score_set.id) # Natural table order: variant number (the integer after '#'), id breaks ties stably. .order_by(cast(func.split_part(Variant.urn, "#", 2), Integer), Variant.id) ) - # The protein projection runs as its own query, NOT a join into the base statement (see - # get_protein_hgvs_by_record for why), stitched back by mapping_record_id below. + # Runs as its own query, not a join (see get_protein_hgvs_by_record), stitched back below. protein_hgvs_by_record = get_protein_hgvs_by_record(db, score_set.id, as_of=as_of) return [ @@ -315,8 +293,8 @@ def get_lean_score_set_variants( mapped=_mapped_triple( assay_level=_level(row.assay_level), assay_level_hgvs=row.hgvs_assay_level, - sibling_level=row.sibling_level, - sibling_hgvs=row.sibling_hgvs, + projection_level=row.projection_level, + projection_hgvs=row.projection_hgvs, protein_hgvs=protein_hgvs_by_record.get(row.mapping_record_id), ), ) diff --git a/src/mavedb/lib/score_sets.py b/src/mavedb/lib/score_sets.py index e684efd7f..4195c4900 100644 --- a/src/mavedb/lib/score_sets.py +++ b/src/mavedb/lib/score_sets.py @@ -714,12 +714,12 @@ def _csv_header_columns(columns: dict[str, list[str]], *, namespaced: Optional[b def _csv_substrate_query(score_set_id: int, *, as_of: Optional[datetime]) -> Select[Any]: """The base per-variant query for the annotation namespaces: each variant LEFT-joined to its live mapping record, its authoritative (measured) allele (digest / ClinGen id / VEP consequence), and the - projection-group nucleotide sibling — mirroring the lean whole-set view's join + projection-group nucleotide projection — mirroring the lean whole-set view's join (:func:`mavedb.lib.score_set_variants.get_lean_score_set_variants`). LEFT joins with ``live_at`` in the ON clause keep unmapped variants (with null mapped fields); a WHERE would collapse them out. The protein slot and the gnomAD/ClinVar dimensions are resolved by separate fetches and stitched on by id.""" - sibling_link = aliased(MappingRecordAllele) - sibling_allele = aliased(Allele) + projection_link = aliased(MappingRecordAllele) + projection_allele = aliased(Allele) return ( select( Variant, @@ -730,10 +730,10 @@ def _csv_substrate_query(score_set_id: int, *, as_of: Optional[datetime]) -> Sel Allele.vrs_digest, Allele.clingen_allele_id, VepAlleleConsequence.functional_consequence, - sibling_allele.level.label("sibling_level"), - # Exactly one of the sibling's hgvs_g/hgvs_c is populated (it is a nucleotide allele), so + projection_allele.level.label("projection_level"), + # Exactly one of the projection's hgvs_g/hgvs_c is populated (it is a nucleotide allele), so # coalesce yields its canonical string regardless of which nucleotide level it sits at. - func.coalesce(sibling_allele.hgvs_g, sibling_allele.hgvs_c).label("sibling_hgvs"), + func.coalesce(projection_allele.hgvs_g, projection_allele.hgvs_c).label("projection_hgvs"), ) .outerjoin(MappingRecord, and_(MappingRecord.variant_id == Variant.id, MappingRecord.live_at(as_of))) .outerjoin( @@ -750,15 +750,15 @@ def _csv_substrate_query(score_set_id: int, *, as_of: Optional[datetime]) -> Sel and_(VepAlleleConsequence.allele_id == Allele.id, VepAlleleConsequence.live_at(as_of)), ) .outerjoin( - sibling_link, + projection_link, and_( - sibling_link.mapping_record_id == MappingRecord.id, - sibling_link.projection_group == MappingRecordAllele.projection_group, - sibling_link.is_authoritative.is_(False), - sibling_link.live_at(as_of), + projection_link.mapping_record_id == MappingRecord.id, + projection_link.projection_group == MappingRecordAllele.projection_group, + projection_link.is_authoritative.is_(False), + projection_link.live_at(as_of), ), ) - .outerjoin(sibling_allele, sibling_allele.id == sibling_link.allele_id) + .outerjoin(projection_allele, projection_allele.id == projection_link.allele_id) .where(Variant.score_set_id == score_set_id) .order_by(*_VARIANT_NUMBER_ORDER) ) @@ -823,8 +823,8 @@ def _csv_mapped_row(row: Any, protein_hgvs_by_record: dict[int, str]) -> _CsvMap slots = mapped_hgvs_by_level( assay_level=SequenceLevel(row.assay_level) if row.assay_level else None, assay_level_hgvs=row.hgvs_assay_level, - sibling_level=row.sibling_level, - sibling_hgvs=row.sibling_hgvs, + projection_level=row.projection_level, + projection_hgvs=row.projection_hgvs, protein_hgvs=protein_hgvs_by_record.get(row.mapping_record_id), ) return _CsvMappedRow( diff --git a/src/mavedb/lib/variant_detail.py b/src/mavedb/lib/variant_detail.py index cf9a95cc0..1ad8fdf14 100644 --- a/src/mavedb/lib/variant_detail.py +++ b/src/mavedb/lib/variant_detail.py @@ -1,19 +1,15 @@ -"""The assayed variant-detail view backing ``GET /variants/{urn}``. - -Assembles the two-tier envelope for a single variant: flat, UI-ergonomic assay fields (the -assay-level HGVS pair, digest, ClinGen id) plus the spec-pure GA4GH ``CategoricalVariant`` (built -on the fly by :mod:`lib.cat_vrs`) and, riding alongside it keyed by VRS digest, the MaveDB layer — -the per-allele identity sidecar (level / HGVS / ClinGen id / member→defining relation) and the -digest-keyed external-annotation map (:mod:`lib.allele_annotations`). Also -carries the per-calibration functional ``classifications`` the variant falls into, and its version -standing (``is_current`` / ``superseded_by_score_set``) so a superseded variant self-describes rather -than reading as current. - -Temporal scope of ``as_of``: only the **molecular** layer is versioned — Cat-VRS membership and the -VEP/gnomAD/ClinVar annotations reconstruct at the past instant. Scores are immutable (``Variant`` is -not ``ValidTime``) and calibrations carry no ``ValidTime`` at all (they are frozen/append-only), so -scores and classifications are as-of-invariant and returned as they stand now. ``as_of`` defaults to -the currently-live rows. +"""Assembles the variant-detail envelope backing ``GET /variants/{urn}``. + +The envelope has two tiers: flat, UI-ergonomic assay-level fields (HGVS pair, digest, ClinGen id), +and a spec-pure GA4GH ``CategoricalVariant`` (built by :mod:`lib.cat_vrs`) with a MaveDB layer +riding alongside it, keyed by VRS digest — per-allele identity (level, HGVS, ClinGen id, +member→defining relation) and external annotations (:mod:`lib.allele_annotations`). Also includes +the variant's functional ``classifications`` per calibration, and its version standing +(``is_current`` / ``superseded_by_score_set``), so a superseded variant self-describes. + +``as_of`` only reconstructs the **molecular** layer (Cat-VRS membership and VEP/gnomAD/ClinVar +annotations) at the past instant. Scores are immutable and calibrations carry no ``ValidTime``, so +both are always returned as they stand now, regardless of ``as_of``. """ from dataclasses import dataclass @@ -26,7 +22,7 @@ from mavedb.lib.allele_annotations import AlleleAnnotations, get_allele_annotations from mavedb.lib.allele_identity import AlleleDerivation, AlleleIdentity from mavedb.lib.alleles import get_live_record_allele_links -from mavedb.lib.cat_vrs import categorical_variant_for_variant, is_convergent_cousin +from mavedb.lib.cat_vrs import categorical_variant_for_variant, is_convergent_encoding from mavedb.lib.score_calibrations import calibration_preference_key from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.models.mapping_record import MappingRecord @@ -41,12 +37,11 @@ @dataclass(frozen=True) class VariantClassificationRecord: - """One functional classification the variant falls into, tagged with its calibration context. + """One functional classification the variant belongs to, tagged with its calibration. - ``calibration_id`` / ``primary`` locate the classification among the score set's (possibly - several) calibrations — ``primary`` is the UI default. ``classification`` is the ORM - :class:`ScoreCalibrationFunctionalClassification` (functional call, ACMG, oddspath), serialized - at the view-model boundary. + ``calibration_id`` / ``primary`` locate it among the score set's calibrations (``primary`` is + the UI default). ``classification`` is the ORM + :class:`ScoreCalibrationFunctionalClassification`, serialized at the view-model boundary. """ calibration_id: int @@ -70,14 +65,14 @@ class VariantDetail: assay_level_digest: Optional[str] clingen_allele_id: Optional[str] - # The raw GA4GH VRS pair, surfaced flat so a bulk/VRS consumer reads them directly rather than - # digging the measured allele out of the Cat-VRS below. + # Raw GA4GH VRS pair, surfaced flat so a bulk/VRS consumer doesn't need to dig the measured + # allele out of the Cat-VRS below. pre_mapped: Optional[dict[str, Any]] post_mapped: Optional[dict[str, Any]] - # Spec-pure GA4GH Cat-VRS (no MaveDB fields inside), plus the MaveDB layer riding alongside: the - # per-allele identity sidecar, keyed by VRS digest — one entry per linked allele (the record-scoped, - # all-levels set, sharing keys with `annotations`), carrying level + HGVS + ClinGen id + relation. + # Spec-pure GA4GH Cat-VRS (no MaveDB fields), plus the MaveDB layer alongside it: a per-allele + # identity sidecar keyed by VRS digest, one entry per linked allele (shares keys with + # `annotations`), carrying level, HGVS, ClinGen id, and relation. molecular_representation: Optional[dict[str, Any]] mode: Optional[str] # CatVrsMode value — projection | reverse_translation alleles: dict[str, AlleleIdentity] @@ -87,21 +82,23 @@ class VariantDetail: # Version standing — self-descriptive for a superseded variant. is_current: bool - # URN of the score-set version that supersedes this variant's, if any (and readable). This is a - # *score set* URN, not a variant URN — supersession is versioned at the score-set level, and a newer - # version may add/drop/renumber variants, so there is no stable superseding-variant to point at; - # consumers resolve the current measurement by looking this variant up within that score set. + # Score-set URN (not variant URN) of the version that supersedes this one, if any and readable. + # Supersession is versioned at the score-set level, and a newer version may add/drop/renumber + # variants, so consumers must look this variant up within that score set rather than follow a + # superseding-variant pointer. superseded_by_score_set: Optional[str] def _classifications_for_variant( db: Session, variant: Variant, *, visible_calibration_ids: Optional[set[int]] ) -> list[VariantClassificationRecord]: - """The functional classifications the variant belongs to (membership is materialized on the - calibration→classification→variant m2m). One row per calibration that classifies the variant, ordered - by the shared calibration preference cascade (``calibration_preference_key`` + id) so the first entry - is the same default the UI and the allele-measurements card surface. ``visible_calibration_ids`` - restricts to calibrations the caller resolved as readable (``None`` = no restriction, for lib-level use).""" + """Functional classifications the variant belongs to, one row per calibration that classifies it. + + Membership is read from the calibration→classification→variant association table, ordered by + ``calibration_preference_key`` (+ id) so the first entry matches the UI's default. + ``visible_calibration_ids`` restricts to readable calibrations (``None`` = no restriction, for + lib-level callers). + """ statement = ( select(ScoreCalibrationFunctionalClassification, ScoreCalibration) .join( @@ -125,37 +122,35 @@ def _classifications_for_variant( ] -def _derivation_for(*, assay_level: Optional[SequenceLevel], is_cousin: bool) -> AlleleDerivation: - """The provenance of a *non-focus* linked allele's representation, from the assay level. +def _derivation_for(*, assay_level: Optional[SequenceLevel], is_convergent: bool) -> AlleleDerivation: + """Provenance of a *non-focus* linked allele, derived from the assay level. - The measured allele is the focus (``is_focus=True``, no derivation). Every *other* allele's - provenance turns on the protein → codon boundary, which surfaces three ways: + The measured allele is the focus (no derivation). For every other allele: - - **protein assay** (``assay_level`` protein) → ``candidate`` for the whole derived (nucleotide) - fan-out: reverse translation is genuinely ambiguous (which codon was measured is unknown). - - **nucleotide assay** (``assay_level`` genomic/cdna), synonymous *cousin* (``is_cousin``) → an nt - member in a different projection group: a distinct, precisely-known variant that merely - *converges* on the measured protein consequence. It is ``convergent`` — not ambiguous (so not a - ``candidate``) and not a projection *of* the measured change. - - **nucleotide assay**, otherwise → ``projection`` for the precise class derived from the measured - change itself (its coordinate partner and its protein consequence, both deterministic). + - **Protein assay** → ``CANDIDATE``: reverse translation is ambiguous, so the derived nucleotide + fan-out is only a candidate (which codon was actually measured is unknown). + - **Nucleotide assay, convergent encoding** (``is_convergent``) → ``CONVERGENT``: an nt member in + a different projection group. A distinct, precisely-known variant that happens to converge on + the same protein consequence, not a projection of the measured change. + - **Nucleotide assay, otherwise** → ``PROJECTION``: the deterministic class derived from the + measured change itself. - ``is_cousin`` is :func:`cat_vrs.is_convergent_cousin` for this allele, keeping the ``derivation`` - (provenance) axis in lockstep with the Cat-VRS ``relation`` axis — ``co_encodes`` ↔ ``convergent``, - ``coordinate_representation_of`` / ``translation_of`` ↔ ``projection``, protein-assay ``encodes`` ↔ - ``candidate``. + ``is_convergent`` comes from :func:`cat_vrs.is_convergent_encoding`, keeping this ``derivation`` + axis aligned with the Cat-VRS ``relation`` axis (``co_encodes`` ↔ ``convergent``, + ``coordinate_representation_of`` / ``translation_of`` ↔ ``projection``, ``encodes`` ↔ + ``candidate``). """ if assay_level == SequenceLevel.protein.value: return AlleleDerivation.CANDIDATE - if is_cousin: + if is_convergent: return AlleleDerivation.CONVERGENT return AlleleDerivation.PROJECTION def _submitted_assay_level_hgvs(variant: Variant, assay_level: Optional[SequenceLevel]) -> Optional[str]: - """The depositor-submitted HGVS in the variant's assay frame: protein for a protein assay, - otherwise the nucleotide expression (genomic or coding share ``hgvs_nt`` and ``hgvs_splice`` is - never an index column).""" + """Depositor-submitted HGVS in the variant's assay frame: ``hgvs_pro`` for a protein assay, + otherwise ``hgvs_nt`` (genomic and coding assays share this column; ``hgvs_splice`` is never an + index column).""" if assay_level == SequenceLevel.protein.value: return variant.hgvs_pro return variant.hgvs_nt @@ -171,11 +166,11 @@ def get_variant_detail( ) -> VariantDetail: """Assemble the variant-detail envelope for ``variant``. - ``superseding_score_set`` is the newer version the caller has already resolved for visibility - (blanked when the user cannot read it, mirroring ``fetch_score_set_by_urn``); its presence drives - ``is_current``/``superseded_by_score_set``. ``visible_calibration_ids`` restricts classifications to - readable calibrations. ``as_of`` reconstructs the molecular layer (Cat-VRS membership + - annotations); scores and classifications are as-of-invariant. See the module docstring. + ``superseding_score_set`` is the newer version the caller already resolved for visibility + (``None`` if unreadable, mirroring ``fetch_score_set_by_urn``); its presence drives + ``is_current`` / ``superseded_by_score_set``. ``visible_calibration_ids`` restricts + classifications to readable calibrations. ``as_of`` reconstructs the molecular layer only — + see the module docstring. """ data = variant.data if isinstance(variant.data, dict) else {} scores = data.get("score_data") @@ -197,9 +192,8 @@ def get_variant_detail( annotations = get_allele_annotations(db, [link.allele for link in links], as_of=as_of) - # Spec-pure Cat-VRS built on the fly, plus the MaveDB layer (mode + per-member relations). The - # relations key the sidecar below; the defining allele is deliberately absent from them, so it - # gets relation=None. + # Spec-pure Cat-VRS built on the fly, plus mode + per-member relations. The defining allele is + # deliberately absent from `relations`, so it gets relation=None below. transit = categorical_variant_for_variant(db, variant.id, name=variant.urn or "", as_of=as_of) if transit is not None: molecular_representation = transit.categorical_variant.model_dump(mode="json", exclude_none=True) @@ -210,28 +204,27 @@ def get_variant_detail( mode = None relations = {} - # The per-allele identity sidecar: one entry per linked allele, keyed by VRS digest (the same - # record-scoped, all-levels link set that seeds `annotations`, so the two maps share keys). Carries - # the molecular-identity facts the UI labels annotations by — level, reference-frame HGVS (exactly one - # of hgvs_g/c/p is populated per allele), ClinGen id — and the member→defining relation. - # Within-record projection grouping: projection_group -> the digests sharing it. A link's projection - # sibling is the ≤1 *other* digest in its group (the c↔g pair); the protein apex and any pre-RT link - # carry a NULL group and so appear in no bucket (projection_of stays None for them). + # Per-allele identity sidecar: one entry per linked allele, keyed by VRS digest (the same link + # set that seeds `annotations`, so the two maps share keys). Carries level, reference-frame HGVS + # (exactly one of hgvs_g/c/p is populated per allele), ClinGen id, and the member→defining relation. + + # projection_group -> the digests sharing it, for within-record projection grouping. A link's + # projection is the ≤1 *other* digest in its group; the protein apex and any pre-RT link carry a + # NULL group, so they appear in no bucket and projection_of stays None for them. group_digests: dict[int, list[str]] = {} for link in links: if link.projection_group is not None and link.allele.vrs_digest is not None: group_digests.setdefault(link.projection_group, []).append(link.allele.vrs_digest) - # The measured change's projection group anchors the cousin test: an nt member in a *different* group - # is a synonymous cousin (co_encodes / convergent), not the measured change's coordinate partner. + # Anchor for the convergence test: an nt member in a *different* projection group than the + # measured change is a convergent encoding (co_encodes / convergent), not a projection of it. defining_group = authoritative.projection_group if authoritative is not None else None alleles: dict[str, AlleleIdentity] = {} for link in links: allele = link.allele - # The projection sibling: the other digest in this link's projection_group, if any. A group has - # ≤2 members, so there is at most one sibling — this allele's projection partner. + # The other digest in this link's projection_group, if any (groups have ≤2 members). projection_of: Optional[str] = None if link.projection_group is not None: projection_of = next( @@ -240,7 +233,7 @@ def get_variant_detail( ) relation = relations.get(allele.vrs_digest) - is_cousin = is_convergent_cousin(allele.level, link.projection_group, defining_group=defining_group) + is_convergent = is_convergent_encoding(allele.level, link.projection_group, defining_group=defining_group) alleles[allele.vrs_digest] = AlleleIdentity( level=allele.level, hgvs=allele.hgvs_g or allele.hgvs_c or allele.hgvs_p, @@ -248,12 +241,13 @@ def get_variant_detail( is_focus=link.is_authoritative, relation=relation.value if relation is not None else None, derivation=( - None if link.is_authoritative else _derivation_for(assay_level=assay_level, is_cousin=is_cousin).value + None if link.is_authoritative else _derivation_for(assay_level=assay_level, is_convergent=is_convergent) ), projection_of=projection_of, ) return VariantDetail( + # TODO(#372) urn=variant.urn or "", scores=scores, counts=counts, diff --git a/src/mavedb/worker/jobs/external_services/clinvar.py b/src/mavedb/worker/jobs/external_services/clinvar.py index 557919299..d2482c6db 100644 --- a/src/mavedb/worker/jobs/external_services/clinvar.py +++ b/src/mavedb/worker/jobs/external_services/clinvar.py @@ -218,7 +218,7 @@ def alleles_linked_at_version(clinvar_version: str) -> set[int]: Disposition.NOT_APPLICABLE, EventReason.PROTEIN_LEVEL_ALLELE, source_version=clinvar_version, - error_message="Protein-level alleles are not linked to ClinVar; see nucleotide siblings.", + error_message="Protein-level alleles are not linked to ClinVar; see the nucleotide encodings.", metadata={"clingen_allele_id": caid}, ) continue diff --git a/src/mavedb/worker/jobs/external_services/gnomad.py b/src/mavedb/worker/jobs/external_services/gnomad.py index 31e541568..4c688692d 100644 --- a/src/mavedb/worker/jobs/external_services/gnomad.py +++ b/src/mavedb/worker/jobs/external_services/gnomad.py @@ -211,7 +211,7 @@ def alleles_linked_at_current_version(allele_ids: set[int]) -> set[int]: allele_id, Disposition.NOT_APPLICABLE, EventReason.PROTEIN_LEVEL_ALLELE, - error_message="Protein-level alleles are not linked to gnomAD; see nucleotide siblings.", + error_message="Protein-level alleles are not linked to gnomAD; see the nucleotide encodings.", metadata={"clingen_allele_id": caid}, ) continue From ca2f72704ad21d4a3a3e32db8fc927143e16b762 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Mon, 17 Aug 2026 11:05:58 -0700 Subject: [PATCH 108/252] docs(alleles): trim and correct comments/docstrings across allele views No identifier or behavior changes. Rewords the sibling/cousin terminology in prose (see the companion refactor commit for code identifiers), cuts repetitive design-history narration, and fixes one inverted claim in lib/allele_measurements.py's module docstring: a protein assay's consequence is reachable through the shared protein node precisely when it has *not* yet been reverse-translated, not "even when" it has. --- src/mavedb/lib/allele_identity.py | 86 ++++++++++++++-------- src/mavedb/lib/allele_measurements.py | 60 +++++++-------- src/mavedb/lib/annotation/context.py | 12 +-- src/mavedb/lib/clinical_controls.py | 30 ++++---- src/mavedb/models/mapping_record_allele.py | 43 +++-------- src/mavedb/routers/alleles.py | 6 +- src/mavedb/routers/score_sets.py | 8 ++ src/mavedb/view_models/allele_detail.py | 17 ++--- src/mavedb/view_models/allele_identity.py | 19 ++--- 9 files changed, 146 insertions(+), 135 deletions(-) diff --git a/src/mavedb/lib/allele_identity.py b/src/mavedb/lib/allele_identity.py index 7f4c12a74..1b065b223 100644 --- a/src/mavedb/lib/allele_identity.py +++ b/src/mavedb/lib/allele_identity.py @@ -8,8 +8,36 @@ This module owns that shared shape (:class:`AlleleIdentity`) and its provenance vocabulary (:class:`AlleleDerivation`). The Cat-VRS structural relation codes live in :mod:`lib.cat_vrs` -(``CatVrsRelation``); this module carries only the derivation (confidence/provenance) axis, which is -orthogonal. +(``CatVrsRelation``); this module carries the derivation (confidence/provenance) axis. + +Why ``relation`` and ``derivation`` are separate axes +---------------------------------------------------- +The two axes describe the *same* pair of alleles — member and focus — but ask questions from different +domains. ``relation`` is **biological**: how does this allele relate to the focus? ``derivation`` is +**provenance**: how did we arrive at this allele from the focus, and hence how much should it be +trusted? Neither answer determines the other, because a biological relationship does not say how it was +found and a route does not say what it found. + +Half of that independence is already visible in the tree: ``projection`` covers *both* ``translation_of`` +(nt→protein) and ``coordinate_representation_of`` (nt↔nt) — one route, two different biological +relationships. The other half is latent only because today exactly one pipeline path produces each +relationship. A ``co_encodes`` allele is biologically "a different codon encoding the same protein +change" however we reached it; reverse translation from the focus, a registry walk, and an independent +measurement in another score set are three different provenance claims about the same biology. + +Two consequences that make the split load-bearing rather than tidy: + +1. **Cat-VRS cannot express confidence, by design.** ``encodes`` is a claim about molecules: this + nucleotide allele encodes that protein change. It is equally true of a codon we measured and one + we guessed. A structural representation spec has no vocabulary for "we do not know which of these + six codons was in the well". +2. **``candidate`` is a fact about our data, not about the molecule.** A protein-level assay reported + an amino-acid change; which codon produced it is unknown *to MaveDB*, not unknown biologically. + Attempting to encode that on a molecular-relationship axis would misstate what the molecule does. + +Today's one-to-one mapping between the two vocabularies is **a fact about today's pipeline, not about +these axes. Never infer one from the other.** They are emitted together by +``variant_detail._derivation_for`` and ``allele_detail._member_label``, the single source of truth. """ from dataclasses import dataclass @@ -18,51 +46,51 @@ class AlleleDerivation(str, Enum): - """How an allele's representation was arrived at, *relative to the focus allele*: its - confidence/provenance axis. Orthogonal to the Cat-VRS structural ``relation`` axis: ``relation`` - says which level relates to which, ``derivation`` says how much to trust the representation. + """How an allele's representation was arrived at, *relative to the focus allele*: the + confidence/provenance axis, and the one the UI badges on. There is deliberately **no** ``authoritative`` value: the focus allele is marked by - :attr:`AlleleIdentity.is_focus`, not by a derivation. That keeps the axis meaningful without a - measurement — on the allele view there is no *measured* allele, only the queried focus. + :attr:`AlleleIdentity.is_focus`, not by a derivation. That keeps the axis meaningful even when a + variant was not explicitly measured. See the module docstring for why this axis is separate from + the Cat-VRS ``relation``, and why neither may be inferred from the other. """ - # Deterministic and precise, given (assembly, transcript): the focus's coordinate partner - # (nucleotide↔nucleotide) and its protein consequence (nucleotide→protein). PROJECTION = "projection" - # Reverse-translation of a *protein* focus: genuinely ambiguous. One member of the fanned-out - # equivalence class, not a precise and deterministic pair. + """Deterministic and precise, given (assembly, transcript): the focus's projection + (nucleotide↔nucleotide) and its protein consequence (nucleotide→protein).""" + CANDIDATE = "candidate" - # A distinct, precisely-known nucleotide change that *converges* on the same protein consequence - # a different codon produces. Not ambiguous like a candidate nor a projection *of* the focus. A - # separate, unmeasured variant that simply shares the consequence. + """Reverse-translation of a *protein* focus: genuinely ambiguous. One member of the fanned-out + equivalence class, not a precise and deterministic pair.""" + CONVERGENT = "convergent" + """A distinct, precisely-known nucleotide change that *converges* on the same protein consequence + a different codon produces. Not ambiguous like a candidate nor a projection *of* the focus. A + separate, unmeasured variant that simply shares the consequence.""" @dataclass(frozen=True) class AlleleIdentity: """The MaveDB molecular-identity facts for one allele in a view's ``alleles`` map, keyed by VRS - digest. ``level`` + reference-frame ``hgvs`` (exactly one of the allele's genomic/coding/protein - columns) are what the UI labels by — never the digest. - - Four axes, all read *relative to the view's focus allele*: - - - ``is_focus``: this allele is the one the view is anchored on (the measured allele on variant - detail; the queried allele/CAID on allele detail). Its ``relation``/``derivation`` are ``None`` — - it is the reference point the others are described against. - - ``relation`` (Cat-VRS, structural): member→focus relation (``coordinate_representation_of`` / - ``translation_of`` / ``encodes`` / ``co_encodes``). ``None`` for the focus itself. - - ``derivation`` (provenance): :class:`AlleleDerivation` — projection / candidate / convergent. - Orthogonal to ``relation``; never conflate them. - - ``projection_of`` (provenance): the VRS digest of this allele's c↔g projection sibling, when + digest. + + These four axes all read *relative to the view's focus allele*: + + - ``is_focus``: whether this allele is the one the view is anchored on. Its ``relation``/``derivation`` + are ``None``. This is the reference point the others are described against. + - ``relation`` (Cat-VRS, biological): how this allele relates to the focus in a biological sense. + - ``derivation`` (:class:`AlleleDerivation`, provenance): How we arrived at this allele from the focus, + hence how much to trust it. + - ``projection_of`` (provenance): the VRS digest of this allele's c↔g projection, when known. ``None`` for the protein apex, pre-reverse-translation data, or where the pairing is not resolved in this view. """ + is_focus: bool level: Optional[str] hgvs: Optional[str] clingen_allele_id: Optional[str] - is_focus: bool + relation: Optional[str] - derivation: Optional[str] + derivation: Optional[AlleleDerivation] projection_of: Optional[str] diff --git a/src/mavedb/lib/allele_measurements.py b/src/mavedb/lib/allele_measurements.py index 6cfb12797..23ca174d6 100644 --- a/src/mavedb/lib/allele_measurements.py +++ b/src/mavedb/lib/allele_measurements.py @@ -1,23 +1,22 @@ """The variant page's measurements list — ``GET /clingen-alleles/{caid}/measurements``. -Given a ClinGen allele id — a nucleotide ``CA`` or a protein ``PA`` — return every measurement related to -it across sequence levels. Two measurements are related when their mapping records share an allele (in the -mapping + reverse-translation graph). A ``CA`` anchors on its nt alleles; a ``PA`` on the protein allele. -Each measurement is labelled by its own measured level relative to the query: ``direct`` (measured at the -query), ``protein_consequence``, or ``nucleotide_encoding``. +Given a ClinGen allele id, return every measurement related to it across sequence levels. Two measurements +are related when their mapping records share an allele. A ``CA`` anchors on its nt alleles; a ``PA`` on the +protein allele. Each measurement is labelled by its own measured level relative to the query: ``direct`` +(measured at the query), ``protein_consequence``, or ``nucleotide_encoding``. -Example. Reference codon ``TTT`` (Phe200): both ``c.598T>C`` and the sibling ``c.600T>A`` encode -``p.Phe200Leu``. Call them ``CA1``, ``CA2``, ``PA1`` — X assays ``c.598T>C`` (nt), Y assays -``p.Phe200Leu`` (protein), Z assays ``c.600T>A`` (nt): +Example. Reference codon ``TTT`` (Phe200): both ``c.598T>C`` and the encoding ``c.600T>A`` encode +``p.Phe200Leu``. Call them ``CA1``, ``CA2``, ``PA1``. Score set X assays ``c.598T>C`` (nt), score set Y +assays ``p.Phe200Leu`` (protein), score set Z assays ``c.600T>A`` (nt): query CA1 → X direct, Y protein_consequence, Z nucleotide_encoding query PA1 → Y direct, X and Z nucleotide_encoding A CA query reaches its protein consequence (Y) through the shared protein node ``PA1``. This is what makes -Y reachable when a protein assay has not been reverse-translated: its record then links only the protein -node, with no nt allele for the CA anchor to match, so the apex is the only path to it. The sibling Z is -reached without the apex — reverse translation links every record to its full synonymous nt set, so Z's -record already links ``CA1`` directly. There is no page/search distinction: every surface runs one query. +Y reachable even when a protein assay has not been reverse-translated: its record then links only the +protein node, with no nt allele for the CA anchor to match, so the apex is the only path to it. The +encoding Z is reached without the apex: reverse translation links every record to its full synonymous nt set, +so Z's record already links ``CA1`` directly. The protein-consequence step is resolved once, in :func:`_resolve_protein_apex`, which also measures it: a consequence resolving to more than one protein id (a data-integrity signal) is logged, and the number of @@ -25,9 +24,10 @@ Authorization (``has_permission``, per ``lib/score_sets.py``): score-set READ gates whether a measurement appears; calibration READ gates only its inline classification. ``as_of`` reconstructs which records and -links were live at a past instant; scores and calibrations are as-of-invariant. +links were live at a past instant. -Distinct from :func:`lib.variant_detail.get_variant_detail` — the record-scoped detail of one measurement. +Distinct from :func:`lib.variant_detail.get_variant_detail`, which is the record-scoped detail of one +measurement. """ import logging @@ -63,17 +63,15 @@ class MeasurementRelationship(str, Enum): direct = "direct" # assayed at the queried allele protein_consequence = "protein_consequence" # protein measurement of a CA's consequence - # an nt measurement of a PA's encoding, or a CA's sibling nt change (encodes the same protein) - nucleotide_encoding = "nucleotide_encoding" + nucleotide_encoding = "nucleotide_encoding" # an nt measurement of the protein apex's encoding @dataclass(frozen=True) class AlleleMeasurement: - """One measurement in the query's equivalence class (transit; serialized by ``view_models``). + """One measurement in the query's equivalence class. - ``assay_level`` is where this measurement was assayed (always shown, the clinically load-bearing - fact). ``preferred_classification`` is the readable classification the UI defaults to (primary-first - cascade, RUO excluded; ``None`` when none applies or the calibration is unreadable). + ``assay_level`` is the sequence level at which this measurement was assayed. ``preferred_classification`` + is the readable classification the UI will default to. """ variant_urn: str @@ -94,6 +92,11 @@ class _ProteinApex: """Protein consequence(s) co-membered with a CA anchor's nt alleles — the shared node a CA query reaches protein measurements through. One PAID is well-formed; ``len(caids) > 1`` is a data-quality signal (a duplicate or divergent consequence). ``unregistered`` counts apex rows without a PAID yet. + + nt→protein is deterministic only per (assembly, transcript), and the anchor includes the CAID's + transcript-agnostic genomic allele. So records targeting different transcripts of one gene reach + different consequences legitimately, and >1 PAID is a signal to investigate rather than proof of + corruption. """ allele_ids: list[int] @@ -107,9 +110,8 @@ def _preferred_classification( """The variant's preferred *readable* functional classification, or ``None``. Mirrors the UI's calibration cascade — ``primary`` then ``investigator_provided``, strongest evidence - within a tier, then ``id`` for determinism. RUO calibrations are excluded outright (never shown here), - and calibrations the caller can't read are skipped rather than blanking a readable call. Kept in sync - with the UI's ``activeCalibrationOptions`` default. + within a tier, then ``id`` for determinism. RUO calibrations are excluded outright, and calibrations + the caller can't read are skipped rather than blanking a readable call. """ candidates = [ (classification, calibration) @@ -191,9 +193,8 @@ def get_allele_measurements( """The measurements in ``clingen_allele_id``'s equivalence class, or ``[]`` if it resolves to no live record. - A CA query returns ``direct`` measurements, its ``protein_consequence`` (reached through the protein - apex, see the module docstring), and the ``nucleotide_encoding`` siblings. A PA query returns its - ``direct`` protein measurements and their ``nucleotide_encoding`` encodings. + A CA query returns ``direct`` measurements, its ``protein_consequence``, and the ``nucleotide_encoding`` + members. A PA query returns its ``direct`` protein measurements and their ``nucleotide_encoding`` encodings. """ anchor = db.execute(select(Allele.id, Allele.level).where(Allele.clingen_allele_id == clingen_allele_id)).all() if not anchor: @@ -203,13 +204,14 @@ def get_allele_measurements( entry_is_protein = any(row.level == SequenceLevel.protein.value for row in anchor) # A CA query always includes its protein consequence, reached through the shared protein node, so it is - # reachable even when a protein assay isn't reverse-translated yet (its record then links only the - # protein node). Transitive, so it's measured: cardinality >1 logged, provenance tallied below. No-op - # for a PA (already anchored on the protein). + # reachable even when a protein assay isn't reverse-translated yet. This is calculated transitively, but + # the calculation should be deterministic. apex: Optional[_ProteinApex] = None if not entry_is_protein: apex = _resolve_protein_apex(db, anchor_ids, as_of=as_of) anchor_ids += apex.allele_ids + + # Warn if multiple distinct protein apexes were resolved for this CA. if len(apex.caids) > 1: logger.warning( msg=( diff --git a/src/mavedb/lib/annotation/context.py b/src/mavedb/lib/annotation/context.py index 3e67f6d39..bf1e3d13b 100644 --- a/src/mavedb/lib/annotation/context.py +++ b/src/mavedb/lib/annotation/context.py @@ -26,9 +26,9 @@ class VariantAnnotationContext: """A variant's annotation inputs, sourced from it's allele-graph. - ``record`` is the live (or as-of) ``MappingRecord`` — its ``mapping_api_version`` / ``mapped_date`` + ``record`` is the live (or as-of) ``MappingRecord``. Its ``mapping_api_version`` / ``mapped_date`` supply VA provenance, and its ``ValidTime`` is what ``as_of`` and supersession are evaluated against. - ``measured_allele`` is the authoritative allele (the assayed representation); its ``post_mapped`` VRS is + ``measured_allele`` is the authoritative allele (the assayed representation) and its ``post_mapped`` VRS is the concrete study-result focus. ``subject_variant`` is the VA *proposition* subject. When projections exist, this is a Cat-VRS ``CategoricalVariant`` object. Otherwise, it is the measured ``MolecularVariation``. """ @@ -45,9 +45,9 @@ def variant_annotation_context( ) -> Optional[VariantAnnotationContext]: """Assemble the annotation context for ``variant`` from the live (or as-of) mapping substrate. - Returns ``None`` when the variant is unmapped at ``as_of`` — no live mapping record, or an authoritative - allele that carries no ``post_mapped`` VRS (nothing to annotate). One record fetch + one allele-link - fetch; the Cat-VRS transit is built from the same links, so the subject costs no redundant query. + Returns ``None`` when the variant is unmapped at ``as_of`` or an authoritative allele that carries + no ``post_mapped`` VRS (nothing to annotate). One record fetch + one allele-link fetch; the Cat-VRS + transit is built from the same links, so the subject costs no redundant query. """ record = db.scalar( select(MappingRecord).where(MappingRecord.variant_id == variant.id).where(MappingRecord.live_at(as_of)) @@ -62,7 +62,7 @@ def variant_annotation_context( # The proposition subject follows the measured as anchor rule: the categorical variant when it carries a # projection member, else the concrete measured variation. The VA subject is deliberately *narrow* - # (include_convergent=False): the synonymous cousins are dropped, because VA-Spec carries no per-member + # (include_convergent=False): the convergent encodings are dropped, because VA-Spec carries no per-member # provenance to mark them as unmeasured, and StudyResult.focusVariant already pins the concrete measured # allele. transit = build_categorical_variant(links, name=variant.urn or "", include_convergent=False) diff --git a/src/mavedb/lib/clinical_controls.py b/src/mavedb/lib/clinical_controls.py index b852ad39e..3f9fd41e0 100644 --- a/src/mavedb/lib/clinical_controls.py +++ b/src/mavedb/lib/clinical_controls.py @@ -4,9 +4,8 @@ score set's variants, reached by walking ``ClinvarAlleleLink → Allele → MappingRecordAllele → MappingRecord → Variant`` with every ``ValidTime`` hop constrained to the same instant (``as_of``, defaulting to currently-live). Both serving endpoints — the controls list -(``GET /score-sets/{urn}/clinical-controls``) and its options facet (``.../options``) — walk that same -chain, so it lives here once: the two cannot drift, and the ``as_of`` semantics are defined in a single -place. +(``GET /score-sets/{urn}/clinical-controls``) and its options facet (``.../options``) walk the chain +that is defined here. """ from dataclasses import dataclass @@ -30,11 +29,13 @@ class ControlVariantLink: """One (score-set variant, annotated allele) pair that a clinical control reaches. - ``allele_digest`` is the VRS digest of the allele the control actually annotates — the entity - ClinVar made its call on. A client applies **D78 precedence** by comparing it to the variant's - authoritative ``assay_level_digest``: a match is a call *on the assayed level* (which wins), a - mismatch is a call on a *projection sibling* (the fallback that only fires when the assayed level - is unannotated). Without the digest the two are indistinguishable and precedence is unimplementable. + ``allele_digest`` is the VRS digest of the allele the control annotates. A client compares it to + the variant's authoritative ``assay_level_digest``: + - a match is a call *on the assayed level* (this call takes precedence) + - a mismatch is a call on an *encoding* level (this call is only used when the assayed level is + unannotated). + + Without the digest, the two are indistinguishable and precedence is unimplementable. """ variant_urn: str @@ -68,9 +69,9 @@ def get_clinical_controls_with_variant_urns( db_version: Optional[str] = None, ) -> list[tuple[ClinvarControl, list[ControlVariantLink]]]: """The live ClinVar controls for a score set, each paired with the score-set variants that link to - it — every link carrying the digest of the allele the control annotates (see - :class:`ControlVariantLink`). Optionally narrowed to one control DB (``db_name``) and/or release - (``db_version``). Controls come back in first-seen order; each control's links preserve query order.""" + it. Every link carrying the digest of the allele the control annotates (see :class:`ControlVariantLink`). + Optionally narrowed to one control DB (``db_name``) and/or release (``db_version``). Controls come back + in first-seen order. Each control's links preserve query order.""" stmt = _scope_to_live_score_set_controls( select(ClinvarControl, Variant.urn.label("variant_urn"), Allele.vrs_digest.label("allele_digest")), score_set_id, @@ -84,8 +85,8 @@ def get_clinical_controls_with_variant_urns( controls: dict[int, ClinvarControl] = {} links_by_control: dict[int, list[ControlVariantLink]] = {} - # A persisted variant reached through the join always carries a URN (unique, set at creation); - # narrow the nullable column so the URN matches the view model's required `str`. + # A persisted variant reached through the join always carries a URN. Narrow the nullable column so the + # URN matches the view model's required `str`. TODO(#372) for ctrl, variant_urn, allele_digest in db.execute(stmt).tuples(): if variant_urn is None: continue @@ -102,8 +103,7 @@ def get_clinical_control_options( db: Session, score_set_id: int, *, as_of: Optional[datetime] = None ) -> list[tuple[str, list[str]]]: """The options facet: each control DB live-linked to the score set, paired with its available - versions newest-first. Options are returned even for pairwise groupings that may have no variants - when ultimately requested together (the list endpoint 404s that case).""" + versions newest-first.""" stmt = _scope_to_live_score_set_controls( select(ClinvarControl.db_name, ClinvarControl.db_version), score_set_id, as_of ) diff --git a/src/mavedb/models/mapping_record_allele.py b/src/mavedb/models/mapping_record_allele.py index 5179877c2..ce57f259d 100644 --- a/src/mavedb/models/mapping_record_allele.py +++ b/src/mavedb/models/mapping_record_allele.py @@ -25,26 +25,13 @@ class MappingRecordAllele(ValidTime, Base): ForeignKey("alleles.id", ondelete="RESTRICT"), nullable=False, ) - # Provenance: ``True`` when this allele was the assay's actual measurement - # for this mapping record (sourced from vrs_map, with real alignment QC), - # ``False`` when it was derived by the cross-level or same-level - # synonymous translator. The flag lives on the link rather than on - # ``Allele`` because the same VRS allele can be authoritative for one - # mapping record and translator-derived for another. + # True for the assay's actual measured allele (from vrs_map); False for translator-derived + # alleles. Lives on the link, not Allele, since the same allele can be authoritative for one + # mapping record and derived for another. is_authoritative: Mapped[bool] = Column(Boolean, nullable=False, default=False) - # Within-record projection grouping. The reverse-translation job fans a variant's protein - # consequence out into projection pairs, each a coding/genomic pair (the same change expressed - # at two levels). The two links of one pair share a ``projection_group`` value - # (an integer index, 0..N-1, local to this mapping record), so we can reconstruct projections - # onto transcripts for more precise provenance in consumers. ``NULL`` for links that belong to no - # pair: the shared protein apex, and any link written before reverse translation has run. - # - # It is a grouping key, NOT a cross-record identity — regenerated whenever a record is - # superseded and re-linked. Identity lives in the allele VRS digests; two records that happen to - # share a transcript and codon re-encode the pairing independently. Serving resolves the - # canonical projection by pairing this with ``is_authoritative``: the authoritative (measured) - # allele's group gathers its sibling links, yielding its c/g projection. See the RT job for the - # group-assignment and authoritative fold-in logic. + # Pairs a link with its coding/genomic counterpart within one mapping record (set by the + # reverse-translation job). NULL for unpaired links. Local to this record, not a cross-record + # identity — see the RT job for group-assignment logic. projection_group = Column(Integer, nullable=True) mapping_record: Mapped["MappingRecord"] = relationship("MappingRecord", back_populates="allele_links") @@ -59,11 +46,8 @@ class MappingRecordAllele(ValidTime, Base): "ix_mapping_record_alleles_allele_id", "allele_id", ), - # At most one live link per (mapping_record, allele). The job retires (closes valid_to) - # rather than deletes, so superseded links remain for point-in-time queries; only the - # live row participates in this constraint. A derived link is never written when an - # authoritative one already exists for the pair, so is_authoritative is intentionally - # absent from the key — a record links a given allele once. + # At most one live link per (mapping_record, allele); retired links (valid_to set) are + # exempt so history is preserved. Index( "uq_mapping_record_alleles_live", "mapping_record_id", @@ -71,14 +55,9 @@ class MappingRecordAllele(ValidTime, Base): unique=True, postgresql_where=text("valid_to IS NULL"), ), - # At most one live *authoritative* link per mapping record. The index above keys on - # (record, allele) and so cannot stop two different alleles both being flagged authoritative - # for one record; this one enforces the "one authoritative (measured) allele per record" - # invariant directly. It is the backstop the serving layer relies on: the lean whole-set view - # (lib/score_set_variants) inner-joins the authoritative link expecting it 1:1 with the - # variant, so a second live authoritative link would silently duplicate the variant (and - # double-count its score) rather than raise. With this index, that mistake fails loud in the - # mapping job at write time instead. + # At most one live authoritative link per mapping record. lib/score_set_variants relies on + # this 1:1 invariant (inner join on the authoritative link); a second one would silently + # duplicate the variant and double-count its score. Index( "uq_mapping_record_alleles_live_authoritative", "mapping_record_id", diff --git a/src/mavedb/routers/alleles.py b/src/mavedb/routers/alleles.py index abd575718..470765063 100644 --- a/src/mavedb/routers/alleles.py +++ b/src/mavedb/routers/alleles.py @@ -79,8 +79,8 @@ def get_allele( *, response: Response, identifier: str = Path( - description="A GA4GH VRS digest (one allele), or a ClinGen allele id — a nucleotide CAID (the " - "nt-canonical change, its genomic + coding frames) or a protein PAID.", + description="A GA4GH VRS digest (one allele), or a ClinGen allele id: a nucleotide CAID " + "(the nt-canonical change, its genomic + coding frames) or a protein PAID.", json_schema_extra={"example": "ga4gh:VA.0123abcd"}, ), as_of: Optional[datetime] = Query( @@ -96,7 +96,7 @@ def get_allele( ): """Fetch the detail envelope for a deduplicated allele, by any of its identifiers. - The allele-grain sibling of ``GET /variants/{urn}``. Flat anchor identity (digest, level, HGVS, + The allele-grain counterpart of ``GET /variants/{urn}``. Flat anchor identity (digest, level, HGVS, ClinGen id, spec-pure VRS) plus the cross-layer equivalence class (each member labelled relative to the focus) and a digest-keyed annotation map. The ``identifier`` may be: diff --git a/src/mavedb/routers/score_sets.py b/src/mavedb/routers/score_sets.py index 9b3203726..7a4698b77 100644 --- a/src/mavedb/routers/score_sets.py +++ b/src/mavedb/routers/score_sets.py @@ -2591,6 +2591,8 @@ async def get_clinical_controls_for_score_set( ) if not controls: + # Can legitimately fire even for a (db_name, db_version) sourced from `.../options`: liveness + # is re-evaluated per call, see that endpoint's docstring. logger.info( msg="No clinical control variants matching the provided filters are associated with the requested score set.", extra=logging_context(), @@ -2647,6 +2649,12 @@ async def get_clinical_controls_options_for_score_set( ) -> list[dict[str, Union[str, list[str]]]]: """ Fetch clinical control options for a given score set. + + Each ``(db_name, db_version)`` pair returned here was live at the moment of this call, but + liveness is re-evaluated independently per request. A pair fetched here can have its backing + ``ClinvarAlleleLink`` retired before a later call to ``GET /score-sets/{urn}/clinical-controls`` + filters on it, in which case that call 404s. Pin an explicit ``as_of`` on both calls to avoid this + possibility. """ save_to_logging_context( { diff --git a/src/mavedb/view_models/allele_detail.py b/src/mavedb/view_models/allele_detail.py index 67ec74bd3..7a6fabe5b 100644 --- a/src/mavedb/view_models/allele_detail.py +++ b/src/mavedb/view_models/allele_detail.py @@ -1,10 +1,7 @@ """Response view models for the allele-detail view (``GET /alleles/{digest|CAID}``). -The pydantic serialization boundary over the :mod:`lib.allele_detail` transit dataclasses. The -allele-grain sibling of ``view_models.variant_detail``. The ``alleles`` map reuses the shared -:class:`AlleleIdentity` (``view_models.allele_identity``) and the digest-keyed -:class:`AlleleAnnotations` block (``view_models.allele_annotation``) — both shared with the -variant view. +Pydantic serialization boundary over the :mod:`lib.allele_detail` transit dataclasses; the +allele-grain counterpart of ``view_models.variant_detail``. """ from typing import Any, Optional @@ -17,13 +14,9 @@ class AlleleDetail(BaseModel): """The allele-detail envelope (``GET /alleles/{digest|CAID}``). - Flat anchor-identity fields (``digest`` / ``level`` / ``hgvs`` / ``clingenAlleleId`` and the spec-pure - GA4GH ``vrs`` variation) plus the MaveDB layer riding alongside. This layer, keyed by VRS digest, - contains the ``alleles`` map: the full cross-layer equivalence class. Each entry is an ``AlleleIdentity``, - labelled relative to the focus (``isFocus`` marks the queried allele / the CAID's representations) and - the digest-keyed ``annotations`` map. The two maps share keys. Measurement-agnostic: no score, - classification, or version standing, and no re-anchored Cat-VRS (those belong to ``GET /variants/{urn}``). - Absent fields are omitted. + ``alleles`` is the full cross-layer equivalence class, keyed by VRS digest; ``isFocus`` marks + the queried allele. ``annotations`` shares those same keys. Measurement-agnostic: no score, + classification, or re-anchored Cat-VRS (those belong to ``GET /variants/{urn}``). """ digest: str diff --git a/src/mavedb/view_models/allele_identity.py b/src/mavedb/view_models/allele_identity.py index b77743df5..a7faf24fc 100644 --- a/src/mavedb/view_models/allele_identity.py +++ b/src/mavedb/view_models/allele_identity.py @@ -1,28 +1,29 @@ """Response view model for the shared allele molecular-identity shape. -The pydantic boundary over :class:`lib.allele_identity.AlleleIdentity` (``from_attributes`` coerces it; -aliases camelize per the shared base config). Shared by both serving views — the ``alleles`` map on -``GET /variants/{urn}`` and on ``GET /alleles/{digest}``. See the lib docstring for the focus-relative -semantics of each axis. +Pydantic boundary over :class:`lib.allele_identity.AlleleIdentity`. Shared by the ``alleles`` map +on both ``GET /variants/{urn}`` and ``GET /alleles/{digest}``. See the lib docstring for the +focus-relative semantics of each axis. """ from typing import Optional +from mavedb.lib.allele_identity import AlleleDerivation from mavedb.view_models.base.base import BaseModel class AlleleIdentity(BaseModel): - """One allele in a view's ``alleles`` map, keyed by VRS digest and labelled relative to the view's - focus allele. ``isFocus`` marks the anchor (measured allele / queried allele); ``relation`` and - ``derivation`` describe every other member's structural + provenance relationship to it, and are - absent on the focus itself.""" + """One allele in a view's ``alleles`` map, keyed by VRS digest and labelled relative to the + view's focus allele. ``isFocus`` marks the anchor; ``relation`` and ``derivation`` describe + every other member's relationship to it and are absent on the focus itself. + """ level: Optional[str] = None hgvs: Optional[str] = None clingen_allele_id: Optional[str] = None is_focus: bool relation: Optional[str] = None - derivation: Optional[str] = None + derivation: Optional[AlleleDerivation] = None + # VRS digest of this allele's c<->g projection counterpart, when one exists. projection_of: Optional[str] = None class Config: From 6e8f407012c482a6c4c59e0d375fe984231b182a Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Mon, 17 Aug 2026 11:57:48 -0700 Subject: [PATCH 109/252] fix(refget): declare Content-Length for the bytes actually streamed The refget sequence endpoint set Content-Length to the length of the whole sequence on every response, including subsequence requests. A request for NC_000016.10?start=2077608&end=2085800 streamed 8192 bytes under a declared 90338345, and the ASGI server aborted the response with "Response content shorter than Content-Length" once the status line was already committed. GZipMiddleware strips Content-Length from compressed responses, so only clients requesting identity encoding were affected, and TestClient's default gzip request meant no existing test could observe the header. - Resolve start/end to concrete half-open bounds as soon as the sequence length is known, and derive validation, Content-Length, Content-Range, and the generator bounds from those same values - Run bounds validation whenever either bound is supplied, not only when both are; a one-sided out-of-range request previously skipped validation and returned an empty 200 carrying the full sequence length - Apply the same one-sided bounds check to the seqrepo sequence endpoint, which never set Content-Length and so could not crash, but returned a truncated body under a 200 for coordinates past the end of the sequence. It now rejects them with 422, the code that endpoint already used for start > end - Cover every request shape with tests asserting Content-Length equals the returned body length, pinning Accept-Encoding: identity so gzip cannot mask a regression Closes #846 --- src/mavedb/routers/refget.py | 34 +++++++++++------ src/mavedb/routers/seqrepo.py | 18 ++++++++- tests/routers/test_refget.py | 72 +++++++++++++++++++++++++++++++++++ tests/routers/test_seqrepo.py | 37 ++++++++++++++++++ 4 files changed, 148 insertions(+), 13 deletions(-) diff --git a/src/mavedb/routers/refget.py b/src/mavedb/routers/refget.py index 979c63d13..70e18e2bf 100644 --- a/src/mavedb/routers/refget.py +++ b/src/mavedb/routers/refget.py @@ -183,37 +183,47 @@ def get_sequence( ) seq_id = seq_ids[0] - seqinfo = sr.sequences.fetch_seqinfo(seq_id) + seq_len = sr.sequences.fetch_seqinfo(seq_id)["len"] - if start is not None and end is not None: - if start >= seqinfo["len"]: + # Resolve to concrete half-open bounds up front so validation, Content-Length, and the streamed + # body all agree, even when only one of start/end is supplied. + seq_start = start if start is not None else 0 + seq_end = end if end is not None else seq_len + + if start is not None or end is not None: + if seq_start >= seq_len: raise HTTPException( status_code=416, detail="Invalid coordinates: start > sequence length", - headers={"Content-Range": f"bytes */{seqinfo['len']}"}, + headers={"Content-Range": f"bytes */{seq_len}"}, ) - if end > seqinfo["len"]: + if seq_end > seq_len: raise HTTPException( status_code=416, detail="Invalid coordinates: end > sequence length", - headers={"Content-Range": f"bytes */{seqinfo['len']}"}, + headers={"Content-Range": f"bytes */{seq_len}"}, ) - if not (0 <= start <= end <= seqinfo["len"]): + if not (0 <= seq_start <= seq_end <= seq_len): raise HTTPException( status_code=416, detail="Invalid coordinates: must obey 0 <= start <= end <= sequence_length", - headers={"Content-Range": f"bytes */{seqinfo['len']}"}, + headers={"Content-Range": f"bytes */{seq_len}"}, ) - headers = {"Content-Length": str(seqinfo["len"])} - if start is not None and end is not None and range_header: + # Content-Length must match bytes actually streamed. Overstating it aborts the response mid-stream + # once the ASGI server has already committed the status line. + headers = {"Content-Length": str(seq_end - seq_start)} + if range_header: status = 206 - headers["Content-Range"] = f"bytes {start}-{end - 1}/{seqinfo['len']}" + headers["Content-Range"] = f"bytes {seq_start}-{seq_end - 1}/{seq_len}" headers["Accept-Ranges"] = "bytes" else: status = 200 headers["Accept-Ranges"] = "none" return StreamingResponse( - sequence_generator(sr, seq_ids[0], start, end), media_type="text/plain", status_code=status, headers=headers + sequence_generator(sr, seq_id, seq_start, seq_end), + media_type="text/plain", + status_code=status, + headers=headers, ) diff --git a/src/mavedb/routers/seqrepo.py b/src/mavedb/routers/seqrepo.py index 42ec14645..9cd271312 100644 --- a/src/mavedb/routers/seqrepo.py +++ b/src/mavedb/routers/seqrepo.py @@ -71,7 +71,23 @@ def get_sequence( status_code=400, detail=f"Multiple sequences exist for alias '{alias}'. Use an explicit namespace." ) - return StreamingResponse(sequence_generator(sr, seq_ids[0], start, end), media_type="text/plain") + seq_id = seq_ids[0] + seq_len = sr.sequences.fetch_seqinfo(seq_id)["len"] + + # Resolve to concrete half-open bounds so the check and the streamed body agree, even when only + # one of start/end is supplied. + seq_start = start if start is not None else 0 + seq_end = end if end is not None else seq_len + + if start is not None or end is not None: + if not 0 <= seq_start < seq_len: + logger.error(msg="Invalid coordinates: start lies outside the sequence.", extra=logging_context()) + raise HTTPException(status_code=422, detail=f"Invalid coordinates: must obey 0 <= start < {seq_len}") + if not seq_start <= seq_end <= seq_len: + logger.error(msg="Invalid coordinates: end lies outside the sequence.", extra=logging_context()) + raise HTTPException(status_code=422, detail=f"Invalid coordinates: must obey start <= end <= {seq_len}") + + return StreamingResponse(sequence_generator(sr, seq_id, seq_start, seq_end), media_type="text/plain") @router.get("/metadata/{alias}", response_model=SeqRepoMetadata, summary="Get sequence metadata by alias") diff --git a/tests/routers/test_refget.py b/tests/routers/test_refget.py index 760b9f022..75ddde71a 100644 --- a/tests/routers/test_refget.py +++ b/tests/routers/test_refget.py @@ -76,6 +76,78 @@ def test_get_sequence_with_range_query(client, entry): assert resp.text == metadata["seq"][start:end] +@pytest.mark.parametrize("entry", TEST_SEQREPO_INITIAL_STATE) +def test_get_sequence_only_start(client, entry): + alias = list(entry.keys())[0] + metadata = list(entry.values())[0] + start = 1 + resp = client.get(f"/api/v1/refget/sequence/{alias}", params={"start": start}) + assert resp.status_code == 200 + assert resp.text == metadata["seq"][start:] + + +@pytest.mark.parametrize("entry", TEST_SEQREPO_INITIAL_STATE) +def test_get_sequence_only_end(client, entry): + alias = list(entry.keys())[0] + metadata = list(entry.values())[0] + end = 3 + resp = client.get(f"/api/v1/refget/sequence/{alias}", params={"end": end}) + assert resp.status_code == 200 + assert resp.text == metadata["seq"][:end] + + +def test_get_sequence_invalid_query_range_only_start_negative(client): + resp = client.get(f"/api/v1/refget/sequence/{VALID_ENSEMBL_IDENTIFIER}", params={"start": -1}) + assert resp.status_code == 416 + assert "Invalid coordinates" in resp.text + assert "Content-Range" in resp.headers + + +def test_get_sequence_invalid_query_range_only_start_too_large(client): + resp = client.get(f"/api/v1/refget/sequence/{VALID_ENSEMBL_IDENTIFIER}", params={"start": 7}) + assert resp.status_code == 416 + assert "Invalid coordinates" in resp.text + assert "Content-Range" in resp.headers + + +def test_get_sequence_invalid_query_range_only_end_too_large(client): + resp = client.get(f"/api/v1/refget/sequence/{VALID_ENSEMBL_IDENTIFIER}", params={"end": 10}) + assert resp.status_code == 416 + assert "Invalid coordinates" in resp.text + assert "Content-Range" in resp.headers + + +@pytest.mark.parametrize( + "params,headers", + [ + ({}, {}), + ({"start": 1, "end": 3}, {}), + ({"start": 1}, {}), + ({"end": 3}, {}), + ({}, {"Range": "bytes=1-3"}), + ], +) +def test_get_sequence_content_length_matches_body(client, params, headers): + resp = client.get( + f"/api/v1/refget/sequence/{VALID_ENSEMBL_IDENTIFIER}", + params=params, + # Accept-Encoding: identity is required — GZipMiddleware strips Content-Length from compressed responses. + headers={**headers, "Accept-Encoding": "identity"}, + ) + assert resp.status_code in (200, 206) + assert resp.headers["Content-Length"] == str(len(resp.content)) + + +def test_get_sequence_range_header_reports_requested_span(client): + resp = client.get( + f"/api/v1/refget/sequence/{VALID_ENSEMBL_IDENTIFIER}", + headers={"Range": "bytes=1-2", "Accept-Encoding": "identity"}, + ) + assert resp.status_code == 206 + assert resp.headers["Content-Range"] == "bytes 1-2/4" + assert resp.headers["Content-Length"] == "2" + + def test_get_sequence_not_found(client): resp = client.get("/api/v1/refget/sequence/notfound") assert resp.status_code == 404 diff --git a/tests/routers/test_seqrepo.py b/tests/routers/test_seqrepo.py index 231f06a5e..130e1a847 100644 --- a/tests/routers/test_seqrepo.py +++ b/tests/routers/test_seqrepo.py @@ -46,12 +46,49 @@ def test_get_sequence_multiple_ids(client): assert "Multiple sequences exist" in resp.text +@pytest.mark.parametrize("entry", TEST_SEQREPO_INITIAL_STATE) +def test_get_sequence_only_start(client, entry): + alias = list(entry.keys())[0] + metadata = list(entry.values())[0] + start = 1 + resp = client.get(f"/api/v1/seqrepo/sequence/{alias}?start={start}") + assert resp.status_code == 200 + assert resp.text == metadata["seq"][start:] + + +@pytest.mark.parametrize("entry", TEST_SEQREPO_INITIAL_STATE) +def test_get_sequence_only_end(client, entry): + alias = list(entry.keys())[0] + metadata = list(entry.values())[0] + end = 3 + resp = client.get(f"/api/v1/seqrepo/sequence/{alias}?end={end}") + assert resp.status_code == 200 + assert resp.text == metadata["seq"][:end] + + def test_get_sequence_invalid_coords(client): resp = client.get(f"/api/v1/seqrepo/sequence/{VALID_ENSEMBL_IDENTIFIER}?start=10&end=5") assert resp.status_code == 422 assert "Invalid coordinates" in resp.text +# Coordinates outside the sequence used to stream a truncated body under a 200 instead of being rejected. +@pytest.mark.parametrize( + "query", + [ + "start=10&end=12", + "start=1&end=12", + "start=10", + "end=12", + "start=-1&end=2", + ], +) +def test_get_sequence_coords_outside_sequence(client, query): + resp = client.get(f"/api/v1/seqrepo/sequence/{VALID_ENSEMBL_IDENTIFIER}?{query}") + assert resp.status_code == 422 + assert "Invalid coordinates" in resp.text + + @pytest.mark.parametrize("entry", TEST_SEQREPO_INITIAL_STATE) def test_get_metadata_success(client, entry): alias = list(entry.keys())[0] From 26664e3fe61268613fe92368fb3282a072ac90f1 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Mon, 17 Aug 2026 11:58:02 -0700 Subject: [PATCH 110/252] chore(tests): sharpen unit and integration markers, extend refget coverage Tighten the marker definitions so the distinction is about mocking and scope rather than a vague notion of size, and mark the refget and seqrepo router suites accordingly. - Redefine `integration` as an end-to-end multi-component flow with no internal mocking, and `unit` as fast and isolated, explicitly allowing real collaborators when they are local, fast, and deterministic - Mark tests/routers/test_refget.py and tests/routers/test_seqrepo.py as unit suites - Cover the refget service-info endpoint, including the HGVS_SEQREPO_DIR-derived data version and its "unknown" fallback - Cover the 400 returned when start/end query parameters are combined with a Range header --- pyproject.toml | 4 ++-- tests/routers/test_refget.py | 30 ++++++++++++++++++++++++++++++ tests/routers/test_seqrepo.py | 2 ++ 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 871d29941..763d50f4a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -111,8 +111,8 @@ testpaths = "tests/" pythonpath = "." norecursedirs = "tests/helpers/" markers = """ - integration: tests that exercise multi-component flows and/or database interactions. - unit: fast unit tests with isolated dependencies. + integration: tests that exercise a multi-component flow end-to-end, with no internal mocking. + unit: fast, isolated tests. Real un-mocked collaborators are fine if they're local, fast, and deterministic. network: tests which interact with external services over a network connection. slow: tests which are slow running. """ diff --git a/tests/routers/test_refget.py b/tests/routers/test_refget.py index 75ddde71a..762f60be4 100644 --- a/tests/routers/test_refget.py +++ b/tests/routers/test_refget.py @@ -3,6 +3,8 @@ import pytest +pytestmark = pytest.mark.unit + arq = pytest.importorskip("arq") cdot = pytest.importorskip("cdot") fastapi = pytest.importorskip("fastapi") @@ -12,6 +14,24 @@ from tests.helpers.constants import TEST_SEQREPO_INITIAL_STATE, VALID_ENSEMBL_IDENTIFIER +@pytest.mark.parametrize( + "env_value,expected_data_version", + [(None, "unknown"), ("/some/path/seqrepo/20240101", "20240101")], +) +def test_service_info(client, monkeypatch, env_value, expected_data_version): + if env_value is None: + monkeypatch.delenv("HGVS_SEQREPO_DIR", raising=False) + else: + monkeypatch.setenv("HGVS_SEQREPO_DIR", env_value) + + resp = client.get("/api/v1/refget/sequence/service-info") + assert resp.status_code == 200 + data = resp.json() + assert data["name"] == "MaveDB API" + assert data["seqrepo_data_version"] == expected_data_version + assert data["refget"]["identifier_types"] == ["refseq", "ensembl"] + + @pytest.mark.parametrize("entry", TEST_SEQREPO_INITIAL_STATE) def test_get_metadata_success(client, entry): alias = list(entry.keys())[0] @@ -96,6 +116,16 @@ def test_get_sequence_only_end(client, entry): assert resp.text == metadata["seq"][:end] +def test_get_sequence_range_header_and_query_params_conflict(client): + resp = client.get( + f"/api/v1/refget/sequence/{VALID_ENSEMBL_IDENTIFIER}", + params={"start": 1}, + headers={"Range": "bytes=1-2"}, + ) + assert resp.status_code == 400 + assert "Cannot use both start/end query parameters and Range header" in resp.text + + def test_get_sequence_invalid_query_range_only_start_negative(client): resp = client.get(f"/api/v1/refget/sequence/{VALID_ENSEMBL_IDENTIFIER}", params={"start": -1}) assert resp.status_code == 416 diff --git a/tests/routers/test_seqrepo.py b/tests/routers/test_seqrepo.py index 130e1a847..c9163c072 100644 --- a/tests/routers/test_seqrepo.py +++ b/tests/routers/test_seqrepo.py @@ -3,6 +3,8 @@ import pytest +pytestmark = pytest.mark.unit + arq = pytest.importorskip("arq") cdot = pytest.importorskip("cdot") fastapi = pytest.importorskip("fastapi") From db2bb496ea4ee02bca3adff1afbedd4a8f8ca54a Mon Sep 17 00:00:00 2001 From: David Reinhart Date: Tue, 18 Aug 2026 10:17:59 -0700 Subject: [PATCH 111/252] fix(worker): clear stale uniprot_id_from_mapped_metadata on remap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The UniProt polling job only writes target_gene.uniprot_id_from_mapped_metadata on a successful lookup. When a remap produced no UniProt ID (no results, ambiguous results, or the target gene was not found), the value from the previous mapping run survived and became mismatched against the freshly written pre/post mapped metadata — a state unreachable for a new target_gene record. Clear the field alongside the other mapped metadata writes in map_variants_for_score_set, so it is reset before the downstream UniProt mapping jobs run. Closes #794 --- .../worker/jobs/variant_processing/mapping.py | 2 + .../jobs/variant_processing/test_mapping.py | 56 +++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/src/mavedb/worker/jobs/variant_processing/mapping.py b/src/mavedb/worker/jobs/variant_processing/mapping.py index efc8226b3..7762a2180 100644 --- a/src/mavedb/worker/jobs/variant_processing/mapping.py +++ b/src/mavedb/worker/jobs/variant_processing/mapping.py @@ -203,6 +203,8 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan target_gene.pre_mapped_metadata = cast(pre_mapped_metadata, JSONB) target_gene.post_mapped_metadata = cast(post_mapped_metadata, JSONB) + target_gene.uniprot_id_from_mapped_metadata = None + job_manager.db.add(target_gene) logger.debug("Added mapping metadata to target gene.", extra=job_manager.logging_context()) diff --git a/tests/worker/jobs/variant_processing/test_mapping.py b/tests/worker/jobs/variant_processing/test_mapping.py index dc2ac843e..bbeafe3af 100644 --- a/tests/worker/jobs/variant_processing/test_mapping.py +++ b/tests/worker/jobs/variant_processing/test_mapping.py @@ -333,6 +333,62 @@ async def dummy_mapping_job(): assert annotation_statuses[0].annotation_type == "vrs_mapping" assert annotation_statuses[0].status == "success" + async def test_map_variants_for_score_set_clears_stale_uniprot_id( + self, + session, + with_independent_processing_runs, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_score_set, + ): + """A remap must clear any UniProt ID left over from a prior mapping run. + + The downstream UniProt job only writes on success, so a stale value would otherwise + persist and be mismatched against the newly mapped metadata. + """ + + async def dummy_mapping_job(): + return await construct_mock_mapping_output( + session=session, + score_set=sample_score_set, + with_gene_info=True, + with_layers={"g", "c", "p"}, + with_pre_mapped=True, + with_post_mapped=True, + with_reference_metadata=True, + with_mapped_scores=True, + with_all_variants=True, + ) + + variant = Variant( + score_set_id=sample_score_set.id, hgvs_nt="NM_000000.1:c.1A>G", hgvs_pro="NP_000000.1:p.Met1Val", data={} + ) + session.add(variant) + + for target in sample_score_set.target_genes: + target.uniprot_id_from_mapped_metadata = "P00000" + session.add(target) + session.commit() + + with ( + patch.object( + _UnixSelectorEventLoop, + "run_in_executor", + return_value=dummy_mapping_job(), + ), + ): + result = await map_variants_for_score_set( + mock_worker_ctx, + sample_independent_variant_mapping_run.id, + JobManager(session, mock_worker_ctx["redis"], sample_independent_variant_mapping_run.id), + ) + + assert isinstance(result, JobExecutionOutcome) + assert result.status == JobStatus.SUCCEEDED + + for target in sample_score_set.target_genes: + assert target.uniprot_id_from_mapped_metadata is None + @pytest.mark.parametrize( "with_layers", [ From 79b3a6b6682747e10e0e1d93ed89198750d490be Mon Sep 17 00:00:00 2001 From: David Reinhart Date: Wed, 19 Aug 2026 11:50:21 -0700 Subject: [PATCH 112/252] Clear stale HGNC gene info on remap when mapping returns none The mapping job only wrote mapped_hgnc_name when gene info was present, so a target gene remapped without gene info kept values from a prior run. Explicitly reset to None in the else branch. --- src/mavedb/worker/jobs/variant_processing/mapping.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/mavedb/worker/jobs/variant_processing/mapping.py b/src/mavedb/worker/jobs/variant_processing/mapping.py index 7762a2180..be2fde455 100644 --- a/src/mavedb/worker/jobs/variant_processing/mapping.py +++ b/src/mavedb/worker/jobs/variant_processing/mapping.py @@ -173,6 +173,9 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan job_manager.save_to_context({"mapped_hgnc_name": target_gene.mapped_hgnc_name}) logger.debug("Added mapped HGNC name to target gene.", extra=job_manager.logging_context()) + else: + target_gene.mapped_hgnc_name = None + logger.debug("No gene-level info found for target gene.", extra=job_manager.logging_context()) # add annotation layer info for annotation_layer in reference_metadata[target_gene_identifier]["layers"]: From bef917a5cc7ae8d06d489e799cf27ead368a464e Mon Sep 17 00:00:00 2001 From: David Reinhart Date: Wed, 19 Aug 2026 20:35:45 -0700 Subject: [PATCH 113/252] test(worker): cover stale mapped_hgnc_name clearing on remap Add a unit test for the fix in 79b3a6b6. The test seeds mapped_hgnc_name on every target gene, runs map_variants_for_score_set against mapping output with no gene_info, and asserts the field is reset to None. The existing missing-gene-info test asserted the same final state but never seeded a prior value, so it passed before the fix. This one fails without the else branch. --- .../jobs/variant_processing/test_mapping.py | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/tests/worker/jobs/variant_processing/test_mapping.py b/tests/worker/jobs/variant_processing/test_mapping.py index bbeafe3af..5c513d7e8 100644 --- a/tests/worker/jobs/variant_processing/test_mapping.py +++ b/tests/worker/jobs/variant_processing/test_mapping.py @@ -389,6 +389,62 @@ async def dummy_mapping_job(): for target in sample_score_set.target_genes: assert target.uniprot_id_from_mapped_metadata is None + async def test_map_variants_for_score_set_clears_stale_mapped_hgnc_name( + self, + session, + with_independent_processing_runs, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_score_set, + ): + """A remap must clear any mapped HGNC name left over from a prior mapping run. + + Gene-level info is only written when the mapping results include it, so a stale value + would otherwise persist and be mismatched against the newly mapped metadata. + """ + + async def dummy_mapping_job(): + return await construct_mock_mapping_output( + session=session, + score_set=sample_score_set, + with_gene_info=False, + with_layers={"g", "c", "p"}, + with_pre_mapped=True, + with_post_mapped=True, + with_reference_metadata=True, + with_mapped_scores=True, + with_all_variants=True, + ) + + variant = Variant( + score_set_id=sample_score_set.id, hgvs_nt="NM_000000.1:c.1A>G", hgvs_pro="NP_000000.1:p.Met1Val", data={} + ) + session.add(variant) + + for target in sample_score_set.target_genes: + target.mapped_hgnc_name = "STALE1" + session.add(target) + session.commit() + + with ( + patch.object( + _UnixSelectorEventLoop, + "run_in_executor", + return_value=dummy_mapping_job(), + ), + ): + result = await map_variants_for_score_set( + mock_worker_ctx, + sample_independent_variant_mapping_run.id, + JobManager(session, mock_worker_ctx["redis"], sample_independent_variant_mapping_run.id), + ) + + assert isinstance(result, JobExecutionOutcome) + assert result.status == JobStatus.SUCCEEDED + + for target in sample_score_set.target_genes: + assert target.mapped_hgnc_name is None + @pytest.mark.parametrize( "with_layers", [ From 0fd9a36b4e4339d3cff7217ecd139330f810ddce Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Fri, 21 Aug 2026 16:43:33 -0700 Subject: [PATCH 114/252] fix(permissions): require ownership to supersede a published score set fetch_score_set_by_urn's owner_or_contributor filter led with ScoreSet.private.is_(False), which only_published already guarantees. The two arguments constrained the same column and the looser subsumed the stricter, so passing user_data narrowed nothing: the call collapsed to an Action.READ check on a published score set, which every caller passes. Any authenticated user could record their own score set as the successor to any published one, and because replaces_id is unique, the first claim permanently blocked the real owner from superseding their own work. Drop the public-visibility disjunct so the argument means what its name and the docstring say. The other three call sites pass None and are unaffected. Supersession still has no Action of its own, so the authorization remains the shape of the query rather than a permission handler; the call site now says so. Contributors are admitted alongside owners, matching the documented rule -- broader than _handle_publish_action, which is owner-only, and worth settling when Action.SUPERSEDE lands. --- src/mavedb/routers/score_sets.py | 11 ++-- tests/routers/test_score_set.py | 94 ++++++++++++++++++++++++++++++++ 2 files changed, 101 insertions(+), 4 deletions(-) diff --git a/src/mavedb/routers/score_sets.py b/src/mavedb/routers/score_sets.py index 843db6f1b..49aec156c 100644 --- a/src/mavedb/routers/score_sets.py +++ b/src/mavedb/routers/score_sets.py @@ -582,17 +582,18 @@ async def fetch_score_set_by_urn( :param user: The user who has requested the score set. If the user does not have read permission, the score set will not be returned. If None, the score set is returned only if publicly visible. :param owner_or_contributor: If not None, require that the result be a score set of which this user is owner or - contributor. + contributor. This is an ownership requirement, not a visibility one: it does not admit score sets that are + merely public. Combining it with only_published therefore yields published score sets owned by this user. :param only_published: If true, only return the score set if it is published. - :return: The score set, or None if the URL was not found or refers to a private score set not owned by the specified - user. + :return: The score set. + :raises HTTPException: 404 if no score set matches the URN and the supplied filters, or 500 if more than one + does. Read permission is asserted on the result and raises through assert_permission. """ try: query = db.query(ScoreSet).filter(ScoreSet.urn == urn) if owner_or_contributor is not None: query = query.filter( or_( - ScoreSet.private.is_(False), ScoreSet.created_by_id == owner_or_contributor.user.id, ScoreSet.contributors.any(Contributor.orcid_id == owner_or_contributor.user.username), ) @@ -1763,6 +1764,8 @@ async def create_score_set( save_to_logging_context({"requested_superseded_score_set": item_create.superseded_score_set_urn}) if item_create.superseded_score_set_urn is not None: + # Passing user_data as owner_or_contributor is what authorizes the supersession: the fetch returns + # only published score sets this user owns or contributes to. There is no Action for supersession yet. superseded_score_set = await fetch_score_set_by_urn( db, item_create.superseded_score_set_urn, user_data, user_data, True ) diff --git a/tests/routers/test_score_set.py b/tests/routers/test_score_set.py index 26a45bb32..be95de75d 100644 --- a/tests/routers/test_score_set.py +++ b/tests/routers/test_score_set.py @@ -362,6 +362,100 @@ def test_cannot_create_score_set_with_invalid_target_gene_category(client, mock_ assert all(field in response_data["detail"][0]["msg"] for field in TargetCategory._member_names_) +######################################################################################################################## +# Score set supersession +######################################################################################################################## + + +def _publish_score_set_owned_by_extra_user(session, data_provider, client, data_files): + """Create and publish a score set, then reassign it to the extra user.""" + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") + + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published = publish_score_set(client, score_set["urn"]) + + change_ownership(session, published["urn"], ScoreSetDbModel) + return published + + +def test_owner_can_supersede_own_published_score_set(session, data_provider, client, setup_router_db, data_files): + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") + + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published = publish_score_set(client, score_set["urn"]) + + score_set_post_payload = deepcopy(TEST_MINIMAL_SEQ_SCORESET) + score_set_post_payload["experimentUrn"] = published["experiment"]["urn"] + score_set_post_payload["supersededScoreSetUrn"] = published["urn"] + + response = client.post("/api/v1/score-sets/", json=score_set_post_payload) + assert response.status_code == 200 + assert response.json()["supersededScoreSet"]["urn"] == published["urn"] + + +def test_cannot_supersede_other_users_published_score_set(session, data_provider, client, setup_router_db, data_files): + """A published score set may only be superseded by its owner or a contributor. + + Regression test: fetch_score_set_by_urn's owner_or_contributor filter previously admitted any + non-private score set, which only_published already guaranteed, so this call was unauthorized. + """ + published = _publish_score_set_owned_by_extra_user(session, data_provider, client, data_files) + + score_set_post_payload = deepcopy(TEST_MINIMAL_SEQ_SCORESET) + score_set_post_payload["experimentUrn"] = published["experiment"]["urn"] + score_set_post_payload["supersededScoreSetUrn"] = published["urn"] + + response = client.post("/api/v1/score-sets/", json=score_set_post_payload) + assert response.status_code == 404 + assert published["urn"] in response.json()["detail"] + + +def test_cannot_lock_owner_out_of_superseding_their_own_score_set( + session, data_provider, client, setup_router_db, data_files +): + """Supersession is one-shot, so an unauthorized claim would permanently block the real owner.""" + published = _publish_score_set_owned_by_extra_user(session, data_provider, client, data_files) + + score_set_post_payload = deepcopy(TEST_MINIMAL_SEQ_SCORESET) + score_set_post_payload["experimentUrn"] = published["experiment"]["urn"] + score_set_post_payload["supersededScoreSetUrn"] = published["urn"] + assert client.post("/api/v1/score-sets/", json=score_set_post_payload).status_code == 404 + + # The owner's own supersession must still be available afterwards. + response = client.get(f"/api/v1/score-sets/{published['urn']}") + assert response.status_code == 200 + assert response.json().get("supersedingScoreSet") is None + + +def test_contributor_can_supersede_score_set( + session, data_provider, client, setup_router_db, data_files, extra_user_app_overrides +): + """A contributor to a published score set may record its successor, as well as its owner.""" + experiment = create_experiment(client, {"contributors": [{"orcidId": EXTRA_USER["username"]}]}) + score_set = create_seq_score_set( + client, experiment["urn"], update={"contributors": [{"orcidId": EXTRA_USER["username"]}]} + ) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") + + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published = publish_score_set(client, score_set["urn"]) + + score_set_post_payload = deepcopy(TEST_MINIMAL_SEQ_SCORESET) + score_set_post_payload["experimentUrn"] = published["experiment"]["urn"] + score_set_post_payload["supersededScoreSetUrn"] = published["urn"] + + # The extra user contributes to the score set but does not own it. + with DependencyOverrider(extra_user_app_overrides): + response = client.post("/api/v1/score-sets/", json=score_set_post_payload) + + assert response.status_code == 200 + assert response.json()["supersededScoreSet"]["urn"] == published["urn"] + + ######################################################################################################################## # Score set updating ######################################################################################################################## From 242fadae545015dc80928aba1d7f8acc1b1a326f Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Fri, 21 Aug 2026 16:47:50 -0700 Subject: [PATCH 115/252] fix(permissions): serve only published experiments from the public search /experiments/search has no auth dependency, and build_search_experiments_query_filter narrows by owner or contributor rather than by visibility -- the router passes None. Unlike /score-sets/search it never constrained published, which defaults to None and applies no filter at all, so an empty request body returned every unpublished experiment in the database, with its title, abstract, method text and owner ORCID iD. Force published to true and reject an explicit false, mirroring the score set endpoint. Unpublished experiments stay reachable through /me/experiments/search, which is unchanged. The UI only ever called that route, so no client change is required. Three tests asserted the old behaviour, one of them checking that another user's given name came back. The replacements assert against one published and one unpublished experiment matching the same search text, so they fail whether the filter is missing or too aggressive. --- src/mavedb/routers/experiments.py | 10 ++++ tests/routers/test_experiments.py | 93 ++++++++++++++++++++++++++++--- 2 files changed, 95 insertions(+), 8 deletions(-) diff --git a/src/mavedb/routers/experiments.py b/src/mavedb/routers/experiments.py index debaebcb0..b78ca27d0 100644 --- a/src/mavedb/routers/experiments.py +++ b/src/mavedb/routers/experiments.py @@ -112,6 +112,16 @@ def search_experiments(search: ExperimentsSearch, db: Session = Depends(deps.get """ Search experiments. """ + # This endpoint is unauthenticated, so it serves published experiments only. `build_search_experiments_query_filter` + # narrows by owner or contributor, not by visibility, and receives None here; without this the search would return + # every unpublished experiment in the database. Private experiments are reached through /me/experiments/search. + if search.published is False: + raise HTTPException( + status_code=422, + detail="Cannot search for private experiments except in the context of the current user's data.", + ) + search.published = True + items = _search_experiments(db, None, search) return [enrich_experiment_with_num_score_sets(exp, None) for exp in items] diff --git a/tests/routers/test_experiments.py b/tests/routers/test_experiments.py index 60d552140..acd949392 100644 --- a/tests/routers/test_experiments.py +++ b/tests/routers/test_experiments.py @@ -1574,14 +1574,61 @@ def test_users_get_one_score_set_from_own_experiment_with_a_superseding_score_se assert pub_score_set["urn"] not in response_data["scoreSetUrns"] -def test_search_experiments(session, client, setup_router_db): - experiment = create_experiment(client) +def _publish_experiment(session, data_provider, client, data_files, update=None): + """Publish an experiment, and return it. + + Publishing a score set is the only path that publishes its experiment, so an experiment cannot be + published without one. + """ + experiment = create_experiment(client, update) + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") + + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published_score_set = publish_score_set(client, score_set["urn"]) + + return published_score_set["experiment"] + + +def test_search_experiments(session, data_provider, client, setup_router_db, data_files): + experiment = _publish_experiment(session, data_provider, client, data_files) search_payload = {"text": experiment["shortDescription"]} response = client.post("/api/v1/experiments/search", json=search_payload) assert response.status_code == 200 assert response.json()[0]["title"] == experiment["title"] +def test_search_experiments_excludes_unpublished(session, data_provider, client, setup_router_db, data_files): + """The public search endpoint serves published experiments only. + + Both experiments match the search text, so this fails whether the visibility filter is too permissive + or too restrictive. Asserting only that an unpublished experiment is absent would also pass if the + search returned nothing at all. + """ + published = _publish_experiment( + session, data_provider, client, data_files, update={"title": "Published Experiment"} + ) + unpublished = create_experiment(client, update={"title": "Unpublished Experiment"}) + + search_payload = {"text": TEST_MINIMAL_EXPERIMENT["shortDescription"]} + response = client.post("/api/v1/experiments/search", json=search_payload) + + assert response.status_code == 200 + returned_urns = [item["urn"] for item in response.json()] + assert published["urn"] in returned_urns + assert unpublished["urn"] not in returned_urns + + +def test_search_experiments_rejects_explicit_unpublished_search(session, client, setup_router_db): + """Unpublished experiments are reached through /me/experiments/search, never this endpoint.""" + response = client.post("/api/v1/experiments/search", json={"published": False}) + assert response.status_code == 422 + assert ( + response.json()["detail"] + == "Cannot search for private experiments except in the context of the current user's data." + ) + + def test_search_my_experiments(session, client, setup_router_db): experiment = create_experiment(client) search_payload = {"text": experiment["shortDescription"]} @@ -1934,8 +1981,8 @@ def test_search_score_sets_for_my_experiments(session, client, setup_router_db, ) -def test_search_their_experiments(session, client, setup_router_db): - experiment = create_experiment(client) +def test_search_their_experiments(session, data_provider, client, setup_router_db, data_files): + experiment = _publish_experiment(session, data_provider, client, data_files) change_ownership(session, experiment["urn"], ExperimentDbModel) change_ownership(session, experiment["experimentSetUrn"], ExperimentSetDbModel) search_payload = {"text": experiment["shortDescription"]} @@ -1945,6 +1992,22 @@ def test_search_their_experiments(session, client, setup_router_db): assert response.json()[0]["createdBy"]["firstName"] == EXTRA_USER["first_name"] +def test_cannot_search_their_unpublished_experiments(session, client, setup_router_db): + """Another user's unpublished experiment is not disclosed by the public search endpoint. + + Regression test: build_search_experiments_query_filter narrows by owner or contributor and receives + None from this endpoint, so before the visibility filter was added this returned the experiment along + with its owner's name and ORCID iD. + """ + experiment = create_experiment(client) + change_ownership(session, experiment["urn"], ExperimentDbModel) + change_ownership(session, experiment["experimentSetUrn"], ExperimentSetDbModel) + search_payload = {"text": experiment["shortDescription"]} + response = client.post("/api/v1/experiments/search", json=search_payload) + assert response.status_code == 200 + assert experiment["urn"] not in [item["urn"] for item in response.json()] + + def test_search_not_my_experiments(session, client, setup_router_db): experiment = create_experiment(client) change_ownership(session, experiment["urn"], ExperimentDbModel) @@ -1955,13 +2018,27 @@ def test_search_not_my_experiments(session, client, setup_router_db): assert len(response.json()) == 0 -def test_anonymous_search_experiments(session, client, anonymous_app_overrides, setup_router_db): - experiment = create_experiment(client) - search_payload = {"text": experiment["shortDescription"]} +def test_anonymous_search_experiments( + session, data_provider, client, anonymous_app_overrides, setup_router_db, data_files +): + """An anonymous caller sees published experiments, and only those. + + Both experiments match the search text, so this fails whether the visibility filter is too permissive + or too restrictive. + """ + published = _publish_experiment( + session, data_provider, client, data_files, update={"title": "Published Experiment"} + ) + unpublished = create_experiment(client, update={"title": "Unpublished Experiment"}) + + search_payload = {"text": TEST_MINIMAL_EXPERIMENT["shortDescription"]} with DependencyOverrider(anonymous_app_overrides): response = client.post("/api/v1/experiments/search", json=search_payload) + assert response.status_code == 200 - assert response.json()[0]["title"] == experiment["title"] + returned_urns = [item["urn"] for item in response.json()] + assert published["urn"] in returned_urns + assert unpublished["urn"] not in returned_urns def test_anonymous_cannot_search_my_experiments(session, client, anonymous_app_overrides, setup_router_db): From 747d61da6e3e02af0d33b2d0a70980c1419c7100 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Fri, 21 Aug 2026 16:51:19 -0700 Subject: [PATCH 116/252] fix(permissions): always filter an experiment's score set URNs in search results enrich_experiment_with_num_score_sets is what filters a nested experiment's score_set_urns by permission, and /score-sets/search ran it only when the request left includeExperimentScoreSetUrnsAndCount unset or true. Sent false, the ORM experiment reached SavedExperiment.generate_score_set_urn_list directly, which lists every score set on the experiment, disclosing the tmp URNs of private ones and their count. SearchView sends false, so this was the served behaviour of the main search page rather than an opt-in path. Remove the field rather than defaulting it safe: the response cannot be correct with the filter off, so it should not be expressible. The view models do not forbid extra fields, so a client still sending it is unaffected. The enrichment is now unconditional on that route, which costs it roughly 1.3ms per result, or about 130ms at the 100-result limit SearchView requests. Building ShortScoreSet rather than ScoreSet in the loop recovers most of that, since the response model narrows to ShortScoreSet anyway; held back as a separate change. --- src/mavedb/routers/score_sets.py | 18 ++++++++------ src/mavedb/view_models/search.py | 1 - tests/routers/test_score_set.py | 42 ++++++++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 9 deletions(-) diff --git a/src/mavedb/routers/score_sets.py b/src/mavedb/routers/score_sets.py index 49aec156c..aff713429 100644 --- a/src/mavedb/routers/score_sets.py +++ b/src/mavedb/routers/score_sets.py @@ -735,15 +735,17 @@ def search_score_sets( ) score_sets, num_score_sets = _search_score_sets(db, None, search).values() + + # Unconditional, because this enrichment is also what filters the nested experiment's score set URNs by + # permission. Serializing the ORM experiment directly instead reaches SavedExperiment's score_set_urns + # validator, which lists every score set on the experiment, disclosing the URNs of private ones. enriched_score_sets = [] - if search.include_experiment_score_set_urns_and_count: - for ss in score_sets: - enriched_experiment = enrich_experiment_with_num_score_sets(ss.experiment, user_data) - response_item = score_set.ScoreSet.model_validate(ss).copy(update={"experiment": enriched_experiment}) - enriched_score_sets.append(response_item) - score_sets = enriched_score_sets - - return {"score_sets": score_sets, "num_score_sets": num_score_sets} + for ss in score_sets: + enriched_experiment = enrich_experiment_with_num_score_sets(ss.experiment, user_data) + response_item = score_set.ScoreSet.model_validate(ss).copy(update={"experiment": enriched_experiment}) + enriched_score_sets.append(response_item) + + return {"score_sets": enriched_score_sets, "num_score_sets": num_score_sets} @router.post("/score-sets/search/filter-options", status_code=200, response_model=ScoreSetsSearchFilterOptionsResponse) diff --git a/src/mavedb/view_models/search.py b/src/mavedb/view_models/search.py index 778b3adb8..229595996 100644 --- a/src/mavedb/view_models/search.py +++ b/src/mavedb/view_models/search.py @@ -32,7 +32,6 @@ class ScoreSetsSearch(BaseModel): publication_identifiers: Optional[list[str]] = None keywords: Optional[list[ControlledKeywordSearch]] = None text: Optional[str] = None - include_experiment_score_set_urns_and_count: Optional[bool] = True offset: Optional[int] = None limit: Optional[int] = None diff --git a/tests/routers/test_score_set.py b/tests/routers/test_score_set.py index be95de75d..7d9290a9a 100644 --- a/tests/routers/test_score_set.py +++ b/tests/routers/test_score_set.py @@ -2613,6 +2613,48 @@ def test_search_public_score_sets_no_match(session, data_provider, client, setup assert len(response.json()["scoreSets"]) == 0 +def test_search_public_score_sets_does_not_disclose_private_sibling_urns( + session, data_provider, client, anonymous_app_overrides, setup_router_db, data_files +): + """A search result's experiment lists only the score set URNs the caller may read. + + Regression test: the enrichment that filters those URNs used to be skipped when the request set + includeExperimentScoreSetUrnsAndCount to false, and SavedExperiment's validator then listed every score + set on the experiment. The field is gone, and an unknown field is ignored rather than rejected, so the + old request shape must now be filtered too. + """ + experiment = create_experiment(client, {"title": "Experiment 1"}) + published = create_seq_score_set(client, experiment["urn"], update={"title": "Test Fnord Score Set"}) + published = mock_worker_variant_insertion(client, session, data_provider, published, data_files / "scores.csv") + + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published = publish_score_set(client, published["urn"]) + + # Unpublished, and inside the now-public experiment. This is the URN that leaked. + private = create_seq_score_set( + client, published["experiment"]["urn"], update={"title": "Unpublished Fnord Score Set"} + ) + + # The owner may read both, so the enrichment is permission-scoped rather than a blanket strip. + response = client.post("/api/v1/score-sets/search", json={"text": "fnord"}) + assert response.status_code == 200 + assert set(response.json()["scoreSets"][0]["experiment"]["scoreSetUrns"]) == {published["urn"], private["urn"]} + + for search_payload in ( + {"text": "fnord"}, + {"text": "fnord", "includeExperimentScoreSetUrnsAndCount": False}, + ): + with DependencyOverrider(anonymous_app_overrides): + response = client.post("/api/v1/score-sets/search", json=search_payload) + + assert response.status_code == 200 + assert len(response.json()["scoreSets"]) == 1 + + score_set_urns = response.json()["scoreSets"][0]["experiment"]["scoreSetUrns"] + assert published["urn"] in score_set_urns + assert private["urn"] not in score_set_urns + + def test_search_public_score_sets_match(session, data_provider, client, setup_router_db, data_files): experiment = create_experiment(client, {"title": "Experiment 1"}) score_set = create_seq_score_set(client, experiment["urn"], update={"title": "Test Fnord Score Set"}) From f597036a6f0b99dafd580f9a8b6c2b89b6a0a7c0 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Fri, 21 Aug 2026 16:56:30 -0700 Subject: [PATCH 117/252] fix(permissions): exclude unpublished score sets from target gene listings /target-genes/names and /target-genes/categories are unauthenticated and aggregate the whole target_genes table with no join to scoresets, so they disclosed the gene symbols and categories under assay in unpublished work. Neither route names an entity to assert a permission on, and both sibling routes in the module already filter. Join the parent score set and require a published date -- the same predicate the statistics routes use through _join_model_and_filter_unpublished. Keeping these two on that shape means the RLS work retires them together rather than one at a time. --- src/mavedb/routers/target_genes.py | 21 +++++++--- tests/routers/test_target_gene.py | 63 +++++++++++++++++++++++++++++- 2 files changed, 78 insertions(+), 6 deletions(-) diff --git a/src/mavedb/routers/target_genes.py b/src/mavedb/routers/target_genes.py index a304ee892..67513578c 100644 --- a/src/mavedb/routers/target_genes.py +++ b/src/mavedb/routers/target_genes.py @@ -1,7 +1,7 @@ from typing import Any, List, Optional from fastapi import APIRouter, Depends, HTTPException -from sqlalchemy.orm import Session, selectinload +from sqlalchemy.orm import Query, Session, selectinload from mavedb import deps from mavedb.lib.authentication import get_current_user @@ -83,16 +83,27 @@ def list_target_genes( return sorted(validated_items, key=lambda i: i.name) +def _published_target_genes(db: Session) -> Query[TargetGene]: + """ + Query the target genes belonging to published score sets. + + The two routes below aggregate over the whole table and are unauthenticated, so they have no entity to + assert a permission on. The parent score set's published state is what keeps the target names and + categories of unpublished work out of the response. + """ + return db.query(TargetGene).join(ScoreSet).filter(ScoreSet.published_date.is_not(None)) + + @router.get("/target-genes/names", status_code=200, response_model=List[str], summary="List target gene names") def list_target_gene_names( *, db: Session = Depends(deps.get_db), ) -> Any: """ - List distinct target gene names, in alphabetical order. + List distinct target gene names from published score sets, in alphabetical order. """ - items = db.query(TargetGene).all() + items = _published_target_genes(db).all() names = map(lambda item: item.name, items) return sorted(list(set(names))) @@ -105,10 +116,10 @@ def list_target_gene_categories( db: Session = Depends(deps.get_db), ) -> Any: """ - List distinct target genes categories, in alphabetical order. + List distinct target gene categories from published score sets, in alphabetical order. """ - items = db.query(TargetGene).all() + items = _published_target_genes(db).all() categories = map(lambda item: item.category, items) return sorted(list(set(categories))) diff --git a/tests/routers/test_target_gene.py b/tests/routers/test_target_gene.py index 5ca1b4a2c..63887cfb9 100644 --- a/tests/routers/test_target_gene.py +++ b/tests/routers/test_target_gene.py @@ -1,5 +1,6 @@ # ruff: noqa: E402 import pytest +from copy import deepcopy from unittest.mock import patch arq = pytest.importorskip("arq") @@ -8,7 +9,8 @@ from mavedb.models.score_set import ScoreSet as ScoreSetDbModel -from tests.helpers.constants import TEST_USER +from tests.helpers.constants import TEST_MINIMAL_SEQ_SCORESET, TEST_USER +from tests.helpers.dependency_overrider import DependencyOverrider from tests.helpers.util.contributor import add_contributor from tests.helpers.util.experiment import create_experiment from tests.helpers.util.user import change_ownership @@ -165,3 +167,62 @@ def test_fetch_public_target_gene_by_id(session, data_provider, client, setup_ro response = client.get("/api/v1/target-genes/1") assert response.status_code == 200 assert response.json()["scoreSetUrn"] == published_score_set["urn"] + + +def _score_set_with_target(client, experiment_urn, name, category): + """Create a score set whose single target gene carries the given name and category.""" + target_genes = deepcopy(TEST_MINIMAL_SEQ_SCORESET["targetGenes"]) + target_genes[0]["name"] = name + target_genes[0]["category"] = category + return create_seq_score_set(client, experiment_urn, update={"targetGenes": target_genes}) + + +def _published_and_unpublished_targets(session, data_provider, client, data_files): + """Put one published and one unpublished score set, with distinct target genes, in one experiment. + + The unpublished score set is created after the publish, against the experiment's published URN, so that + it sits inside a public experiment. That is the arrangement in which its target gene leaks. + """ + experiment = create_experiment(client, {"title": "Experiment 1"}) + published = _score_set_with_target(client, experiment["urn"], "PUBLISHEDGENE", "protein_coding") + published = mock_worker_variant_insertion(client, session, data_provider, published, data_files / "scores.csv") + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published = publish_score_set(client, published["urn"]) + + _score_set_with_target(client, published["experiment"]["urn"], "UNPUBLISHEDGENE", "regulatory") + + +def test_list_target_gene_names_excludes_unpublished(session, data_provider, client, setup_router_db, data_files): + """Target gene names from unpublished score sets are not disclosed by this unauthenticated route. + + Asserted against one published and one unpublished score set so that the test fails whether the filter + is missing or too aggressive. + """ + _published_and_unpublished_targets(session, data_provider, client, data_files) + + response = client.get("/api/v1/target-genes/names") + assert response.status_code == 200 + assert "PUBLISHEDGENE" in response.json() + assert "UNPUBLISHEDGENE" not in response.json() + + +def test_list_target_gene_categories_excludes_unpublished(session, data_provider, client, setup_router_db, data_files): + _published_and_unpublished_targets(session, data_provider, client, data_files) + + response = client.get("/api/v1/target-genes/categories") + assert response.status_code == 200 + assert "protein_coding" in response.json() + assert "regulatory" not in response.json() + + +def test_anonymous_list_target_gene_names_excludes_unpublished( + session, data_provider, client, anonymous_app_overrides, setup_router_db, data_files +): + _published_and_unpublished_targets(session, data_provider, client, data_files) + + with DependencyOverrider(anonymous_app_overrides): + response = client.get("/api/v1/target-genes/names") + + assert response.status_code == 200 + assert "PUBLISHEDGENE" in response.json() + assert "UNPUBLISHEDGENE" not in response.json() From 1ecd9457c8dd2ff2194182259235fa62149a3854 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Fri, 21 Aug 2026 16:58:00 -0700 Subject: [PATCH 118/252] chore: bump version to 2026.2.7.2 --- pyproject.toml | 2 +- src/mavedb/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f8d0de49c..44135b9ec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "mavedb" -version = "2026.2.7.1" +version = "2026.2.7.2" description = "API for MaveDB, the database of Multiplexed Assays of Variant Effect." license = "AGPL-3.0-only" readme = "README.md" diff --git a/src/mavedb/__init__.py b/src/mavedb/__init__.py index a24dc3167..9e6e7566c 100644 --- a/src/mavedb/__init__.py +++ b/src/mavedb/__init__.py @@ -6,7 +6,7 @@ logger = module_logging.getLogger(__name__) __project__ = "mavedb-api" -__version__ = "2026.2.7.1" +__version__ = "2026.2.7.2" logger.info(f"MaveDB {__version__}") From ed9b923de7fcd4238ea81962319fe3dd1666d2ed Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 26 Aug 2026 09:52:25 -0700 Subject: [PATCH 119/252] fix(alleles): recompute VRS identity from content on ingest and backfill vrs_digest asserts that an allele's identifier is the GA4GH digest of its own post_mapped content, but nothing enforced it: writers adopted the id minted by the producer, which for mapped variants is computed before normalization. Because vrs_digest is a UNIQUE dedup key on an immutable, ValidTime-less table, a wrong value there cannot be corrected in place afterwards. Add canonical_variation_document() to re-derive identity and a single canonical serialization for any variation arriving from outside, and route both the live mapping ingest and the mapped_variants backfill migration through it so identity is recomputed rather than inherited. Add the audit_allele_identifiers script to measure drift before and after the backfill and to repair safely-correctable rows. --- ...ate_mapped_variants_to_allele_substrate.py | 60 +++- src/mavedb/lib/vrs_utils.py | 70 ++++- .../scripts/audit_allele_identifiers.py | 258 ++++++++++++++++++ .../worker/jobs/variant_processing/mapping.py | 12 +- tests/lib/test_vrs_utils.py | 163 +++++++++++ .../test_reverse_translation.py | 27 +- 6 files changed, 560 insertions(+), 30 deletions(-) create mode 100644 src/mavedb/scripts/audit_allele_identifiers.py diff --git a/alembic/manual_migrations/migrate_mapped_variants_to_allele_substrate.py b/alembic/manual_migrations/migrate_mapped_variants_to_allele_substrate.py index 32c752a6b..68db9d1d6 100644 --- a/alembic/manual_migrations/migrate_mapped_variants_to_allele_substrate.py +++ b/alembic/manual_migrations/migrate_mapped_variants_to_allele_substrate.py @@ -9,9 +9,9 @@ Variant ─< MappingRecord (1 live/variant, ValidTime) ─< MappingRecordAllele (ValidTime) >─ Allele (dedup by vrs_digest) -This script populates the new tables from the *existing* ``mapped_variants`` data so -the read-cutover (the ``v_variant_annotations`` view / ``published_variants`` MV -rewrites and the serving readers) resolves for historical score sets, which are +This script populates the new tables from the *existing* ``mapped_variants`` data so +the read-cutover (the ``v_variant_annotations`` view / ``published_variants`` MV +rewrites and the serving readers) resolves for historical score sets, which are otherwise empty in the new substrate. This is the **reshape half of the hybrid migration**: it reconstructs the @@ -131,7 +131,10 @@ from mavedb.models import * # noqa: F401,F403 pylint: disable=wildcard-import — register all mappers for configure_mappers() from mavedb.db.session import SessionLocal from mavedb.lib.annotation_status_manager import AnnotationStatusManager +from pydantic import ValidationError + from mavedb.lib.variants import get_hgvs_from_post_mapped +from mavedb.lib.vrs_utils import canonical_variation_document from mavedb.models.allele import Allele from mavedb.models.annotation_event import AnnotationEvent from mavedb.models.clinvar_allele_link import ClinvarAlleleLink @@ -397,9 +400,23 @@ def _authoritative_allele( flush, inside the same per-variant SAVEPOINT it always has — this only removes the redundant reads, not the write-time integrity check. """ - post_mapped = mv.post_mapped or {} - digest = post_mapped.get("id") - if not digest: + stored = mv.post_mapped or {} + if not stored.get("id"): + return None + + # Recompute identity rather than adopt the stored id. `vrs_digest` is the UNIQUE dedup key the + # whole allele graph hangs off, `alleles` rows are immutable once written, and `alleles` is the + # only table here with no ValidTime - so a wrong digest cannot be corrected in place or + # reconstructed by `as_of`, and it silently collapses two different alleles into one row. + # Historical `post_mapped` blobs cannot be trusted on this point: an id minted before VRS + # normalization moved a del/dup span survives unchanged, because `ga4gh_identify`'s default + # `in_place` returns a non-empty id untouched. The live mapping job canonicalizes for the same + # reason (`worker/jobs/variant_processing/mapping.py`); a backfill that trusted the blob would + # write drift into the substrate permanently. See `scripts/audit_allele_identifiers.py`. + try: + post_mapped, digest = canonical_variation_document(stored, subject=f"mapped_variant {mv.id}") + except (ValidationError, ValueError): + logger.exception(f"Could not canonicalize post_mapped for mapped_variant {mv.id}; skipping its allele.") return None existing = allele_cache.get(digest) @@ -564,7 +581,11 @@ def _emit_vep_consequence( if already is not None: # idempotent re-run return source_version = _resolve_vep_source_version(access_date) - stats["vep_source_version_legacy_fallback" if source_version == LEGACY_VEP_SOURCE_VERSION else "vep_source_version_resolved"] += 1 + stats[ + "vep_source_version_legacy_fallback" + if source_version == LEGACY_VEP_SOURCE_VERSION + else "vep_source_version_resolved" + ] += 1 db.add( VepAlleleConsequence( allele_id=allele_id, @@ -835,7 +856,9 @@ def _migrate_variant( # (mv, record, allele-or-None, hgvs_assay_level, valid_from, valid_to) per window actually created # this pass — resolved to real ids by one flush below, then walked again to emit events and collect # touched allele ids. - pending: list[tuple[MappedVariant, MappingRecord, Optional[Allele], Optional[str], datetime, Optional[datetime]]] = [] + pending: list[ + tuple[MappedVariant, MappingRecord, Optional[Allele], Optional[str], datetime, Optional[datetime]] + ] = [] for mv, valid_from, valid_to in result.windows: if (variant_id, mv.mapped_date) in existing_records: @@ -861,10 +884,23 @@ def _migrate_variant( elif hgvs_source == "vrs_expression": stats["hgvs_recovered_from_vrs_expression"] += 1 + # Same treatment for the pre-mapped representation. MappingRecord.vrs_digest is not a dedup + # key, so a wrong value here is less destructive than on Allele - but it is still an identity + # claim about content this migration is republishing, and the drift has the same cause. + pre_mapped_document, pre_mapped_digest = None, None + if (mv.pre_mapped or {}).get("id"): + try: + pre_mapped_document, pre_mapped_digest = canonical_variation_document( + mv.pre_mapped, subject=f"mapped_variant {mv.id} pre_mapped" + ) + except (ValidationError, ValueError): + logger.exception(f"Could not canonicalize pre_mapped for mapped_variant {mv.id}; storing it unkeyed.") + pre_mapped_document = mv.pre_mapped + record = MappingRecord( variant_id=variant_id, - vrs_digest=(mv.pre_mapped or {}).get("id"), - pre_mapped=mv.pre_mapped or None, + vrs_digest=pre_mapped_digest, + pre_mapped=pre_mapped_document, assay_level=level, hgvs_assay_level=hgvs_assay_level, mapped_date=mv.mapped_date, @@ -1077,9 +1113,7 @@ def do_migration( ) ).all() ) - candidate_digests = { - (mv.post_mapped or {}).get("id") for mvs in mvs_by_variant.values() for mv in mvs - } + candidate_digests = {(mv.post_mapped or {}).get("id") for mvs in mvs_by_variant.values() for mv in mvs} candidate_digests.discard(None) allele_cache: dict[str, Allele] = {} if candidate_digests: diff --git a/src/mavedb/lib/vrs_utils.py b/src/mavedb/lib/vrs_utils.py index 7979cdaec..af4455140 100644 --- a/src/mavedb/lib/vrs_utils.py +++ b/src/mavedb/lib/vrs_utils.py @@ -9,6 +9,8 @@ current content. """ +import logging +from collections.abc import Mapping from itertools import cycle from typing import Any @@ -27,6 +29,10 @@ from ga4gh.vrs.normalize import normalize from mavedb.lib.hgvs import split_cis_phased_hgvs +from mavedb.lib.logging.context import logging_context +from mavedb.lib.vrs import vrs_object_from_mapped_variant + +logger = logging.getLogger(__name__) # HGVS type letter (``accession:g.``) → VRS Expression syntax. _HGVS_SYNTAX_BY_TYPE = { @@ -133,9 +139,20 @@ def identify_allele(allele: Allele) -> str: carries one — e.g. stamped by an ``identify=True`` ``AlleleTranslator`` — would otherwise keep its stale id even with the digests cleared. Use in_place="always" to force it to be recomputed from the content-derived digest. + + The location needs identifying in its own right. ``ga4gh_identify`` writes only the + ``id`` of the object it is handed; for sub-objects it calls ``get_or_create_digest`` + and never ``get_or_create_ga4gh_identifier``. Clearing the location digest therefore + fixes what the *allele* id is derived from while leaving ``location.id`` holding + whatever was stamped before normalization — the allele id comes out correct and the + location it points at claims a digest belonging to the un-normalized span. Mirrors + ``dcd_mapping:vrs_utils.py::identify_allele``; both writers populate + ``alleles.post_mapped`` and have to agree byte-for-byte. """ if isinstance(allele.location, SequenceLocation): allele.location.digest = None + allele.location.id = None + ga4gh_identify(allele.location, in_place="always") allele.digest = None digest = ga4gh_identify(allele, in_place="always") @@ -150,17 +167,16 @@ def identify_variation(variation: Allele | CisPhasedBlock) -> str: Generalizes :func:`identify_allele` to cis-phased blocks. A block's Merkle digest is derived from its members' digests, so a stale member digest would silently propagate into - the block id. Clear every member (and its location) plus the block itself before - identifying so the id always reflects current content. + the block id. Re-identifying every member through :func:`identify_allele` clears the member + and its location and restamps both, so a block never embeds a member whose ``id`` disagrees + with its own coordinates. """ if isinstance(variation, Allele): return identify_allele(variation) for member in variation.members: if isinstance(member, Allele): - if isinstance(member.location, SequenceLocation): - member.location.digest = None - member.digest = None + member.id = identify_allele(member) variation.digest = None digest = ga4gh_identify(variation, in_place="always") @@ -219,3 +235,47 @@ def _rle_to_lse( c = cycle(subsequence) derived_sequence = "".join(next(c) for _ in range(rle.length)) return LiteralSequenceExpression(sequence=derived_sequence) + + +def canonical_variation_document(document: Mapping[str, Any], *, subject: str) -> tuple[dict[str, Any], str]: + """Re-derive identity and canonical serialization for a variation arriving from outside. + + An identifier computed elsewhere is an *assertion*, not an identity: it may have been minted by a + different ``ga4gh.vrs`` version, or — the case that actually bit us — before a normalization step + this process does not control. ``AlleleTranslator`` stamps ``id`` at translation time by default + (``identify=True``), and ``ga4gh_identify``'s default ``in_place`` returns a non-empty ``id`` + unchanged, so a subsequent normalize-then-identify silently preserves the pre-normalization id. + Deletions and duplications are the variant classes whose normalized span differs, so they are the + ones that drift; a substitution normalizes to itself and its minted id stays correct by luck. + + Recomputing here makes the stored ``id`` a fact about the stored content rather than a claim + inherited from the producer, and returns one canonical serialization regardless of which writer + (or which library version) produced the input — so the same object stored twice is byte-identical. + + A disagreement is logged rather than raised: the incoming document is still usable, and refusing + the variant would lose a mapping over a repairable identifier. Silence is not an option either, + because a corrected identifier changes what the row deduplicates against. A disagreement in identifier + implies an upstream issue and should be investigated. + + :param document: the producer's serialized VRS Allele or CisPhasedBlock + :param subject: what this variation belongs to, for the warning message + :return: the canonical document and its GA4GH identifier + """ + variation = vrs_object_from_mapped_variant(dict(document)).root + if not isinstance(variation, Allele | CisPhasedBlock): + raise ValueError(f"Expected Allele or CisPhasedBlock for {subject}, got {type(variation).__name__}") + + identifier = identify_variation(variation) + + incoming = document.get("id") + if incoming and incoming != identifier: + logger.warning( + msg=( + f"VRS identifier mismatch for {subject}: producer supplied {incoming!r}, but the " + f"document's own content identifies as {identifier!r}. Storing the recomputed " + "identifier. This usually means the id was minted before normalization." + ), + extra=logging_context(), + ) + + return variation.model_dump(mode="json", exclude_none=True), identifier diff --git a/src/mavedb/scripts/audit_allele_identifiers.py b/src/mavedb/scripts/audit_allele_identifiers.py new file mode 100644 index 000000000..a05f366dc --- /dev/null +++ b/src/mavedb/scripts/audit_allele_identifiers.py @@ -0,0 +1,258 @@ +""" +Audit stored allele identifiers against their own content. + +Usage: +``` +python3 -m mavedb.scripts.audit_allele_identifiers audit +python3 -m mavedb.scripts.audit_allele_identifiers audit-locations +``` + +Every row in `alleles` asserts that `vrs_digest` is the GA4GH computed identifier of `post_mapped`. +Nothing enforced that until the mapping ingest path began recomputing identity on arrival, so rows +written earlier can carry an identifier minted before their own normalization. `audit` reports them, and `--repair` corrects the ones that can be corrected safely. + +Run this before and after +`alembic/manual_migrations/migrate_mapped_variants_to_allele_substrate.py`. That migration now +recomputes identity as it writes, so it will not import drift — but `vrs_digest` is a UNIQUE dedup key +on an immutable, ValidTime-less table, so a wrong value there cannot be corrected in place afterwards. +Measuring before and expecting zero after is how that is kept honest. + +`--repair` renames only: it rewrites `vrs_digest` where the corrected identifier is unused. Where the +corrected identifier already exists, the row is a duplicate of one that was always right, and merging +it means repointing five tables (`annotation_events`, `clinvar_allele_links`, `gnomad_allele_links`, +`mapping_record_alleles`, `vep_allele_consequences`), four of which carry a unique index including +`allele_id` and can therefore collide. That is deliberately not automated. A duplicate appearing now +means a writer got past the ingest guard in `worker/jobs/variant_processing/mapping.py`, and that wants +investigating rather than papering over. + +`audit-pre-mapped` covers `mapping_records.vrs_digest` against `pre_mapped`. Not a dedup key, so a wrong +value is less destructive, but it is the same identity claim with the same cause and the backfill +republishes it. + +`audit-locations` covers the same defect one level down, on the nested `SequenceLocation.id`, which +`audit` does not look at and which no repair here can fix. The two are separate because the causes and +the remedies are separate — see that command. + +Read-only by default. Safe to run against production. +""" + +import logging +from collections import Counter +from typing import Any, Optional + +from ga4gh.core import ga4gh_identify +from ga4gh.vrs.models import SequenceLocation + +import asyncclick as click +from sqlalchemy import select +from sqlalchemy.orm import Session + +from mavedb.lib.vrs import vrs_object_from_mapped_variant +from mavedb.lib.vrs_utils import identify_variation +from mavedb.models.allele import Allele +from mavedb.models.mapping_record import MappingRecord +from mavedb.scripts.environment import script_environment, with_database_session + +logger = logging.getLogger(__name__) + + +def recomputed_identifier(post_mapped: dict[str, Any]) -> Optional[str]: + """The identifier this document's own content implies, or ``None`` if it cannot be hydrated. + + Hydration goes through ``vrs_object_from_mapped_variant`` rather than the strict VRS models + because older rows carry fields the current models forbid (``type`` on ``Extension``, ``label`` on + ``SequenceReference``). Neither affects the digest — it is computed over coordinates and state — so + the shim's tolerance costs nothing here. + """ + try: + return identify_variation(vrs_object_from_mapped_variant(dict(post_mapped)).root) + except Exception: + logger.debug("Could not hydrate an allele for identification", exc_info=True) + return None + + +@script_environment.command() +@click.option("--limit", type=int, default=None, help="Examine at most this many alleles.") +@click.option("--repair", is_flag=True, help="Rewrite vrs_digest where the corrected identifier is unused.") +@with_database_session +def audit(db: Session, limit: Optional[int], repair: bool) -> None: + """Report — and with ``--repair``, rename — alleles whose identifier disagrees with their content.""" + query = select(Allele).where(Allele.post_mapped.isnot(None)).order_by(Allele.id) + if limit is not None: + query = query.limit(limit) + + known = {digest for (digest,) in db.execute(select(Allele.vrs_digest)).all()} + counts: Counter[str] = Counter() + renames: list[tuple[Allele, str]] = [] + duplicates: list[tuple[Allele, str]] = [] + + for allele in db.scalars(query): + counts["examined"] += 1 + computed = recomputed_identifier(allele.post_mapped) + + if computed is None: + counts["could not hydrate"] += 1 + continue + if computed == allele.vrs_digest: + counts["identifier matches content"] += 1 + continue + + counts["DRIFTED"] += 1 + # A corrected identifier that already exists means the allele is a duplicate of a row that was + # always right: the drifted key simply failed to dedup against it. Those merge; the rest rename. + (duplicates if computed in known else renames).append((allele, computed)) + + for label, count in counts.most_common(): + logger.info("%8d %s", count, label) + logger.info("%8d drifted, corrected identifier is unused (rename)", len(renames)) + logger.info("%8d drifted, corrected identifier already exists (duplicate)", len(duplicates)) + + if repair and renames: + _rename(db, renames) + elif renames: + logger.warning("Run with --repair to correct %d rename(s).", len(renames)) + + if duplicates: + logger.error( + "%d allele(s) duplicate a row that was always right. Not repaired here — merging repoints " + "five tables, four with a unique index on allele_id. Investigate how they were written.", + len(duplicates), + ) + for allele, computed in duplicates[:20]: + logger.error(" allele id=%s stored=%s content=%s", allele.id, allele.vrs_digest, computed) + + +def _rename(db: Session, renames: list[tuple["Allele", str]]) -> None: + """Rewrite ``vrs_digest`` in place for rows whose corrected identifier is unused. + + Safe because the corrected identifier is known unused at classification time and this runs in one + transaction: a concurrent writer taking it in between surfaces as a ``uq_alleles_vrs_digest`` + violation on flush rather than a silent overwrite. Only the key changes — ``post_mapped`` is left + exactly as stored, since the body was never in question and allele bodies are immutable. + """ + for allele, computed in renames: + logger.info("renaming allele id=%s %s -> %s", allele.id, allele.vrs_digest, computed) + allele.vrs_digest = computed + + db.flush() + logger.info("renamed %d allele(s)", len(renames)) + + +@script_environment.command() +@click.option("--limit", type=int, default=None, help="Examine at most this many mapping records.") +@with_database_session +def audit_pre_mapped(db: Session, limit: Optional[int]) -> None: + """Report mapping records whose ``vrs_digest`` disagrees with their ``pre_mapped`` content. + + Read-only, and no repair: ``mapping_records.vrs_digest`` is indexed but not unique, so a wrong value + does not collapse two records into one the way it does on ``alleles``. It is still an identity claim + about content the backfill republishes, and it has the same cause — an id minted before VRS + normalization moved a del/dup span. The pre-mapped side had never been measured before 2026-08-25. + """ + query = select(MappingRecord).where(MappingRecord.pre_mapped.isnot(None)).order_by(MappingRecord.id) + if limit is not None: + query = query.limit(limit) + + counts: Counter[str] = Counter() + drifted: list[tuple[int, Optional[str], str]] = [] + + for record in db.scalars(query): + counts["examined"] += 1 + computed = recomputed_identifier(record.pre_mapped) + + if computed is None: + counts["could not hydrate"] += 1 + elif computed == record.vrs_digest: + counts["identifier matches content"] += 1 + elif record.vrs_digest is None: + counts["no stored identifier"] += 1 + else: + counts["DRIFTED"] += 1 + drifted.append((record.id, record.vrs_digest, computed)) + + for label, count in counts.most_common(): + logger.info("%8d %s", count, label) + + for record_id, stored, computed in drifted[:20]: + logger.warning(" mapping_record id=%s stored=%s content=%s", record_id, stored, computed) + if len(drifted) > 20: + logger.warning(" ... and %d more", len(drifted) - 20) + + +@script_environment.command() +@click.option("--limit", type=int, default=None, help="Examine at most this many alleles.") +@with_database_session +def audit_locations(db: Session, limit: Optional[int]) -> None: + """Report nested ``SequenceLocation`` identifiers that disagree with their own coordinates. + + Separate from :func:`audit` because it is a separate defect with a separate cause. ``audit`` checks + ``vrs_digest`` against ``post_mapped`` — the allele's own identifier. ``ga4gh_identify`` writes only + the ``id`` of the object it is handed; for sub-objects it calls ``get_or_create_digest`` and never + ``get_or_create_ga4gh_identifier``. So an allele can carry a correct identifier over a location whose + ``id`` was minted before normalization moved the span, and the allele-level audit reports it clean. + + Read-only. A drifted location is repaired by re-mapping the score set, since both writers now + restamp the location — see ``lib/vrs_utils.py::identify_allele``. + """ + query = select(Allele).where(Allele.post_mapped.isnot(None)).order_by(Allele.id) + if limit is not None: + query = query.limit(limit) + + counts: Counter[str] = Counter() + drifted: set[int] = set() + + for allele in db.scalars(query): + counts["alleles examined"] += 1 + for path, location in _stored_locations(allele.post_mapped): + stored = location.get("id") + computed = _recomputed_location_identifier(location) + + if computed is None: + counts["could not hydrate location"] += 1 + elif stored is None: + counts["location carries no id"] += 1 + elif stored == computed: + counts["location id matches content"] += 1 + else: + counts["location id DRIFTED"] += 1 + drifted.add(allele.id) + logger.debug("Allele %s %s: %s -> %s", allele.id, path, stored, computed) + + for label, count in counts.most_common(): + logger.info("%9d %s", count, label) + logger.info("%9d distinct allele rows carrying a drifted location", len(drifted)) + + if drifted: + logger.warning( + "Re-map the affected score sets. A drifted location id is not repairable in place here — the " + "allele's own identifier is unaffected, so nothing in this audit's scope is wrong." + ) + + +def _stored_locations(post_mapped: Any) -> list[tuple[str, dict[str, Any]]]: + """Every ``SequenceLocation`` in a stored document, with a label saying where it sat. + + A ``CisPhasedBlock`` embeds one allele per member, each with its own location, and those member + copies are how one writer's convention leaked into another's rows. + """ + if not isinstance(post_mapped, dict): + return [] + + locations = [] + if isinstance(post_mapped.get("location"), dict): + locations.append(("location", post_mapped["location"])) + for index, member in enumerate(post_mapped.get("members") or []): + if isinstance(member, dict) and isinstance(member.get("location"), dict): + locations.append((f"members[{index}].location", member["location"])) + + return locations + + +def _recomputed_location_identifier(location: dict[str, Any]) -> Optional[str]: + """The identifier this location's own coordinates imply, ignoring what it currently claims.""" + try: + content = {key: value for key, value in location.items() if key not in ("id", "digest")} + return ga4gh_identify(SequenceLocation(**content), in_place="always") + except Exception: + logger.debug("Could not hydrate a location for identification", exc_info=True) + return None diff --git a/src/mavedb/worker/jobs/variant_processing/mapping.py b/src/mavedb/worker/jobs/variant_processing/mapping.py index 4e1e2510a..19a76e224 100644 --- a/src/mavedb/worker/jobs/variant_processing/mapping.py +++ b/src/mavedb/worker/jobs/variant_processing/mapping.py @@ -28,12 +28,13 @@ from mavedb.lib.types.workflow import JobExecutionOutcome from mavedb.lib.variant_translations import get_or_create_allele from mavedb.lib.variants import get_hgvs_from_post_mapped +from mavedb.lib.vrs_utils import canonical_variation_document from mavedb.models.allele import Allele as AlleleDbModel -from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.models.enums.annotation_type import AnnotationType from mavedb.models.enums.disposition import Disposition from mavedb.models.enums.job_pipeline import FailureCategory from mavedb.models.enums.mapping_state import MappingState +from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.models.mapping_record import MappingRecord from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.score_set import ScoreSet @@ -383,13 +384,18 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan # Only variants with a post-mapped representation yield an authoritative Allele; # failed and benign-absent variants get a MappingRecord but no linked allele. if post_mapped_allele: + # Recompute rather than trust the incoming id, which may predate normalization for a + # deletion/duplication. vrs_digest is the dedup key the whole allele graph hangs off. + canonical_post_mapped, allele_digest = canonical_variation_document( + post_mapped_allele, subject=f"variant {variant.urn}" + ) allele_draft = AlleleDbModel( - vrs_digest=post_mapped_allele["id"], + vrs_digest=allele_digest, level=sequence_level, hgvs_g=assay_level_hgvs if sequence_level == SequenceLevel.genomic else None, hgvs_c=assay_level_hgvs if sequence_level == SequenceLevel.cdna else None, hgvs_p=assay_level_hgvs if sequence_level == SequenceLevel.protein else None, - post_mapped=post_mapped_allele, + post_mapped=canonical_post_mapped, ) authoritative_allele = get_or_create_allele(job_manager.db, allele_draft) job_manager.db.flush() diff --git a/tests/lib/test_vrs_utils.py b/tests/lib/test_vrs_utils.py index 3b279560c..07823c20e 100644 --- a/tests/lib/test_vrs_utils.py +++ b/tests/lib/test_vrs_utils.py @@ -1,5 +1,7 @@ # ruff: noqa: E402 +import logging +from copy import deepcopy from types import SimpleNamespace import pytest @@ -20,6 +22,8 @@ from mavedb.lib import vrs_utils from mavedb.lib.vrs_utils import ( _rle_to_lse, + canonical_variation_document, + identify_allele, identify_variation, normalize_and_identify, translate_hgvs_to_variation, @@ -160,6 +164,52 @@ def test_identify_variation_clears_stale_block_digest(): assert digest.startswith("ga4gh:CPB.") # recomputed from content, not the stale cache +# A location id left over from before normalization moved the span. The allele id comes out +# correct while the location it points at still claims the un-normalized interval. +_STALE_LOCATION_ID = "ga4gh:SL." + "Y" * 32 + + +def test_identify_allele_restamps_a_stale_location_id(): + """``ga4gh_identify`` writes only the id of the object it is handed, never a sub-object's.""" + reference = _allele(1000, "G") + identify_allele(reference) + assert isinstance(reference.location, SequenceLocation) + + drifted = _allele(1000, "G") + assert isinstance(drifted.location, SequenceLocation) + drifted.location.id = _STALE_LOCATION_ID + + identify_allele(drifted) + + assert drifted.location.id == reference.location.id + assert drifted.location.id != _STALE_LOCATION_ID + + +def test_an_identified_location_id_restates_its_own_digest(): + """The invariant that makes drift detectable by string comparison rather than recomputation.""" + allele = _allele(1000, "G") + identify_allele(allele) + assert isinstance(allele.location, SequenceLocation) + + assert allele.location.id == f"ga4gh:SL.{allele.location.digest}" + + +def test_identify_variation_restamps_a_stale_member_location_id(): + """A block embeds allele copies, so a drifted member location rides into the block unnoticed.""" + member = _allele(10, "A") + assert isinstance(member.location, SequenceLocation) + member.location.id = _STALE_LOCATION_ID + block = CisPhasedBlock(members=[member, _allele(20, "G")]) # type: ignore[call-arg] + + identify_variation(block) + + embedded = block.members[0] + assert isinstance(embedded, Allele) + assert isinstance(embedded.location, SequenceLocation) + assert embedded.location.id != _STALE_LOCATION_ID + assert embedded.location.id == f"ga4gh:SL.{embedded.location.digest}" + + def _rle_allele(start: int, *, length: int, repeat_subunit_length: int) -> Allele: return Allele( location=SequenceLocation( @@ -240,3 +290,116 @@ def test_normalize_and_identify_rejects_iri_location_for_rle(monkeypatch): with pytest.raises(AssertionError): normalize_and_identify(bad, data_proxy=_NEVER_READ) + + +# --------------------------- canonical_variation_document --------------------------- +# +# The ingest guard: identifiers arriving from the mapper are assertions, not identity. These pin the +# behaviour that keeps `vrs_digest` — the dedup key the allele graph hangs off — true of its own content. + + +def _document(start: int, alt: str, *, identifier: str | None = None) -> dict: + """A serialized, correctly-identified allele; ``identifier`` overrides the id with a supplied one.""" + allele = _allele(start, alt) + allele.id = identify_allele(allele) + document = allele.model_dump(mode="json", exclude_none=True) + if identifier is not None: + document["id"] = identifier + return document + + +def test_canonicalization_is_a_no_op_for_a_correct_identifier(caplog): + """The overwhelming majority of incoming alleles are already right; they must pass through unchanged.""" + correct = identify_allele(_allele(10, "A")) + + document, identifier = canonical_variation_document(_document(10, "A", identifier=correct), subject="v1") + + assert identifier == correct + assert document["id"] == correct + assert "mismatch" not in caplog.text + + +def test_a_stale_identifier_is_replaced_and_warned_about(caplog): + """The drift case: an id minted over different content than the document now carries.""" + stale = identify_allele(_allele(99, "T")) + caplog.set_level(logging.WARNING) + + document, identifier = canonical_variation_document(_document(10, "A", identifier=stale), subject="urn:x#1") + + assert identifier != stale + assert identifier == identify_allele(_allele(10, "A")) + assert document["id"] == identifier + assert "urn:x#1" in caplog.text + assert stale in caplog.text + + +def test_an_absent_identifier_is_minted_without_a_warning(caplog): + caplog.set_level(logging.WARNING) + document = _document(10, "A") + document.pop("id", None) + + _, identifier = canonical_variation_document(document, subject="v1") + + assert identifier == identify_allele(_allele(10, "A")) + assert "mismatch" not in caplog.text + + +def test_the_same_content_canonicalizes_to_the_same_bytes(): + """Two writers, two serializations, one identifier is what corrupts a normalized export.""" + verbose = _document(10, "A") + verbose["digest"] = verbose["id"].split(".")[-1] + verbose["location"]["sequenceReference"]["label"] = "NC_000001.11" + verbose["extensions"] = [] + + terse = _document(10, "A") + terse.pop("digest", None) + + assert canonical_variation_document(verbose, subject="v")[0] == canonical_variation_document(terse, subject="v")[0] + + +def test_canonicalization_does_not_mutate_the_caller_s_document(): + document = _document(10, "A", identifier="ga4gh:VA.definitelyWrong") + before = deepcopy(document) + + canonical_variation_document(document, subject="v") + + assert document == before + + +def test_a_cis_phased_block_is_canonicalized_too(): + """CisPhasedBlock members embed allele copies, which is how one writer's convention leaks.""" + block = CisPhasedBlock(members=[_allele(10, "A"), _allele(20, "G")]) # type: ignore[call-arg] + block.id = "ga4gh:CPB.stale" + document = block.model_dump(mode="json", exclude_none=True) + + canonical, identifier = canonical_variation_document(document, subject="v") + + assert identifier.startswith("ga4gh:CPB.") + assert canonical["id"] == identifier + + +def test_canonicalization_repairs_a_stale_location_id(): + """The observed drift shape: a correct allele id over a location id minted pre-normalization. + + This is what put 938 identifiers on more than one coordinate span in + ``urn:mavedb:00000662-0-1`` — the allele level was repaired and the location level was not. + """ + document = _document(10, "A") + document["location"]["id"] = _STALE_LOCATION_ID + + canonical, _ = canonical_variation_document(document, subject="v") + + assert canonical["location"]["id"] != _STALE_LOCATION_ID + assert canonical["location"]["id"] == f"ga4gh:SL.{canonical['location']['digest']}" + + +def test_two_spans_that_shared_one_location_id_canonicalize_apart(): + """One key, two bodies is what corrupts a keyed export; distinct spans must get distinct ids.""" + first = _document(10, "A") + second = _document(11, "A") + second["location"]["id"] = first["location"]["id"] + + canonical_first, _ = canonical_variation_document(first, subject="v") + canonical_second, _ = canonical_variation_document(second, subject="v") + + assert canonical_first["location"]["id"] != canonical_second["location"]["id"] diff --git a/tests/worker/jobs/variant_processing/test_reverse_translation.py b/tests/worker/jobs/variant_processing/test_reverse_translation.py index 030d17020..d3ffb004e 100644 --- a/tests/worker/jobs/variant_processing/test_reverse_translation.py +++ b/tests/worker/jobs/variant_processing/test_reverse_translation.py @@ -33,7 +33,8 @@ reverse_translate_variants_for_score_set, ) from mavedb.worker.lib.managers.job_manager import JobManager -from tests.helpers.constants import TEST_GA4GH_IDENTIFIER +from mavedb.lib.vrs_utils import canonical_variation_document +from tests.helpers.constants import TEST_VALID_POST_MAPPED_VRS_ALLELE from tests.helpers.util.setup.worker import construct_mock_mapping_output pytestmark = pytest.mark.usefixtures("patch_db_session_ctxmgr") @@ -126,6 +127,14 @@ def _translate(hgvs: str, translator=None) -> FakeVrsVariation: return _translate +# Mapping recomputes an incoming allele's identity from its own content rather than trusting the +# producer's `id`, so a stored authoritative allele carries this rather than the fixture's placeholder. +# The fold-in tests below depend on the reverse-translated coding allele identifying to the same value. +CANONICAL_AUTHORITATIVE_DIGEST = canonical_variation_document( + TEST_VALID_POST_MAPPED_VRS_ALLELE, subject="mapping fixture" +)[1] + + async def _map_variants(session, mock_worker_ctx, mapping_run, score_set, with_layers=frozenset({"g", "c", "p"})): """Run the (mocked) mapping job so the score set's variants gain authoritative MappingRecords/Alleles — the real inputs the reverse translation job reads.""" @@ -566,14 +575,14 @@ async def test_authoritative_allele_is_folded_into_its_projection_group( # Exactly one allele exists after mapping: the authoritative (measured) allele. assert session.query(Allele).count() == 1 - assert session.query(Allele).one().vrs_digest == TEST_GA4GH_IDENTIFIER + assert session.query(Allele).one().vrs_digest == CANONICAL_AUTHORITATIVE_DIGEST assay_hgvs = "NM_000000.1:c.1A>G" c_candidate = "NM_000001.1:c.5A>G" g_sibling = "NC_000001.11:g.1000A>G" construct = fake_construct({assay_hgvs: [(c_candidate, g_sibling)]}) # The coding member IS the measured allele (folds in); the genomic member is its projection. - translate = fake_translate({c_candidate: TEST_GA4GH_IDENTIFIER, g_sibling: "ga4gh:VA.genomic"}) + translate = fake_translate({c_candidate: CANONICAL_AUTHORITATIVE_DIGEST, g_sibling: "ga4gh:VA.genomic"}) with ( patch(f"{RT_MODULE}.construct_equivalent_variants", construct), @@ -601,7 +610,7 @@ async def test_authoritative_allele_is_folded_into_its_projection_group( group_links = [link for link in _live_links(session, record.id) if link.projection_group == 0] assert len(group_links) == 2 assert {link.is_authoritative for link in group_links} == {True, False} - assert auth_link.allele.vrs_digest == TEST_GA4GH_IDENTIFIER + assert auth_link.allele.vrs_digest == CANONICAL_AUTHORITATIVE_DIGEST assert derived_links[0].allele.vrs_digest == "ga4gh:VA.genomic" async def test_authoritative_fold_in_when_measured_at_genomic_level( @@ -629,14 +638,14 @@ async def test_authoritative_fold_in_when_measured_at_genomic_level( await _map_variants( session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set, with_layers={"g", "c"} ) - assert session.query(Allele).one().vrs_digest == TEST_GA4GH_IDENTIFIER + assert session.query(Allele).one().vrs_digest == CANONICAL_AUTHORITATIVE_DIGEST assay_hgvs = "NC_000001.11:g.1000A>G" c_sibling = "NM_000001.1:c.5A>G" g_candidate = "NC_000001.11:g.1000A>G" construct = fake_construct({assay_hgvs: [(c_sibling, g_candidate)]}) # The genomic member IS the measured allele (folds in); the coding member is its projection. - translate = fake_translate({c_sibling: "ga4gh:VA.coding", g_candidate: TEST_GA4GH_IDENTIFIER}) + translate = fake_translate({c_sibling: "ga4gh:VA.coding", g_candidate: CANONICAL_AUTHORITATIVE_DIGEST}) with ( patch(f"{RT_MODULE}.construct_equivalent_variants", construct), @@ -652,7 +661,7 @@ async def test_authoritative_fold_in_when_measured_at_genomic_level( derived_links = _non_authoritative_links(session) assert auth_link.projection_group == 0 assert [link.projection_group for link in derived_links] == [0] - assert auth_link.allele.vrs_digest == TEST_GA4GH_IDENTIFIER + assert auth_link.allele.vrs_digest == CANONICAL_AUTHORITATIVE_DIGEST assert derived_links[0].allele.vrs_digest == "ga4gh:VA.coding" async def test_projection_failed_candidate_is_a_one_member_group( @@ -737,7 +746,7 @@ async def test_protein_assay_leaves_authoritative_group_null_over_a_fan_out( await _map_variants( session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set, with_layers={"p"} ) - assert session.query(Allele).one().vrs_digest == TEST_GA4GH_IDENTIFIER + assert session.query(Allele).one().vrs_digest == CANONICAL_AUTHORITATIVE_DIGEST assay_hgvs = "NP_000000.1:p.Met1Val" # A degenerate protein consequence -> three coding/genomic projection pairs (the fan-out). @@ -765,7 +774,7 @@ async def test_protein_assay_leaves_authoritative_group_null_over_a_fan_out( # The protein authoritative allele is not part of any pair: its group stays NULL. auth_link = _authoritative_link(session, record.id) assert auth_link.projection_group is None - assert auth_link.allele.vrs_digest == TEST_GA4GH_IDENTIFIER + assert auth_link.allele.vrs_digest == CANONICAL_AUTHORITATIVE_DIGEST # Three distinct groups, each a coding/genomic pair (two members). groups: dict[int, int] = {} From 66976f34d686b42d2f97f88799b56ebf55d37a7e Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 26 Aug 2026 09:52:33 -0700 Subject: [PATCH 120/252] fix(annotation): drop wall-clock date from the API contribution The API contribution stamped date=datetime.today(), which is not provenance: it records when a view was serialized, not when anything was contributed. What this contribution asserts is which software produced the annotation, and that is the agent's version. The wall-clock stamp also made the object unique per call, so identical provenance never deduplicated and va.ndjson changed bytes on every run over unchanged data, defeating checksum manifests and incremental sync for consumers of the public dump. --- src/mavedb/lib/annotation/contribution.py | 13 +++++++-- tests/lib/annotation/test_contribution.py | 32 ++++++++++++++++------- 2 files changed, 33 insertions(+), 12 deletions(-) diff --git a/src/mavedb/lib/annotation/contribution.py b/src/mavedb/lib/annotation/contribution.py index 839721b5f..a63979920 100644 --- a/src/mavedb/lib/annotation/contribution.py +++ b/src/mavedb/lib/annotation/contribution.py @@ -1,5 +1,4 @@ import logging -from datetime import datetime from ga4gh.core.models import Extension from ga4gh.va_spec.base.core import Contribution @@ -21,12 +20,22 @@ def mavedb_api_contribution() -> Contribution: """ Create a [VA Contribution](https://va-ga4gh.readthedocs.io/en/latest/core-information-model/entities/activities/contribution.html#contribution) object for an arbitary contribution from the MaveDB API/software distribution. + + Carries no ``date``. What this contribution asserts is *which software* produced the annotation, and + that is the agent's version — see :func:`mavedb_api_agent`, which stamps ``__version__``. The only + date available at this point is the instant of serialization, which is not provenance: it describes + when a view was rendered, not when anything was contributed. The sibling builders in this module all + carry a real stored date instead. + + Stamping wall-clock time here also made the object unique per call, so identical provenance never + deduplicated and ``va.ndjson`` changed bytes on every run of unchanged data — defeating checksum + manifests and incremental sync for consumers of the public dump. When an artifact needs a generation + time it belongs on that artifact's metadata, once, not on every nested object inside it. """ return Contribution( name="MaveDB API", description="Contribution from the MaveDB API", contributor=mavedb_api_agent(), - date=datetime.today(), activityType="software application programming interface", ) diff --git a/tests/lib/annotation/test_contribution.py b/tests/lib/annotation/test_contribution.py index 60990baf6..bbc9ae13d 100644 --- a/tests/lib/annotation/test_contribution.py +++ b/tests/lib/annotation/test_contribution.py @@ -16,6 +16,7 @@ from ga4gh.core.models import Extension from ga4gh.va_spec.base import Contribution +from mavedb import __version__ from mavedb.lib.annotation.contribution import ( mavedb_api_contribution, mavedb_creator_contribution, @@ -59,11 +60,11 @@ def test_has_correct_activity_type(self): assert contribution.activityType == "software application programming interface" - def test_has_current_date(self): - """Test that contribution uses current date.""" + def test_carries_no_date(self): + """The software's provenance is its version, carried by the agent; serialization time is not provenance.""" contribution = mavedb_api_contribution() - assert contribution.date.date() == datetime.today().date() + assert contribution.date is None def test_has_contributor(self): """Test that contribution has a contributor.""" @@ -71,14 +72,20 @@ def test_has_contributor(self): assert contribution.contributor is not None - def test_consistency(self): - """Test that multiple calls produce consistent results.""" - contribution1 = mavedb_api_contribution() - contribution2 = mavedb_api_contribution() + def test_contributor_carries_the_api_version(self): + """The API version is what this contribution actually asserts.""" + contribution = mavedb_api_contribution() + + versions = [e.value for e in contribution.contributor.extensions if e.name == "mavedbApiVersion"] + assert versions == [__version__] - assert contribution1.name == contribution2.name - assert contribution1.description == contribution2.description - assert contribution1.activityType == contribution2.activityType + def test_repeated_calls_are_identical(self): + """Two calls must serialize to the same bytes. + + This is what makes identical provenance deduplicate in a normalized export, and what makes the + public `va.ndjson` reproducible across runs of unchanged data. + """ + assert mavedb_api_contribution() == mavedb_api_contribution() @pytest.mark.unit @@ -376,7 +383,12 @@ def test_all_contributions_return_consistent_structure(self): assert contrib.description is not None assert contrib.activityType is not None assert contrib.contributor is not None + + # A date is carried only where a stored one exists to carry. The API contribution has none: + # its provenance is the agent's version, not the moment of serialization. + for contrib in [vrs_contrib, cal_contrib, creator_contrib, modifier_contrib]: assert contrib.date is not None + assert api_contrib.date is None def test_date_formatting_consistency(self): """Test that all functions format dates consistently.""" From 45050d59325337514aaffcd69f05702edea0ca0c Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 26 Aug 2026 16:00:43 -0700 Subject: [PATCH 121/252] build(dev): replace LocalStack with moto for local S3 emulation LocalStack archived its Community edition in March 2026 and moved S3 behind a paid token, so `docker compose up` fails with auth errors for anyone pulling the current image. Swap in moto (Apache-2.0), which is built for boto3 and runs the CSV upload/download flow with an unmodified client -- no path-style or expect-100-continue workarounds, unlike Adobe S3Mock (Java-SDK-first). moto has no auto-create, so a one-shot curl container PUTs the upload bucket once moto is up; this removes the awslocal-based init script. The app reaches moto at moto:5000 internally; the host port is 5001 to avoid the macOS AirPlay Receiver on 5000. Refs #768 --- bin/localstack-init.sh | 4 ---- docker-compose-dev.yml | 23 ++++++++++++----------- settings/.env.template | 2 +- 3 files changed, 13 insertions(+), 16 deletions(-) delete mode 100755 bin/localstack-init.sh diff --git a/bin/localstack-init.sh b/bin/localstack-init.sh deleted file mode 100755 index 1a00cfcbc..000000000 --- a/bin/localstack-init.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh -echo "Initializing local S3 bucket..." -awslocal s3 mb s3://score-set-csv-uploads-dev -echo "S3 bucket 'score-set-csv-uploads-dev' created." \ No newline at end of file diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 9835cff4e..925c30f0d 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -103,17 +103,19 @@ services: volumes: - mavedb-redis-dev:/data - localstack: - image: localstack/localstack:latest + # AWS mock. + moto: + image: motoserver/moto:latest ports: - - "4566:4566" - env_file: - - settings/.env.dev - environment: - - SERVICES=s3:4566 # We only need S3 for MaveDB - volumes: - - mavedb-localstack-dev:/var/lib/localstack - - "./bin/localstack-init.sh:/etc/localstack/init/ready.d/localstack-init.sh" + - "5001:5000" + + moto-init: + # One-shot: create the upload bucket once moto is up (moto has no auto-create). + # A raw PUT is all it takes; --retry-all-errors waits out moto's startup. + image: curlimages/curl:latest + depends_on: + - moto + command: -sf --retry 10 --retry-all-errors --retry-delay 1 -X PUT http://moto:5000/score-set-csv-uploads-dev seqrepo: image: biocommons/seqrepo:2024-12-20 @@ -124,4 +126,3 @@ volumes: mavedb-data-dev: mavedb-redis-dev: mavedb-seqrepo-dev: - mavedb-localstack-dev: diff --git a/settings/.env.template b/settings/.env.template index 585bd354f..4574c6dd4 100644 --- a/settings/.env.template +++ b/settings/.env.template @@ -105,7 +105,7 @@ GNOMAD_DATA_VERSION=v4.1 AWS_ACCESS_KEY_ID=test AWS_SECRET_ACCESS_KEY=test -S3_ENDPOINT_URL=http://localstack:4566 +S3_ENDPOINT_URL=http://moto:5000 UPLOAD_S3_BUCKET_NAME=score-set-csv-uploads-dev #################################################################################################### From ccf3eeff8c931a1d602cef706eecadc37a600e23 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Fri, 28 Aug 2026 10:27:26 -0700 Subject: [PATCH 122/252] fix: return 4XX for malformed input to hgvs/validate The parse step in hgvs_validate ran outside the try/except, so an HGVSParseError on a malformed variant string escaped to the catch-all 500 handler (and fired a Slack alert). A missing "variant" body field raised KeyError with the same outcome. Parse inside the error handler and catch the broad HGVSError so any caller-supplied HGVS failure (unparseable string, inconsistent variant, unknown accession) becomes a 400. Replace the free-form dict body with a typed HgvsValidationRequest model, which makes a missing/mistyped field a 422 and self-documents the request schema. --- src/mavedb/routers/hgvs.py | 22 ++++++++++++---------- src/mavedb/view_models/hgvs.py | 5 +++++ tests/routers/test_hgvs.py | 16 ++++++++++++++++ 3 files changed, 33 insertions(+), 10 deletions(-) create mode 100644 src/mavedb/view_models/hgvs.py diff --git a/src/mavedb/routers/hgvs.py b/src/mavedb/routers/hgvs.py index c2352c81d..ac6277300 100644 --- a/src/mavedb/routers/hgvs.py +++ b/src/mavedb/routers/hgvs.py @@ -5,10 +5,11 @@ from cdot.hgvs.dataproviders import RESTDataProvider from fastapi import APIRouter, Depends, HTTPException from hgvs import parser, validator -from hgvs.exceptions import HGVSDataNotAvailableError, HGVSInvalidVariantError +from hgvs.exceptions import HGVSDataNotAvailableError, HGVSError from mavedb.deps import hgvs_data_provider -from mavedb.routers.shared import BASE_400_RESPONSE, PUBLIC_ERROR_RESPONSES, ROUTER_BASE_PREFIX +from mavedb.routers.shared import PUBLIC_ERROR_RESPONSES, ROUTER_BASE_PREFIX, VALIDATION_ERROR_RESPONSES +from mavedb.view_models.hgvs import HgvsValidationRequest TAG_NAME = "Transcripts" @@ -44,22 +45,23 @@ def hgvs_fetch(accession: str, hdp: RESTDataProvider = Depends(hgvs_data_provide "/validate", status_code=200, response_model=bool, - responses={**BASE_400_RESPONSE}, + responses={**VALIDATION_ERROR_RESPONSES}, summary="Validate a provided variant", ) -def hgvs_validate(variant: dict[str, str], hdp: RESTDataProvider = Depends(hgvs_data_provider)) -> bool: +def hgvs_validate(request: HgvsValidationRequest, hdp: RESTDataProvider = Depends(hgvs_data_provider)) -> bool: """ Validate the provided HGVS variant string. + + Parsing and validation failures both stem from caller-supplied input, so any ``HGVSError`` — a syntactic + parse failure, an inconsistent variant, an unknown accession — is surfaced as a 400 rather than escaping + to the catch-all 500 handler. """ hp = parser.Parser() - variant_hgvs = hp.parse(variant["variant"]) - try: - valid = validator.Validator(hdp=hdp).validate(variant_hgvs, strict=False) - except HGVSInvalidVariantError as e: + variant_hgvs = hp.parse(request.variant) + return validator.Validator(hdp=hdp).validate(variant_hgvs, strict=False) + except HGVSError as e: raise HTTPException(400, str(e)) - else: - return valid @router.get("/assemblies", status_code=200, response_model=list[str], summary="List stored assemblies") diff --git a/src/mavedb/view_models/hgvs.py b/src/mavedb/view_models/hgvs.py new file mode 100644 index 000000000..e295cc9df --- /dev/null +++ b/src/mavedb/view_models/hgvs.py @@ -0,0 +1,5 @@ +from mavedb.view_models.base.base import BaseModel + + +class HgvsValidationRequest(BaseModel): + variant: str diff --git a/tests/routers/test_hgvs.py b/tests/routers/test_hgvs.py index 6011953f2..b8a689dfb 100644 --- a/tests/routers/test_hgvs.py +++ b/tests/routers/test_hgvs.py @@ -24,6 +24,7 @@ INVALID_TRANSCRIPT = "NX_99999.1" VALID_VARIANT = VALID_NT_ACCESSION + ":c.1G>A" INVALID_VARIANT = VALID_NT_ACCESSION + ":c.1delA" +UNPARSEABLE_VARIANT = "not a valid hgvs string" HAS_PROTEIN_ACCESSION = "NM_000014.4" PROTEIN_ACCESSION = "NP_000005.2" @@ -64,6 +65,21 @@ def test_hgvs_validate_invalid(client, setup_router_db): assert "does not agree" in response.json()["detail"] +def test_hgvs_validate_unparseable(client, setup_router_db): + # A syntactically invalid HGVS string fails at the parse stage, before validation. This is a caller + # error and must surface as a 400 rather than escaping to the catch-all 500 handler. + payload = {"variant": UNPARSEABLE_VARIANT} + response = client.post("/api/v1/hgvs/validate", json=payload) + + assert response.status_code == 400 + + +def test_hgvs_validate_missing_field(client, setup_router_db): + response = client.post("/api/v1/hgvs/validate", json={}) + + assert response.status_code == 422 + + def test_hgvs_list_assemblies(client, setup_router_db): response = client.get("/api/v1/hgvs/assemblies") assert response.status_code == 200 From cff006c898244c24e251ec28a725b187f6848ed2 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Fri, 28 Aug 2026 13:13:40 -0700 Subject: [PATCH 123/252] feat(models): add VEP consequence provenance schema functional_consequence has been the only stored fact about a VEP result: no record of which transcript_consequences entry it came from, or whether it was a transcript-specific match at all versus VEP's cross-transcript most_severe headline, which regularly describes a different overlapping isoform than the allele's own transcript (#772). Add four nullable columns to vep_allele_consequences: consequence_terms (every term from the matched transcript entry, severity-ordered), consequence_source (transcript / most_severe / reference_identical, mirrored from variant_annotation.lib.vep.ConsequenceSource by VepConsequenceSource), matched_transcript (set only when the source is transcript), and resolver_version (variant_annotation.lib.vep.RESOLVER_VERSION at write time). resolver_version pairs with the existing source_version as the second axis the current-release skip keys on, so a resolution-rule fix re-queries every allele instead of looking current forever under an unchanged Ensembl release. All nullable with no backfill: a legacy NULL never matches the current resolver_version, so it is re-queried once and filled in place. --- ...c2e1b7d6_add_vep_consequence_provenance.py | 73 ++++++++++++++++ src/mavedb/models/enums/vep.py | 28 ++++++ src/mavedb/models/vep_allele_consequence.py | 86 ++++++++++++------- 3 files changed, 156 insertions(+), 31 deletions(-) create mode 100644 alembic/versions/a3f9c2e1b7d6_add_vep_consequence_provenance.py create mode 100644 src/mavedb/models/enums/vep.py diff --git a/alembic/versions/a3f9c2e1b7d6_add_vep_consequence_provenance.py b/alembic/versions/a3f9c2e1b7d6_add_vep_consequence_provenance.py new file mode 100644 index 000000000..ff1790bf3 --- /dev/null +++ b/alembic/versions/a3f9c2e1b7d6_add_vep_consequence_provenance.py @@ -0,0 +1,73 @@ +"""add resolution provenance to vep_allele_consequences + +Revision ID: a3f9c2e1b7d6 +Revises: b2e5a8d3f0c1 +Create Date: 2026-08-26 + +Adds four nullable columns to ``vep_allele_consequences`` so a stored consequence records *how* it was +resolved, not just the headline term (#772): + +- ``consequence_terms`` (JSONB) — every term from the matched transcript entry, severity-ordered. +- ``consequence_source`` (varchar check-constraint enum) — ``transcript`` | ``most_severe`` | + ``reference_identical``. +- ``matched_transcript`` (varchar) — the versioned transcript VEP used; NULL unless the source is + ``transcript``. +- ``resolver_version`` (varchar) — ``variant_annotation.lib.vep.RESOLVER_VERSION``, the version of our + resolution *rule* (severity ranking, transcript-matching, Recoder combination). Pairs with + ``source_version`` (the Ensembl release) as the second version axis the current-release skip keys on, + so a rule fix shipped in the shared kernel re-queries every allele instead of looking current forever. + +All nullable, no backfill: rows written before this migration keep NULL provenance / resolver_version. +A NULL ``resolver_version`` never matches the current one, so a legacy row is re-queried on the next VEP +run and then filled in place (the linker supersedes on a value change, and a NULL→populated column for +an unchanged headline term advances in place rather than churning history). +""" + +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "a3f9c2e1b7d6" +down_revision = "b2e5a8d3f0c1" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column( + "vep_allele_consequences", + sa.Column("consequence_terms", postgresql.JSONB(none_as_null=True), nullable=True), + ) + op.add_column( + "vep_allele_consequences", + sa.Column( + "consequence_source", + sa.Enum( + "transcript", + "most_severe", + "reference_identical", + name="vepconsequencesource", + native_enum=False, + create_constraint=True, + length=32, + ), + nullable=True, + ), + ) + op.add_column( + "vep_allele_consequences", + sa.Column("matched_transcript", sa.String(), nullable=True), + ) + op.add_column( + "vep_allele_consequences", + sa.Column("resolver_version", sa.String(), nullable=True), + ) + + +def downgrade() -> None: + op.drop_column("vep_allele_consequences", "resolver_version") + op.drop_column("vep_allele_consequences", "matched_transcript") + op.drop_column("vep_allele_consequences", "consequence_source") + op.drop_column("vep_allele_consequences", "consequence_terms") diff --git a/src/mavedb/models/enums/vep.py b/src/mavedb/models/enums/vep.py new file mode 100644 index 000000000..a45384083 --- /dev/null +++ b/src/mavedb/models/enums/vep.py @@ -0,0 +1,28 @@ +from enum import Enum + + +class VepConsequenceSource(str, Enum): + """Which resolution path produced a stored VEP consequence — provenance, not severity. + + Persisted on every :class:`~mavedb.models.vep_allele_consequence.VepAlleleConsequence` so a stored + consequence can be audited without re-querying Ensembl, and so a consumer can tell a + transcript-specific call from a cross-transcript one, see #772. + + - ``transcript`` — read from the ``transcript_consequences`` entry matching the allele's own + transcript. The trustworthy case: the consequence applies to the transcript the allele lives on. + - ``most_severe`` — VEP's top-level ``most_severe_consequence``, the worst call across *every* + transcript overlapping the position. Used only when no transcript could be matched (a genomic + allele carrying no transcript, or a transcript absent from VEP's set). Lower confidence: it + regularly reports a consequence that might not apply to the transcript of interest. + - ``reference_identical`` — the allele describes no sequence change (a wild-type control); no VEP + call was made. Recorded distinctly so it is never mistaken for ``synonymous_variant`` or a null. + + Values mirror ``variant_annotation.lib.vep.ConsequenceSource``: the library computes the source, the + api owns this closed set. The two are kept in sync by + :func:`mavedb.lib.vep.link_vep_consequences_to_alleles`, which maps one to the other by value at the + write boundary. + """ + + transcript = "transcript" + most_severe = "most_severe" + reference_identical = "reference_identical" diff --git a/src/mavedb/models/vep_allele_consequence.py b/src/mavedb/models/vep_allele_consequence.py index 4133a1231..8d6eaeacb 100644 --- a/src/mavedb/models/vep_allele_consequence.py +++ b/src/mavedb/models/vep_allele_consequence.py @@ -1,11 +1,13 @@ from datetime import date -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING -from sqlalchemy import Column, Date, ForeignKey, Index, Integer, String, text +from sqlalchemy import Column, Date, Enum, ForeignKey, Index, Integer, String, text +from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.orm import Mapped, relationship from mavedb.db.base import Base from mavedb.db.mixins import ValidTime +from mavedb.models.enums.vep import VepConsequenceSource if TYPE_CHECKING: from .allele import Allele @@ -14,34 +16,41 @@ class VepAlleleConsequence(ValidTime, Base): """Valid-time VEP functional-consequence result for a deduplicated :class:`Allele`. - Replaces the frozen ``vep_functional_consequence``/``vep_access_date`` columns on - ``MappedVariant`` for new-model writes (Step 2 of the annotation infrastructure migration, - docs/design/annotation-infrastructure-migration.md). A row is live while ``valid_to`` is NULL; - the partial unique index enforces **a single live consequence per allele** — VEP's most-severe - consequence is one current value, so a changed result supersedes the prior row rather than - accumulating. This matches the gnomAD link shape, not ClinVar's multi-live shape. - - ``source_version`` is the Ensembl release the consequence was resolved under (e.g. ``"116"``, - from ``/info/software``). An Ensembl release is coordinated — software + transcript set + - consequence vocabulary all bump together under one number — so this single value version-keys the - upstream result exactly like gnomAD's ``db_version``. The job skips re-querying any allele already - live at the current release. What it does **not** capture is our own ``VEP_CONSEQUENCES`` severity - ordering (the list we pick "most severe" from); a change to that is a manual ``force`` re-run, not - an automatic supersede. - - Supersede is deliberately **value-keyed, not version-keyed** (the one divergence from gnomAD): a VEP - consequence is categorical and usually identical across releases, so superseding on every release - bump would churn history every quarter with rows recording "still missense, still missense." Instead - a new release that resolves the *same* consequence advances ``source_version``/``access_date`` in - place — no supersede — and only a *changed* consequence retires the old row and inserts a successor. - The trade-off: the live row's ``source_version`` is the latest release that confirmed the value, not - the release it first appeared; acceptable because it describes the currently-held value's - provenance, not when it became true. ``access_date`` is retained as a human-facing "last confirmed" - audit stamp; it is no longer load-bearing for the skip. - - ``functional_consequence`` is nullable to leave room for a future negative cache (NULL = "VEP ran - and found nothing"); the current job writes only non-null consequences and re-queries no-result - alleles each run, mirroring gnomAD's no-match handling. + A row is live while ``valid_to`` is NULL; the partial unique index enforces a + single live consequence per allele, matching the gnomAD link shape (not ClinVar's multi-live shape). + + Two version axes key a stored consequence, and the job's current-release skip requires *both* to + match: + + - ``source_version`` — the Ensembl release the consequence was resolved under (e.g. ``"116"``), + version-keying the upstream result like gnomAD's ``db_version``. Always the *latest* release that + confirmed the value, not the release it first appeared. + - ``resolver_version`` — ``variant_annotation.lib.vep.RESOLVER_VERSION``, the version of *our* + resolution rule (severity ranking, transcript matching, Recoder combination). Catches a rule fix + the Ensembl release can't see; a NULL (legacy) row never matches and is re-queried once, then + filled in place. + + Supersede is **value-keyed, not version-keyed** (the one divergence from gnomAD): a re-run + confirming the *same* consequence advances both versions and ``access_date`` in place rather than + churning history; only a *changed* consequence retires the old row and inserts a successor. + ``access_date`` is a human-facing "last confirmed" stamp — it plays no part in the skip itself. + + ``functional_consequence`` is nullable to leave room for a future negative cache; the current job + writes only non-null consequences and re-queries no-result alleles each run. + + **Resolution provenance (#772).** ``functional_consequence`` is the headline term; three columns + record *how* it was reached, since VEP's cross-transcript ``most_severe`` headline routinely + describes a different overlapping isoform than the allele's own transcript: + + - ``consequence_terms`` — every term from the matched transcript entry, severity-ordered; + ``functional_consequence`` is its first element. + - ``consequence_source`` — transcript-matched, cross-transcript headline, or reference-identical. + - ``matched_transcript`` — the transcript VEP actually used, set only when ``consequence_source = + 'transcript'`` (not assumed equal to the allele's own transcript, since the match is + version-insensitive). + + Transient request failures are not stored here, they live in the annotation event stream + (``AnnotationEvent``). """ __tablename__ = "vep_allele_consequences" @@ -52,8 +61,23 @@ class VepAlleleConsequence(ValidTime, Base): ForeignKey("alleles.id", ondelete="RESTRICT"), nullable=False, ) - functional_consequence: Mapped[Optional[str]] = Column(String, nullable=True) + functional_consequence = Column(String, nullable=True) + consequence_terms = Column(JSONB(none_as_null=True), nullable=True) + consequence_source = Column( + Enum( + VepConsequenceSource, + name="vepconsequencesource", + create_constraint=True, + length=32, + native_enum=False, + validate_strings=True, + ), + nullable=True, + ) + matched_transcript = Column(String, nullable=True) source_version: Mapped[str] = Column(String, nullable=False) + # Nullable only for rows predating the column (treated as stale — re-queried, then filled). + resolver_version = Column(String, nullable=True) access_date: Mapped[date] = Column(Date, nullable=False) allele: Mapped["Allele"] = relationship("Allele") From 72a31dcd12e0b97817fda46a8884f561a255c618 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Fri, 28 Aug 2026 13:14:02 -0700 Subject: [PATCH 124/252] refactor(vep): delegate resolution to the shared kernel and persist provenance lib/vep.py's local VEP_CONSEQUENCES ranking, get_functional_consequence, and run_variant_recoder duplicated variant_annotation.lib.vep's resolution rule under a different, staler ranking. Drop them in favor of resolve_consequences over the shared library, so this job and the offline CLI pipeline resolve a consequence identically by construction, and write the new provenance columns (consequence_terms, consequence_source, matched_transcript, resolver_version) in link_vep_consequences_to_alleles alongside the existing headline-term write. Add VepLinkVerdict.RETAINED_ON_ABSENCE: when VEP returns nothing for an allele that already has a live consequence, the prior value is kept rather than overwritten with a null, since a settled negative from one run should not erase a previously confirmed positive. The worker job now builds a transcript-aware VepInput per allele (only coding and genomic alleles are submitted; a protein allele's consequence is carried by the coding alleles its reverse translation enumerates from, so it is never VEP queried directly), wires in UTA's coding_interval_reference for best-effort reference-identical detection, and gates the current-release skip on resolver_version in addition to source_version so a resolution-rule change re-queries every allele instead of looking current forever. Drops the protein/Recoder end-to-end test now that protein alleles are never submitted. --- src/mavedb/lib/allele_annotations.py | 2 +- src/mavedb/lib/score_set_variants.py | 2 +- src/mavedb/lib/vep.py | 385 +++++++--------- src/mavedb/view_models/allele_annotation.py | 2 +- .../worker/jobs/external_services/vep.py | 388 ++++++++-------- tests/lib/test_vep.py | 426 ++++++++---------- .../external_services/network/test_vep.py | 54 +-- .../worker/jobs/external_services/test_vep.py | 304 ++++++++++--- 8 files changed, 774 insertions(+), 789 deletions(-) diff --git a/src/mavedb/lib/allele_annotations.py b/src/mavedb/lib/allele_annotations.py index 4e161dbf5..3d179f055 100644 --- a/src/mavedb/lib/allele_annotations.py +++ b/src/mavedb/lib/allele_annotations.py @@ -35,7 +35,7 @@ @dataclass(frozen=True) class VepAnnotation: - """A VEP most-severe functional consequence and the Ensembl release it was resolved under.""" + """A VEP molecular consequence for an allele and the Ensembl release it was resolved under.""" consequence: Optional[str] source_version: str diff --git a/src/mavedb/lib/score_set_variants.py b/src/mavedb/lib/score_set_variants.py index 262d11654..0955b875e 100644 --- a/src/mavedb/lib/score_set_variants.py +++ b/src/mavedb/lib/score_set_variants.py @@ -72,7 +72,7 @@ class MappedTriple: class LeanVariantRecord: """One pre-chewed per-variant record, keyed by ``variant_urn``. - ``consequence`` is a representative (most severe) VEP call. ``clingen_allele_id``/``assay_level_digest`` + ``consequence`` is the allele's VEP molecular-consequence call. ``clingen_allele_id``/``assay_level_digest`` bridge into the annotation dimensions. ``mapped`` is the canonical :class:`MappedTriple`; ``assay_level`` names which of its slots is the measured one. Any field is ``None`` when its source is absent (an unmapped variant → null ``assay_level`` and an empty triple). diff --git a/src/mavedb/lib/vep.py b/src/mavedb/lib/vep.py index 25c0732fa..b02c0ad34 100644 --- a/src/mavedb/lib/vep.py +++ b/src/mavedb/lib/vep.py @@ -1,4 +1,14 @@ -"""VEP (Variant Effect Predictor) library functions for functional consequence prediction.""" +"""VEP (Variant Effect Predictor) resolution and linking for molecular-consequence prediction. + +The resolution itself — VEP querying batched by transcript set, transcript-matched consequence +selection, the Variant Recoder fallback, and how recoded forms combine — is the shared +``variant_annotation.lib.vep`` orchestration, run over an Ensembl REST client. The api keeps no second +copy of that flow, so this pipeline and any other consumer (the lab CLI) resolve identically — the same +answer *and* the same flow for the same input. This module owns only the api-side concerns: running +that (synchronous) orchestration off the worker's event loop on a job-scoped client, the Ensembl release +lookup that version-keys the skip, and writing resolved consequences against deduplicated +:class:`Allele` rows. The worker job composes these; it holds none of the resolution mechanics itself. +""" import asyncio import functools @@ -6,13 +16,25 @@ import os from datetime import date from enum import Enum -from typing import Mapping, NamedTuple, Optional, Sequence +from typing import Mapping, Optional, Sequence +import requests from sqlalchemy import select from sqlalchemy.orm import Session +from variant_annotation.lib.clients.ensembl import EnsemblRestClient +from variant_annotation.lib.vep import ( + RESOLVER_VERSION, + ConsequenceOutcome, + ConsequenceResolution, + ConsequenceSource, + ReferenceSequence, + VepConfig, + VepInput, +) +from variant_annotation.lib.vep import resolve_consequences as resolve_consequences_kernel from mavedb.lib.logging.context import logging_context, save_to_logging_context -from mavedb.lib.utils import request_with_backoff +from mavedb.models.enums.vep import VepConsequenceSource from mavedb.models.vep_allele_consequence import VepAlleleConsequence logger = logging.getLogger(__name__) @@ -20,67 +42,6 @@ ENSEMBL_API_URL = os.environ.get("ENSEMBL_API_URL", "https://rest.ensembl.org") -VEP_CONSEQUENCES = [ - "transcript_ablation", - "splice_acceptor_variant", - "splice_donor_variant", - "stop_gained", - "frameshift_variant", - "stop_lost", - "start_lost", - "transcript_amplification", - "inframe_insertion", - "inframe_deletion", - "missense_variant", - "disruptive_inframe_insertion", - "disruptive_inframe_deletion", - "protein_altering_variant", - "splice_region_variant", - "incomplete_terminal_codon_variant", - "start_retained", - "stop_retained", - "synonymous_variant", - "coding_sequence_variant", - "mature_miRNA_variant", - "5_prime_UTR_premature_start_codon_gain_variant", - "5_prime_UTR_variant", - "3_prime_UTR_variant", - "non_coding_transcript_exon_variant", - "non_coding_exon_variant", - "non_coding_transcript_variant", - "nc_transcript_variant", - "upstream_gene_variant", - "downstream_gene_variant", - "TFBS_ablation", - "TFBS_amplification", - "TF_binding_site_variant", - "regulatory_region_ablation", - "enhancer_ablation", - "regulatory_region_amplification", - "enhancer_amplification", - "regulatory_region_variant", - "feature_elongation", - "regulatory_region", - "TFBS", - "feature_truncation", - "exon_variant", - "disruptive_inframe_deletion", - "gene_variant", - "variant_affecting_coding_sequence_conservation", - "variant_affecting_genome_assembly_quality", - "variant_of_unknown_significance", - "sequence_variant", - "rare_amino_acid_variant", - "splice_region_variant", - "downstream_gene_variant", - "upstream_gene_variant", - "intron_variant", - "intergenic_variant", -] -""" -List of all functional consequences VEP can return, in order of severity (most severe first). -""" - class VepLinkVerdict(str, Enum): """Per-allele outcome of a VEP linking run, returned for every allele whose status is decided. @@ -90,190 +51,134 @@ class VepLinkVerdict(str, Enum): the map had no live consequence and resolved none this run (the caller reads that as "no result"). - ``CREATED`` — a new or changed consequence was created/superseded this run. - - ``UNCHANGED`` — a live consequence was retained (value matched, or held against a null run). + - ``UNCHANGED`` — a live consequence was re-confirmed (VEP resolved the same headline term). + - ``RETAINED_ON_ABSENCE`` — VEP found *nothing* this run, but a prior live consequence was held in + place rather than overwritten with a null. Kept distinct from ``UNCHANGED`` so the caller can + surface it: the allele still has a consequence, but the current Ensembl release did not confirm + it, which may mean a transient miss or a genuine upstream disappearance (transcript retirement). """ CREATED = "created" UNCHANGED = "unchanged" + RETAINED_ON_ABSENCE = "retained_on_absence" -class VepResolution(NamedTuple): - """Outcome of resolving a set of HGVS strings, splitting the two kinds of "no consequence". - - This outcome allows us to differentiate between a genuine empty (VEP found nothing) and an - unknown (VEP failed to answer). - - - ``consequences`` — HGVS that resolved to a most-severe consequence (the hits). - - ``errored`` — HGVS whose VEP/Recoder request *failed* (HTTP/transport error after retries); the - result is unknown and the allele should be retried, not treated as a negative. - - Any queried HGVS in neither set was answered (HTTP 200) with no consequence — a genuine **empty**. - """ - - consequences: dict[str, str] - errored: set[str] - - -async def run_variant_recoder(missing_hgvs: Sequence[str]) -> dict[str, list[str]]: - """Call the Variant Recoder API and return a mapping from input HGVS strings to genomic HGVS strings. - - Args: - missing_hgvs (Sequence[str]): List of HGVS strings to recode. - - Returns: - dict[str, list[str]]: Mapping of input HGVS to list of genomic HGVS strings (hgvsg). An input - with no recodable genomic mapping is simply absent (a genuine empty). +async def get_ensembl_release() -> str: + """Return the current Ensembl release the REST API is serving, e.g. ``"116"`` (``/info/software``). - Raises: - requests.exceptions.RequestException: if the Recoder request fails (HTTP/transport error after - retries). The caller attributes the failure to this batch's inputs so they are reported as - errored (unknown, retry) rather than silently conflated with a genuine empty. + A release is coordinated — software, transcript set, and consequence vocabulary all bump together + under one number — so this single value version-keys VEP results the way gnomAD keys on its data + version. The job stamps it on each consequence and skips alleles already live at the current release + (paired with ``RESOLVER_VERSION``). Delegates to the client's ``software_release`` so the release + lookup shares the resolution transport rather than being a second HTTP path. The client's synchronous + call runs off the event loop. Raises on failure: the version is load-bearing for the skip, so a job + that cannot determine it must abort rather than mis-version its writes. """ - headers = {"Content-Type": "application/json", "Accept": "application/json"} - - # request_with_backoff is synchronous (requests lib + time.sleep backoff); run_in_executor - # keeps the event loop free during the full request + any retry wait time. loop = asyncio.get_running_loop() - response = await loop.run_in_executor( - None, - functools.partial( - request_with_backoff, - method="POST", - url=f"{ENSEMBL_API_URL}/variant_recoder/human", - headers=headers, - json={"ids": list(missing_hgvs)}, - timeout=600, # Variant Recoder can be very slow for large batches and 504s are common; generous timeout and backoff retries are needed - ), - ) - - data = response.json() - # request_with_backoff handles http errors, so no need to check response status - hgvs_to_genomic: dict[str, list[str]] = {} - for input_variant in data: - for variant_str, variant_data in input_variant.items(): - hgvs_string = variant_data.get("input") if isinstance(variant_data, dict) else None - if variant_str == "input" or not hgvs_string: - continue - - genomic_strings = variant_data.get("hgvsg") if isinstance(variant_data, dict) else None - if genomic_strings: - for genomic_hgvs in genomic_strings: - if genomic_hgvs.startswith("NC_"): - hgvs_to_genomic.setdefault(hgvs_string, []).append(genomic_hgvs) - - return hgvs_to_genomic - - -async def get_functional_consequence(hgvs_strings: Sequence[str]) -> dict[str, Optional[str]]: - """Get VEP functional consequences for a batch of HGVS strings. - - Submits HGVS strings to the Ensembl VEP API and retrieves functional consequence - predictions. For any HGVS strings not found in the initial VEP response, attempts - to recode them using Variant Recoder and retries with VEP. - - Args: - hgvs_strings (Sequence[str]): List of HGVS strings to process (max 200 per call). - - Returns: - dict[str, Optional[str]]: Mapping of HGVS string to functional consequence. An HGVS the - successful response carried no consequence for maps to None (a genuine - miss — the caller may try Recoder, else treats it as empty). - - Raises: - requests.exceptions.RequestException: if the VEP request fails (HTTP/transport error after - retries). The caller attributes the failure to this batch's inputs so they are reported as - errored (unknown, retry) rather than silently conflated with a genuine empty. + with requests.Session() as session: + client = EnsemblRestClient(api_url=ENSEMBL_API_URL, session=session) + return await loop.run_in_executor(None, client.software_release) + + +async def resolve_consequences( + inputs: Sequence[VepInput], *, reference: Optional[ReferenceSequence] = None +) -> dict[str, ConsequenceResolution]: + """Resolve each input to a molecular consequence, keyed by its HGVS. One resolution per input. + + Delegates the full resolution flow to :func:`variant_annotation.lib.vep.resolve_consequences`, run + over an :class:`EnsemblRestClient` that satisfies both the VEP-lookup and Variant-Recoder ports: VEP + querying (batched by transcript set, resolved against each input's own transcript) and the Recoder + fallback for misses. The api keeps no second copy of that flow. + + ``reference`` is the optional transcript-reference port that lights up reference-identical detection: + a no-change input (an explicit ``c.=`` form, or a ``delins`` whose replacement bases equal what the + transcript already reads) resolves to ``REFERENCE_IDENTICAL`` rather than ``ABSENT``. It is consulted + only for inputs VEP and the Recoder both left unresolved — the wild-type controls and unparseables — + so it costs nothing for the inputs VEP resolves normally. When omitted, those inputs stay ``ABSENT``. + + The library call is synchronous; we run it in one executor call so the worker's event loop stays + free. It resolves serially (``VepConfig`` default ``max_workers=1``) to stay within Ensembl's + per-client rate limit. There is no per-input progress hook, so a caller reporting progress on a long + run chunks its inputs and awaits this once per chunk. + + The Ensembl client runs on a session we own and close when the call finishes, so its pooled + connections do not linger idle; the ``reference`` port's lifecycle is the caller's. Every input + appears in the result exactly once, with outcome ``RESOLVED`` / ``ABSENT`` / ``ERRORED`` — the caller + must retry ``ERRORED`` (unknown), never store it as a genuine empty. """ - if len(hgvs_strings) > 200: - raise ValueError( - "VEP API can process a maximum of 200 HGVS strings per request. This function does not handle batching." - ) - - headers = {"Content-Type": "application/json", "Accept": "application/json"} - result: dict[str, Optional[str]] = {} + if not inputs: + return {} - # request_with_backoff is synchronous (requests lib + time.sleep backoff); run_in_executor - # keeps the event loop free during the full request + any retry wait time. loop = asyncio.get_running_loop() - response = await loop.run_in_executor( - None, - functools.partial( - request_with_backoff, - method="POST", - url=f"{ENSEMBL_API_URL}/vep/human/hgvs", - headers=headers, - json={"hgvs_notations": list(hgvs_strings)}, - timeout=60, # VEP can be slow for large batches. - ), - ) - - data = response.json() - for entry in data: - hgvs = entry.get("input") - most_severe_consequence = entry.get("most_severe_consequence") - if hgvs: - result[hgvs] = most_severe_consequence + with requests.Session() as session: + client = EnsemblRestClient(api_url=ENSEMBL_API_URL, session=session) + resolutions = await loop.run_in_executor( + None, + functools.partial( + resolve_consequences_kernel, + inputs, + vep=client, + recoder=client, + reference=reference, + config=VepConfig(), + ), + ) + return {resolution.input.hgvs: resolution for resolution in resolutions} - return result - -async def get_ensembl_release() -> str: - """Return the current Ensembl release the REST API is serving, e.g. ``"116"`` (``/info/software``). - - An Ensembl release is coordinated — software, transcript set, and consequence vocabulary all bump - together under one number — so this single value version-keys VEP results the way gnomAD keys on its - data version. The job stamps it on each consequence and skips re-querying alleles already live at the - current release. Raises on failure: the version is load-bearing for the skip, so a job that cannot - determine it must abort rather than mis-version its writes. - """ - headers = {"Content-Type": "application/json", "Accept": "application/json"} - loop = asyncio.get_running_loop() - response = await loop.run_in_executor( - None, - functools.partial( - request_with_backoff, - method="GET", - url=f"{ENSEMBL_API_URL}/info/software", - headers=headers, - timeout=30, - ), - ) - return str(response.json()["release"]) +def _to_db_source(source: Optional[ConsequenceSource]) -> Optional[VepConsequenceSource]: + """Map the kernel's resolution source onto the api's persisted enum (identical values, distinct owners).""" + return VepConsequenceSource(source.value) if source is not None else None def link_vep_consequences_to_alleles( db: Session, - consequence_by_allele_id: Mapping[int, Optional[str]], + resolutions_by_allele_id: Mapping[int, ConsequenceResolution], *, source_version: str, access_date: date, ) -> dict[int, VepLinkVerdict]: - """Store VEP consequences against deduplicated alleles, superseding only on change. - - ``consequence_by_allele_id`` maps each queried allele to the consequence VEP resolved this run - (``None`` when VEP + Variant Recoder found nothing). ``source_version`` is the Ensembl release the - run resolved against. Each allele holds at most one live :class:`VepAlleleConsequence`, handled per - allele: - - - **unchanged** (live row already carries this consequence): advance ``source_version`` and - ``access_date`` in place — no supersede. Supersede is value-keyed, not version-keyed: a new - release that resolves the same categorical consequence must not fabricate a transaction-time - boundary, which would churn history every release. - - **new or changed** (no live row, or a different consequence): supersede keyed on ``allele_id`` - (retire the old, insert the successor stamped with this ``source_version``/``access_date``). - - **None this run**: leave any live row in place — do not overwrite a held consequence with a null result. - Log a warning if VEP found no consequence for an allele which previously had a live consequence. + """Store VEP consequences against deduplicated alleles, superseding only on a headline-term change. + + ``resolutions_by_allele_id`` maps each queried allele to the kernel's :class:`ConsequenceResolution`. + Only ``RESOLVED`` and ``ABSENT`` resolutions belong here — an ``ERRORED`` allele's failure is + recorded in the annotation event stream and it is retried, never linked, so passing one is a caller + bug and is ignored defensively. ``source_version`` is the Ensembl release the run resolved against. + Each allele holds at most one live :class:`VepAlleleConsequence`, handled per allele: + + - **unchanged** (live row already carries this headline consequence): advance ``source_version``, + ``resolver_version``, ``access_date``, and the provenance columns (``consequence_terms``/ + ``consequence_source``/``matched_transcript``) in place — no supersede. Supersede is keyed on the + headline term, not the release, resolver version, or provenance: a re-run that resolves the same + term must not fabricate a transaction-time boundary, and filling in a version/provenance column + for an already-held term (e.g. a pre-#772 row with NULL provenance, or a row predating + ``resolver_version``) is not a value change either. + - **new or changed** (no live row, or a different headline term): supersede keyed on ``allele_id`` + (retire the old, insert the successor stamped with this run's version + provenance). + - **absent this run** (``ABSENT`` outcome) with a live row: leave it in place — do not overwrite a + held consequence with a null result — and return :attr:`~VepLinkVerdict.RETAINED_ON_ABSENCE`. + This is loudly logged (per allele and in aggregate) because it can signal a genuine upstream + disappearance, not just a transient miss; the caller surfaces the aggregate in job metadata. + - **absent this run** with no live row: nothing to do; the allele is left out of the returned map. Does not commit. Returns a verdict per allele whose status is decided this run: - :attr:`VepLinkVerdict.CREATED` for a created/superseded consequence, :attr:`~VepLinkVerdict.UNCHANGED` - for a live consequence retained (value matched, or held against a null run). An allele absent from - the map had no live row and resolved nothing — the caller reads that as "no result". This is the - single source of truth for per-allele status; callers must not re-derive it from consequence state. + :attr:`~VepLinkVerdict.CREATED` (created/superseded), :attr:`~VepLinkVerdict.UNCHANGED` (a live + consequence re-confirmed), or :attr:`~VepLinkVerdict.RETAINED_ON_ABSENCE` (VEP found nothing, prior + consequence held). An allele absent from the returned map had no live row and resolved nothing — the + caller reads that as "no result". This is the single source of truth for per-allele status; callers + must not re-derive it from consequence state. """ - save_to_logging_context({"num_alleles_to_link_vep": len(consequence_by_allele_id)}) + save_to_logging_context({"num_alleles_to_link_vep": len(resolutions_by_allele_id)}) logger.debug(msg="Linking VEP consequences to alleles", extra=logging_context()) verdicts: dict[int, VepLinkVerdict] = {} - for allele_id, consequence in consequence_by_allele_id.items(): + for allele_id, resolution in resolutions_by_allele_id.items(): + if resolution.outcome is ConsequenceOutcome.ERRORED: + # Failures are not this table's business; the caller records them in the event stream. + continue + + consequence = resolution.most_severe_consequence if resolution.outcome is ConsequenceOutcome.RESOLVED else None + live = db.scalar( select(VepAlleleConsequence).where( VepAlleleConsequence.allele_id == allele_id, @@ -281,35 +186,45 @@ def link_vep_consequences_to_alleles( ) ) - # TODO#780 - VEP found nothing this run. Do not overwrite a held consequence with a null result; a retained - # consequence is UNCHANGED (status preexisting), while no live row at all leaves the allele out of - # the map (the caller reads that as a no-result). + # VEP found nothing this run. Do not overwrite a held consequence with a null result; a retained + # consequence is RETAINED_ON_ABSENCE (surfaced), while no live row at all leaves the allele out + # of the map (the caller reads that as a no-result). if consequence is None: if live is not None: logger.warning( - f"VEP found no consequence for allele {allele_id} this run; leaving prior consequence " - f"'{live.functional_consequence}' in place.", + f"VEP returned NO consequence for allele {allele_id} (release {source_version}) that " + f"previously resolved to '{live.functional_consequence}'; retaining the prior value " + f"(not superseding). Investigate if this recurs — it can mean transcript retirement or " + f"an upstream Ensembl change, not a transient miss.", extra=logging_context(), ) - verdicts[allele_id] = VepLinkVerdict.UNCHANGED + verdicts[allele_id] = VepLinkVerdict.RETAINED_ON_ABSENCE continue - # Unchanged: advance version/freshness in place. + # Unchanged headline term: advance version/freshness and refresh provenance in place. if live is not None and live.functional_consequence == consequence: live.source_version = source_version + live.resolver_version = RESOLVER_VERSION live.access_date = access_date + live.consequence_terms = list(resolution.consequence_terms) + live.consequence_source = _to_db_source(resolution.source) + live.matched_transcript = resolution.matched_transcript verdicts[allele_id] = VepLinkVerdict.UNCHANGED continue - # New or changed consequence: retire any live row, insert the successor. + # New or changed consequence: retire any live row, insert the successor with full provenance. VepAlleleConsequence.supersede_live_where( db, [ VepAlleleConsequence( allele_id=allele_id, functional_consequence=consequence, + consequence_terms=list(resolution.consequence_terms), + consequence_source=_to_db_source(resolution.source), + matched_transcript=resolution.matched_transcript, source_version=source_version, + resolver_version=RESOLVER_VERSION, access_date=access_date, ) ], @@ -318,9 +233,21 @@ def link_vep_consequences_to_alleles( verdicts[allele_id] = VepLinkVerdict.CREATED changed_allele_count = sum(1 for v in verdicts.values() if v is VepLinkVerdict.CREATED) - save_to_logging_context({"changed_allele_count": changed_allele_count}) + retained_on_absence_count = sum(1 for v in verdicts.values() if v is VepLinkVerdict.RETAINED_ON_ABSENCE) + save_to_logging_context( + {"changed_allele_count": changed_allele_count, "retained_on_absence_count": retained_on_absence_count} + ) logger.info( msg=f"Created or superseded {changed_allele_count} VEP allele consequences this run.", extra=logging_context(), ) + if retained_on_absence_count: + # Aggregate signal the job lifts into its outcome metadata: a large or growing count means VEP + # is now returning nothing for alleles that used to resolve — worth a look, not just a log line. + logger.warning( + msg=f"VEP returned no consequence for {retained_on_absence_count} allele(s) that previously had one; " + f"prior consequences were RETAINED, not superseded. A large or growing count may indicate " + f"transcript retirement or an upstream Ensembl data change.", + extra=logging_context(), + ) return verdicts diff --git a/src/mavedb/view_models/allele_annotation.py b/src/mavedb/view_models/allele_annotation.py index a995fabcb..995d16088 100644 --- a/src/mavedb/view_models/allele_annotation.py +++ b/src/mavedb/view_models/allele_annotation.py @@ -13,7 +13,7 @@ class VepAnnotation(BaseModel): - """VEP most-severe functional consequence and the Ensembl release it resolved under.""" + """VEP molecular consequence for an allele and the Ensembl release it resolved under.""" consequence: Optional[str] = None source_version: str diff --git a/src/mavedb/worker/jobs/external_services/vep.py b/src/mavedb/worker/jobs/external_services/vep.py index 0264938f4..a6195d1b6 100644 --- a/src/mavedb/worker/jobs/external_services/vep.py +++ b/src/mavedb/worker/jobs/external_services/vep.py @@ -1,46 +1,104 @@ -"""VEP functional consequence jobs for variant effect prediction. +"""VEP molecular-consequence job for variant effect prediction. -This module links deduplicated alleles to their Ensembl VEP functional consequence. Submission is -batched against the VEP API, with a Variant Recoder fallback for HGVS strings VEP cannot resolve -directly (notably protein HGVS). +Links deduplicated alleles to their Ensembl VEP molecular consequence. The job owns the lifecycle — +selecting which alleles to annotate, the current-release skip, linking, and status events — while the +resolution itself (transport, batching, Recoder fallback, the transcript-matching rule) lives in +``mavedb.lib.vep`` and the shared ``variant_annotation.lib.vep`` kernel, so the pipeline and the lab +CLI produce identical consequences for the same input (#772). """ -import asyncio +import contextlib import logging -import os from collections import Counter from datetime import date +from typing import Iterator, Optional -import requests from sqlalchemy import select +from variant_annotation.lib.vep import ( + RESOLVER_VERSION, + ConsequenceOutcome, + ConsequenceResolution, + ReferenceSequence, + VepInput, +) from mavedb.lib.annotation_status_manager import AnnotationStatusManager from mavedb.lib.clingen.alleles import ScoreSetAlleleRow, get_alleles_for_score_set, group_alleles_for_annotation from mavedb.lib.types.workflow import JobExecutionOutcome from mavedb.lib.utils import batched from mavedb.lib.vep import ( - VEP_CONSEQUENCES, VepLinkVerdict, - VepResolution, get_ensembl_release, - get_functional_consequence, link_vep_consequences_to_alleles, - run_variant_recoder, + resolve_consequences, ) from mavedb.models.enums.annotation_type import AnnotationType from mavedb.models.enums.disposition import Disposition from mavedb.models.enums.event_reason import EventReason +from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.models.score_set import ScoreSet from mavedb.models.vep_allele_consequence import VepAlleleConsequence from mavedb.worker.jobs.utils.setup import validate_job_params from mavedb.worker.lib.decorators.pipeline_management import with_pipeline_management from mavedb.worker.lib.managers.job_manager import JobManager +from mavedb.worker.lib.translation_ports import uta_transcript_source logger = logging.getLogger(__name__) -_VEP_BATCH_SIZE = 200 -_RECODER_BATCH_SIZE = int(os.getenv("RECODER_BATCH_SIZE", "25")) -_RECODER_CONCURRENCY = int(os.getenv("RECODER_CONCURRENCY", "5")) +# Inputs resolved per await so progress can be reported between chunks. Matches the kernel's own VEP +# batch ceiling (200 HGVS/request), so each chunk is ~one wire batch. Progress must be reported from +# this coroutine, never from inside resolve_consequences: that call runs on an executor thread and the +# progress commit touches this job's DB session, which is not thread-safe. +_RESOLUTION_CHUNK_SIZE = 200 + + +class _BestEffortReference: + """Wraps the UTA transcript-reference port so a mid-run failure disables reference-identical + detection for the affected input (returns ``None``) instead of failing the VEP job. + + Reference-identical labelling is a refinement for wild-type controls; it must never take down + resolution. A lookup that raises (a dropped UTA connection, a query error) is logged once and + treated as "cannot decide", so the input falls back to ``ABSENT`` exactly as it would with no + reference port. Runs on the resolution executor thread, so it logs without ``extra`` context. + """ + + def __init__(self, inner: ReferenceSequence) -> None: + self._inner = inner + self._warned = False + + def coding_interval_reference(self, transcript: str, start: int, stop: int) -> Optional[str]: + try: + return self._inner.coding_interval_reference(transcript, start, stop) + except Exception as exc: # noqa: BLE001 - any UTA failure degrades to no reference-identical label + if not self._warned: + logger.warning("UTA reference lookup failed mid-run; reference-identical detection degraded: %s", exc) + self._warned = True + return None + + +@contextlib.contextmanager +def _reference_source_for_vep(job_manager: JobManager) -> Iterator[Optional[ReferenceSequence]]: + """Yield a transcript-reference port for reference-identical detection, or ``None`` when UTA is + unavailable. + + Best-effort by design: VEP resolution must not fail because UTA is unset or unreachable. The port is + consulted only for no-change controls and unparseables — everything VEP resolves normally is + untouched — so without it those inputs simply fall back to ``ABSENT`` rather than being labelled + ``REFERENCE_IDENTICAL``. An open-time failure is caught here; a mid-run failure is caught by + :class:`_BestEffortReference`. + """ + with contextlib.ExitStack() as stack: + try: + source = stack.enter_context(uta_transcript_source()) + except Exception as exc: # noqa: BLE001 - opening UTA failed; run without reference-identical + logger.warning( + msg=f"UTA transcript source unavailable; reference-identical detection disabled for this run " + f"(wild-type controls resolve to ABSENT). Cause: {exc}", + extra=job_manager.logging_context(), + ) + yield None + return + yield _BestEffortReference(source) def _annotate_vep( @@ -74,167 +132,57 @@ def _annotate_vep( ) -def _vep_hgvs_payload(row: ScoreSetAlleleRow) -> str | None: - """Build VEP's HGVS input for an allele: each allele will only have one of these, so the first - non-null is the one to submit. - """ - return row.hgvs_g or row.hgvs_c or row.hgvs_p or None - +def _vep_input_for_allele(row: ScoreSetAlleleRow) -> VepInput | None: + """Build a VEP input for an allele, or ``None`` to skip it. -async def _resolve_consequences(unique_hgvs: list[str], job_manager: JobManager) -> VepResolution: - """Resolve a set of HGVS strings to their most-severe VEP consequence, splitting empty from errored. + Only coding and genomic alleles are submitted to VEP: a protein allele's consequence is carried + instead by the coding alleles reverse translation enumerates from it, so protein alleles are never + sent. Each allele carries exactly one of ``hgvs_g``/``hgvs_c``/``hgvs_p``. - Phase 1 submits the HGVS strings to VEP. Phase 2 runs Variant Recoder on the misses (a VEP entry - with a null consequence is treated as a miss — VEP knew the variant but could not classify it). - Phase 3 re-submits the recoded genomic strings to VEP and maps the most-severe consequence back to - the original HGVS. - - Returns a :class:`VepResolution`: ``consequences`` for the hits, and ``errored`` for HGVS whose VEP - or Recoder *request failed* (HTTP/transport error after retries) — those are unknown and must be - retried, never conflated with a genuine empty. An input in neither set was queried successfully and - yielded no consequence (a genuine empty). A failed batch does not abort the run: only that batch's - inputs are marked errored and processing continues. + No transcript is set on the input. A coding allele carries its transcript inside the ``c.`` HGVS + accession, which the kernel infers, so it resolves against its own transcript (the #772 fix). A + genomic allele carries only an ``NC_`` chromosome accession, so it resolves to VEP's + cross-transcript ``most_severe`` headline — deliberately, not as a gap. Alleles are deduplicated + and content-addressed, so one genomic allele is shared across every score set that maps to it; + there is no single "assay transcript" to attribute to it, and borrowing the coding transcript from + one arbitrary pairing would be less faithful than the transcript-agnostic ``most_severe`` call. + ``most_severe`` is the honest reconstruction for a genomic allele, and its + ``consequence_source='most_severe'`` records that provenance so consumers can tell it apart from a + transcript-matched call. """ - all_consequences: dict[str, str] = {} - errored: set[str] = set() - batches = list(batched(unique_hgvs, _VEP_BATCH_SIZE)) + if row.level == SequenceLevel.protein.value: + return None - # --- Phase 1: initial VEP pass --- - all_missing_hgvs: set[str] = set() - for batch_idx, batch in enumerate(batches): - try: - consequences = await get_functional_consequence(list(batch)) - # VEP rejected the batch (e.g. 400 for protein HGVS it cannot parse). Route all - # items to Variant Recoder — the input may still be resolvable via genomic recoding. - except requests.exceptions.HTTPError: - logger.warning( - msg=f"VEP returned an HTTP error for batch {batch_idx + 1}/{len(batches)} ({len(batch)} HGVS); routing to Variant Recoder.", - extra=job_manager.logging_context(), - ) - all_missing_hgvs.update(batch) - continue - # Transport/network error — result unknown. Mark errored and do not route to Recoder. - except requests.exceptions.RequestException as exc: - logger.warning( - msg=f"VEP request failed for batch {batch_idx + 1}/{len(batches)} ({len(batch)} HGVS); marking errored.", - exc_info=exc, - extra=job_manager.logging_context(), - ) - errored.update(batch) - continue + hgvs = row.hgvs_c or row.hgvs_g + if not hgvs: + return None - hit_consequences = {h: c for h, c in consequences.items() if c is not None} - all_consequences.update(hit_consequences) - all_missing_hgvs.update(set(batch) - set(hit_consequences.keys())) - - job_manager.update_progress( - int((batch_idx + 1) / len(batches) * 33), - 100, - f"Processed initial VEP batch {batch_idx + 1}/{len(batches)}", - ) - - logger.info( - msg=f"Completed initial VEP processing. {len(all_missing_hgvs)} HGVS strings require Variant Recoder fallback.", - extra=job_manager.logging_context(), - ) - - if not all_missing_hgvs: - return VepResolution(all_consequences, errored) - - # --- Phase 2: Variant Recoder fallback for HGVS strings VEP could not resolve --- - recoder_batch_list = list(batched(list(all_missing_hgvs), _RECODER_BATCH_SIZE)) - semaphore = asyncio.Semaphore(_RECODER_CONCURRENCY) - completed_recoder_batches = 0 - - async def _recoder_with_semaphore(batch: list[str], total: int) -> dict[str, list[str]]: - nonlocal completed_recoder_batches - async with semaphore: - result = await run_variant_recoder(batch) - completed_recoder_batches += 1 - job_manager.update_progress( - 33 + int(completed_recoder_batches / total * 33), - 100, - f"Completed Variant Recoder batch {completed_recoder_batches}/{total}", - ) - return result - - total_recoder_batches = len(recoder_batch_list) - recoder_results = await asyncio.gather( - *[_recoder_with_semaphore(list(b), total_recoder_batches) for b in recoder_batch_list], - return_exceptions=True, - ) - - # A failed Recoder batch marks its inputs errored (they were misses; we still don't know them) and - # the run continues — one bad batch must not abort every allele's annotation. - hgvs_to_genomic: dict[str, list[str]] = {} - for batch, result in zip(recoder_batch_list, recoder_results): - if isinstance(result, BaseException): - logger.warning( - msg=f"Variant Recoder request failed for a batch of {len(batch)} HGVS; marking errored.", - exc_info=result, - extra=job_manager.logging_context(), - ) - errored.update(batch) - continue - hgvs_to_genomic.update(result) - - logger.info( - msg=f"Completed Variant Recoder processing. {len(hgvs_to_genomic)} HGVS strings successfully recoded.", - extra=job_manager.logging_context(), - ) - - # --- Phase 3: VEP pass on the recoded genomic HGVS strings --- - all_recoded_genomic_hgvs = list({g for genomic_list in hgvs_to_genomic.values() for g in genomic_list}) - recoded_vep_batch_list = list(batched(all_recoded_genomic_hgvs, _VEP_BATCH_SIZE)) - all_recoded_consequences: dict[str, str | None] = {} - errored_genomic: set[str] = set() - - for recoded_idx, recoded_batch in enumerate(recoded_vep_batch_list): - try: - all_recoded_consequences.update(await get_functional_consequence(list(recoded_batch))) - except requests.exceptions.RequestException as exc: - logger.warning( - msg=f"VEP request failed for recoded batch {recoded_idx + 1}/{len(recoded_vep_batch_list)} " - f"({len(recoded_batch)} HGVS); marking errored.", - exc_info=exc, - extra=job_manager.logging_context(), - ) - errored_genomic.update(recoded_batch) - job_manager.update_progress( - 66 + int((recoded_idx + 1) / len(recoded_vep_batch_list) * 33), - 100, - f"Processed recoded VEP batch {recoded_idx + 1}/{len(recoded_vep_batch_list)}", - ) - - # Map the most-severe consequence from the recoded genomic strings back to the original HGVS. An - # original HGVS is a hit if any recoded form resolved; else errored if a recoded query failed and - # nothing resolved; else a genuine empty (Recoder ran, VEP found no consequence) — left out of both. - for original_hgvs, recoded_hgvs_list in hgvs_to_genomic.items(): - recoded_consequences = [c for h in recoded_hgvs_list if (c := all_recoded_consequences.get(h))] - most_severe = next((c for c in VEP_CONSEQUENCES if c in recoded_consequences), None) - if most_severe: - all_consequences[original_hgvs] = most_severe - elif any(g in errored_genomic for g in recoded_hgvs_list): - errored.add(original_hgvs) - - return VepResolution(all_consequences, errored) + return VepInput(hgvs=hgvs) @with_pipeline_management async def populate_vep_for_score_set(ctx: dict, job_id: int, job_manager: JobManager) -> JobExecutionOutcome: - """Link deduplicated alleles to their VEP functional consequence. + """Link deduplicated alleles to their VEP molecular consequence. + + Runs over the score set's current coding and genomic alleles (authoritative and RT-derived), + resolves each against its own transcript via the shared kernel (with a Variant Recoder fallback), + and stores the consequence plus its resolution provenance in a valid-time + :class:`VepAlleleConsequence`, superseding only on a changed headline term. Protein alleles are not + submitted — their consequence is carried by the coding alleles reverse translation enumerates. - Runs over the score set's current alleles (authoritative and RT-derived), submits each allele's - HGVS to VEP (with Variant Recoder fallback), and stores the most-severe consequence in a valid-time - :class:`VepAlleleConsequence`, superseding only on change. + Wild-type controls (an input describing no sequence change) are resolved to a ``reference_identical`` + consequence via a best-effort UTA transcript-reference port, so a control is stored distinctly rather + than dropped as a no-result. If UTA is unavailable the run proceeds without that label (those alleles + fall back to absent); see :func:`_reference_source_for_vep`. Job Parameters: - score_set_id (int): The ID of the ScoreSet whose alleles to process. - correlation_id (str): Correlation ID for tracing requests across services. - - force (bool, optional): Bypass the current-release skip and re-query every HGVS-bearing - allele. The linker still supersedes only on a value change, so a forced re-run of unchanged - data writes no new rows. Use for re-ingestion, to heal suspected corruption, or after editing - the VEP_CONSEQUENCES severity ordering (a change the release version cannot see). + - force (bool, optional): Bypass the current-release/resolver skip and re-query every eligible + allele. The linker still supersedes only on a headline-term change, so a forced re-run of + unchanged data writes no new rows. Use for re-ingestion or to heal suspected corruption. A + resolution-rule change no longer needs force: it bumps RESOLVER_VERSION, which the skip keys + on, so those alleles re-query on the next ordinary run. Args: ctx (dict): The job context dictionary. @@ -242,7 +190,9 @@ async def populate_vep_for_score_set(ctx: dict, job_id: int, job_manager: JobMan job_manager (JobManager): Manager for job lifecycle and DB operations. Returns: - JobExecutionOutcome: outcome with per-allele created/preexisting/skipped counts. + JobExecutionOutcome: outcome with per-allele created/preexisting/absent/errored counts, plus + ``retained_on_absence_count`` — preexisting alleles whose consequence was kept because VEP + returned nothing this run (a genuine-disappearance signal, surfaced in the job metadata). """ job = job_manager.get_job() @@ -265,11 +215,11 @@ async def populate_vep_for_score_set(ctx: dict, job_id: int, job_manager: JobMan job_manager.update_progress(0, 100, "Starting VEP consequence linkage.") logger.info(msg="Started VEP consequence linkage", extra=job_manager.logging_context()) - # One work-unit per allele (payload = HGVS; alleles without one are skipped). Events are allele-keyed, - # so each allele records its own event. - allele_data = group_alleles_for_annotation( + # One work-unit per eligible allele (payload = a transcript-aware VepInput; protein alleles and + # alleles without HGVS are skipped). Events are allele-keyed, so each allele records its own event. + allele_inputs = group_alleles_for_annotation( get_alleles_for_score_set(job_manager.db, score_set.id), - payload=_vep_hgvs_payload, + payload=_vep_input_for_allele, ) annotation_counts: Counter[str] = Counter( @@ -278,21 +228,25 @@ async def populate_vep_for_score_set(ctx: dict, job_id: int, job_manager: JobMan "preexisting_allele_count": 0, "absent_allele_count": 0, "errored_allele_count": 0, + # A subset of preexisting: alleles that kept a prior consequence because VEP returned nothing + # this run. Surfaced in the job outcome so a genuine-disappearance pattern is visible without + # digging through logs (see VepLinkVerdict.RETAINED_ON_ABSENCE). + "retained_on_absence_count": 0, } ) - num_alleles_with_hgvs = len(allele_data) + num_alleles_with_hgvs = len(allele_inputs) job_manager.save_to_context({"num_alleles_to_link_vep": num_alleles_with_hgvs}) - if not allele_data: + if not allele_inputs: logger.warning( - msg="No current alleles with HGVS were found for this score set. Skipping VEP linkage (nothing to do).", + msg="No current coding/genomic alleles with HGVS were found for this score set. Skipping VEP linkage (nothing to do).", extra=job_manager.logging_context(), ) job_manager.db.flush() return JobExecutionOutcome.succeeded(data=dict(annotation_counts)) - all_allele_ids = set(allele_data.keys()) + all_allele_ids = set(allele_inputs.keys()) # The Ensembl release version-keys the run (coordinated software + transcript set + vocabulary). It # is load-bearing for the skip below, so a failure here aborts the job rather than mis-versioning @@ -301,7 +255,8 @@ async def populate_vep_for_score_set(ctx: dict, job_id: int, job_manager: JobMan job_manager.save_to_context({"ensembl_release": ensembl_release}) def alleles_at_current_release(allele_ids: set[int]) -> set[int]: - """Allele ids (within the given set) holding a live VEP consequence at the current Ensembl release.""" + """Allele ids (within the given set) holding a live VEP consequence at the current Ensembl release + *and* the current resolver version — both axes must match to count as up to date.""" if not allele_ids: return set() return set( @@ -311,39 +266,78 @@ def alleles_at_current_release(allele_ids: set[int]) -> set[int]: .where(VepAlleleConsequence.current) .where(VepAlleleConsequence.functional_consequence.isnot(None)) .where(VepAlleleConsequence.source_version == ensembl_release) + # A resolution-rule change bumps RESOLVER_VERSION without an Ensembl release bump; gating + # on both axes re-queries those alleles automatically instead of them looking current + # forever. A NULL resolver_version (pre-column row) never matches and so is re-queried. + .where(VepAlleleConsequence.resolver_version == RESOLVER_VERSION) ).all() ) - # Skip alleles already resolved at the current Ensembl release (they cannot change without a - # release bump). force re-queries all — including alleles unchanged upstream but whose VEP_CONSEQUENCES - # severity ordering we have since edited; the linker still supersedes only on a value change, so a - # forced no-op writes nothing. + # Skip alleles already resolved at the current Ensembl release AND resolver version (they cannot + # change without one of those bumping). force re-queries all regardless — for re-ingestion or to heal + # suspected corruption. The linker still supersedes only on a value change, so a forced (or + # resolver-bumped) no-op advances the version in place and writes no new history row. already_current = set() if force else alleles_at_current_release(all_allele_ids) - hgvs_by_allele = {aid: allele_data[aid] for aid in allele_data if aid not in already_current} - unique_hgvs = sorted(set(hgvs_by_allele.values())) + inputs_by_allele = {aid: allele_inputs[aid] for aid in allele_inputs if aid not in already_current} + # HGVS alone identifies a unique resolution question here: this job never sets VepInput.transcript + # (a coding allele's transcript rides inside its c. HGVS; a genomic allele is transcript-agnostic — + # see _vep_input_for_allele), and resolve_consequences re-keys its result by HGVS to match. If this + # job ever passes explicit transcripts, both this dedup key and that return keying must become + # (hgvs, transcript), or same-HGVS/different-transcript inputs collapse into one. + unique_inputs = list({i.hgvs: i for i in inputs_by_allele.values()}.values()) job_manager.save_to_context( { "num_alleles_already_current": len(already_current), - "num_hgvs_to_query": len(unique_hgvs), + "num_hgvs_to_query": len(unique_inputs), "force": force, } ) verdicts: dict[int, VepLinkVerdict] = {} errored_allele_ids: set[int] = set() - if unique_hgvs: - job_manager.update_progress(10, 100, f"Querying VEP for {len(unique_hgvs)} HGVS strings.") - resolution = await _resolve_consequences(unique_hgvs, job_manager) - consequence_by_allele_id = {aid: resolution.consequences.get(hgvs) for aid, hgvs in hgvs_by_allele.items()} - # Alleles whose VEP/Recoder request failed: unknown, not a negative — kept distinct from empties. - errored_allele_ids = {aid for aid, hgvs in hgvs_by_allele.items() if hgvs in resolution.errored} + if unique_inputs: + # The shared orchestration exposes no per-input progress hook, so resolve in chunks and report + # between them. Progress is emitted here on the coroutine, not from inside resolve_consequences: + # that runs on an executor thread and the progress commit writes this job's (non-thread-safe) DB + # session. Chunk-level granularity is the resolution; finer would require a kernel progress hook. + job_manager.update_progress(10, 100, f"Resolving VEP consequences for {len(unique_inputs)} HGVS strings.") + resolutions_by_hgvs: dict[str, ConsequenceResolution] = {} + chunks = list(batched(unique_inputs, _RESOLUTION_CHUNK_SIZE)) + # A no-change control (an explicit c.= form, or a delins whose bases equal the transcript's own) + # is not a VEP-resolvable variant; the reference port lets it resolve to REFERENCE_IDENTICAL + # instead of vanishing into ABSENT. Best-effort: a missing/down UTA just disables that label. + with _reference_source_for_vep(job_manager) as reference: + for chunk_idx, chunk in enumerate(chunks): + resolutions_by_hgvs.update(await resolve_consequences(list(chunk), reference=reference)) + job_manager.update_progress( + 10 + int((chunk_idx + 1) / len(chunks) * 85), + 100, + f"Resolved VEP consequences for batch {chunk_idx + 1}/{len(chunks)}.", + ) + + # Every queried input gets a resolution. An allele missing a resolution is treated as genuinely empty + # and never a failure. This ensures a dropped input can only under-report, never overwrite a held consequence. + resolution_by_allele = { + aid: resolutions_by_hgvs.get(inp.hgvs) + or ConsequenceResolution(input=inp, outcome=ConsequenceOutcome.ABSENT) + for aid, inp in inputs_by_allele.items() + } + + # Alleles whose VEP/Recoder request failed: unknown, not a negative — kept distinct from a genuine + # empty. Never linked, since that would overwrite a held consequence with a failure. + errored_allele_ids = { + aid for aid, res in resolution_by_allele.items() if res.outcome is ConsequenceOutcome.ERRORED + } + linkable = { + aid: res for aid, res in resolution_by_allele.items() if res.outcome is not ConsequenceOutcome.ERRORED + } verdicts = link_vep_consequences_to_alleles( - job_manager.db, consequence_by_allele_id, source_version=ensembl_release, access_date=date.today() + job_manager.db, linkable, source_version=ensembl_release, access_date=date.today() ) job_manager.db.flush() else: logger.info( - msg="All HGVS-bearing alleles are already resolved at the current Ensembl release; skipping VEP query.", + msg="All eligible alleles are already resolved at the current Ensembl release; skipping VEP query.", extra=job_manager.logging_context(), ) job_manager.update_progress(99, 100, "All alleles already current at this Ensembl release.") @@ -351,7 +345,7 @@ def alleles_at_current_release(allele_ids: set[int]) -> set[int]: annotation_manager = AnnotationStatusManager( job_manager.db, job_run_id=job_manager.job_id, score_set_id=score_set.id ) - for allele_id, hgvs in allele_data.items(): + for allele_id, vep_input in allele_inputs.items(): verdict = verdicts.get(allele_id) if verdict is VepLinkVerdict.CREATED: annotation_counts["created_allele_count"] += 1 @@ -361,18 +355,25 @@ def alleles_at_current_release(allele_ids: set[int]) -> set[int]: Disposition.PRESENT, EventReason.CREATED, source_version=ensembl_release, - metadata={"hgvs": hgvs}, + metadata={"hgvs": vep_input.hgvs}, ) - elif allele_id in already_current or verdict is VepLinkVerdict.UNCHANGED: + elif allele_id in already_current or verdict in ( + VepLinkVerdict.UNCHANGED, + VepLinkVerdict.RETAINED_ON_ABSENCE, + ): + # The allele still has a live consequence, so its status is preexisting either way. When it + # was retained despite VEP finding nothing this run, also tally it separately for the outcome. annotation_counts["preexisting_allele_count"] += 1 + if verdict is VepLinkVerdict.RETAINED_ON_ABSENCE: + annotation_counts["retained_on_absence_count"] += 1 _annotate_vep( annotation_manager, allele_id, Disposition.PRESENT, EventReason.PREEXISTING, source_version=ensembl_release, - metadata={"hgvs": hgvs}, + metadata={"hgvs": vep_input.hgvs}, ) elif allele_id in errored_allele_ids: @@ -384,7 +385,7 @@ def alleles_at_current_release(allele_ids: set[int]) -> set[int]: EventReason.API_ERROR, source_version=ensembl_release, error_message="The VEP/Variant Recoder request for this allele failed; result unknown.", - metadata={"hgvs": hgvs}, + metadata={"hgvs": vep_input.hgvs}, ) else: @@ -396,7 +397,7 @@ def alleles_at_current_release(allele_ids: set[int]) -> set[int]: EventReason.NO_RECORD, source_version=ensembl_release, error_message="VEP found no functional consequence for this allele, even after Variant Recoder fallback.", - metadata={"hgvs": hgvs}, + metadata={"hgvs": vep_input.hgvs}, ) annotation_manager.flush() @@ -409,7 +410,8 @@ def alleles_at_current_release(allele_ids: set[int]) -> set[int]: ( f"Completed VEP linkage: {annotation_counts['created_allele_count'] + annotation_counts['preexisting_allele_count']} linked, " f"{annotation_counts['absent_allele_count']} absent (no result), " - f"{annotation_counts['errored_allele_count']} errored." + f"{annotation_counts['errored_allele_count']} errored, " + f"{annotation_counts['retained_on_absence_count']} retained despite no result this run." ), ) logger.info(msg="Done linking VEP consequences to alleles.", extra=job_manager.logging_context()) diff --git a/tests/lib/test_vep.py b/tests/lib/test_vep.py index bea141ab2..94251c0ed 100644 --- a/tests/lib/test_vep.py +++ b/tests/lib/test_vep.py @@ -1,7 +1,10 @@ -"""Unit tests for mavedb.lib.vep — covers response parsing for both Variant Recoder and VEP APIs. +"""Unit tests for mavedb.lib.vep — the Ensembl release lookup, the resolution-delegation adapter, and +the allele linker. -These tests mock the underlying HTTP call (request_with_backoff) and assert that the parsing -logic correctly handles the actual Ensembl REST API response shapes. +The resolution flow itself — VEP querying, transcript matching, the Recoder fallback, how forms combine +— lives in ``variant_annotation.lib.vep`` and is covered by that library's own tests. This module tests +only what the api owns: that ``resolve_consequences`` delegates to the library over an Ensembl client +and re-keys the result by input HGVS, the release lookup, and the DB linker (run against the DB). """ from datetime import date, timedelta @@ -10,256 +13,91 @@ import pytest from sqlalchemy import select +from variant_annotation.lib.vep import ( + RESOLVER_VERSION, + ConsequenceOutcome, + ConsequenceResolution, + ConsequenceSource, + VepInput, +) + from mavedb.lib.vep import ( VepLinkVerdict, get_ensembl_release, - get_functional_consequence, link_vep_consequences_to_alleles, - run_variant_recoder, + resolve_consequences, ) from mavedb.models.allele import Allele +from mavedb.models.enums.vep import VepConsequenceSource from mavedb.models.vep_allele_consequence import VepAlleleConsequence -def _mock_response(data) -> MagicMock: - """Return a mock requests.Response whose .json() returns *data*.""" - mock = MagicMock() - mock.json.return_value = data - return mock +def _resolved( + hgvs, + term, + *, + source=ConsequenceSource.TRANSCRIPT, + terms=None, + matched=None, +) -> ConsequenceResolution: + """A RESOLVED resolution the linker can persist, defaulting to a transcript-matched single term.""" + return ConsequenceResolution( + input=VepInput(hgvs=hgvs), + outcome=ConsequenceOutcome.RESOLVED, + consequence_terms=list(terms) if terms is not None else [term], + most_severe_consequence=term, + source=source, + matched_transcript=matched, + ) -# --------------------------------------------------------------------------- -# Realistic Variant Recoder response payloads -# -# Ensembl Variant Recoder returns a list of single-key dicts. The key is the -# called allele (e.g. "T"). The allele dict contains input, hgvsg, hgvsc, etc. -# --------------------------------------------------------------------------- +def _absent(hgvs) -> ConsequenceResolution: + """A resolution for an input VEP answered with no consequence — a genuine empty.""" + return ConsequenceResolution(input=VepInput(hgvs=hgvs), outcome=ConsequenceOutcome.ABSENT) -RECODER_RESPONSE_SINGLE = [ - { - "T": { - "hgvsc": [ - "NM_007294.4:c.5G>A", - ], - "spdi": ["NC_000017.11:43094691:C:T"], - "input": "NP_009225.1:p.Cys2Tyr", - "hgvsp": ["NP_009225.1:p.Cys2Tyr"], - "hgvsg": [ - "NC_000017.11:g.43094692C>T", - "LRG_292:g.100C>T", # non-NC_ entry — should be filtered out - ], - } - } -] - -RECODER_RESPONSE_MULTIPLE = [ - { - "A": { - "input": "NM_007294.4:c.5G>T", - "hgvsg": ["NC_000017.11:g.43094692C>A"], - } - }, - { - "T": { - "input": "NM_007294.4:c.5G>A", - "hgvsg": ["NC_000017.11:g.43094692C>T"], - } - }, -] - -RECODER_RESPONSE_MULTI_ALLELE = [ - { - # Both allele keys share the same "input" value; each has distinct NC_ hgvsg entries. - "CAT": { - "input": "NP_009225.1:p.Val1696His", - "hgvsg": [ - "NC_000017.11:g.43063938_43063940delinsATG", - "LRG_292:g.154044_154046delinsCAT", - ], - "hgvsc": ["NM_007294.4:c.5086_5088delinsCAT"], - }, - "CAC": { - "input": "NP_009225.1:p.Val1696His", - "hgvsg": [ - "NC_000017.11:g.43063938_43063940inv", - "LRG_292:g.154044_154046inv", - ], - "hgvsc": ["NM_007294.4:c.5086_5088inv"], - }, - } -] - -RECODER_RESPONSE_NO_HGVSG = [ - { - "T": { - "input": "NP_009225.1:p.Xxx1Yyy", - "hgvsc": ["NM_007294.4:c.5G>A"], - # deliberately no "hgvsg" key - } - } -] - - -# --------------------------------------------------------------------------- -# Realistic VEP /vep/human/hgvs response payloads -# -# VEP returns a list of dicts where "input" and "most_severe_consequence" are -# *top-level* keys in each element (unlike Recoder where "input" is nested). -# --------------------------------------------------------------------------- - -VEP_RESPONSE_SINGLE = [ - { - "input": "NM_007294.4:c.5G>A", - "most_severe_consequence": "missense_variant", - "id": "rs80357382", - "seq_region_name": "17", - } -] - -VEP_RESPONSE_MULTIPLE = [ - { - "input": "NM_007294.4:c.5G>A", - "most_severe_consequence": "missense_variant", - }, - { - "input": "NM_007294.4:c.10C>T", - "most_severe_consequence": "synonymous_variant", - }, -] - - -# --------------------------------------------------------------------------- -# run_variant_recoder -# --------------------------------------------------------------------------- + +### Tests for get_ensembl_release function ### @pytest.mark.asyncio @pytest.mark.unit -class TestRunVariantRecoder: - async def test_extracts_nc_genomic_hgvs_from_real_response_format(self): - """Correctly extracts NC_ genomic HGVS from the nested allele-key response structure.""" - with patch("mavedb.lib.vep.request_with_backoff", return_value=_mock_response(RECODER_RESPONSE_SINGLE)): - result = await run_variant_recoder(["NP_009225.1:p.Cys2Tyr"]) - - assert result == {"NP_009225.1:p.Cys2Tyr": ["NC_000017.11:g.43094692C>T"]} - - async def test_filters_non_nc_genomic_hgvs(self): - """Non-NC_ entries in hgvsg (e.g. LRG accessions) are excluded from the result.""" - with patch("mavedb.lib.vep.request_with_backoff", return_value=_mock_response(RECODER_RESPONSE_SINGLE)): - result = await run_variant_recoder(["NP_009225.1:p.Cys2Tyr"]) - - returned_hgvsg = result.get("NP_009225.1:p.Cys2Tyr", []) - assert all(h.startswith("NC_") for h in returned_hgvsg) - assert not any("LRG" in h for h in returned_hgvsg) - - async def test_handles_multiple_inputs(self): - """Multiple input variants in one batch are all mapped correctly.""" - with patch("mavedb.lib.vep.request_with_backoff", return_value=_mock_response(RECODER_RESPONSE_MULTIPLE)): - result = await run_variant_recoder(["NM_007294.4:c.5G>T", "NM_007294.4:c.5G>A"]) - - assert result == { - "NM_007294.4:c.5G>T": ["NC_000017.11:g.43094692C>A"], - "NM_007294.4:c.5G>A": ["NC_000017.11:g.43094692C>T"], - } - - async def test_handles_multiple_allele_keys_per_element(self): - """A single response element with multiple allele keys (e.g. CAT, CAC) collects NC_ hgvsg from all of them. - - Variant Recoder can return multiple possible allele representations for the same input variant - in a single element dict. Each allele key is independent and may carry different hgvsg entries. - """ - with patch("mavedb.lib.vep.request_with_backoff", return_value=_mock_response(RECODER_RESPONSE_MULTI_ALLELE)): - result = await run_variant_recoder(["NP_009225.1:p.Val1696His"]) - - genomic_hgvsg = result.get("NP_009225.1:p.Val1696His", []) - assert "NC_000017.11:g.43063938_43063940delinsATG" in genomic_hgvsg - assert "NC_000017.11:g.43063938_43063940inv" in genomic_hgvsg - assert all(h.startswith("NC_") for h in genomic_hgvsg) - - async def test_returns_empty_for_variant_without_hgvsg(self): - """A variant whose allele dict has no hgvsg field is not included in the result.""" - with patch("mavedb.lib.vep.request_with_backoff", return_value=_mock_response(RECODER_RESPONSE_NO_HGVSG)): - result = await run_variant_recoder(["NP_009225.1:p.Xxx1Yyy"]) - - assert result == {} - - async def test_skips_non_dict_allele_values(self): - """If an allele value is not a dict (e.g. null in the response), it is skipped gracefully.""" - response_with_null_allele = [{"T": None}] - with patch("mavedb.lib.vep.request_with_backoff", return_value=_mock_response(response_with_null_allele)): - result = await run_variant_recoder(["some_hgvs"]) - - assert result == {} - - async def test_input_field_at_allele_level_not_top_level(self): - """Regression: the 'input' field is nested inside the allele key, not at the top level of each list element. - - The old buggy implementation called entry.get("input") on the outer dict, - which always returned None and silently dropped all results. - """ - with patch("mavedb.lib.vep.request_with_backoff", return_value=_mock_response(RECODER_RESPONSE_SINGLE)): - result = await run_variant_recoder(["NP_009225.1:p.Cys2Tyr"]) - - # If "input" were read from the wrong level, result would be empty. - assert result != {} - assert "NP_009225.1:p.Cys2Tyr" in result +async def test_get_ensembl_release_returns_the_clients_release(): + """The release lookup returns whatever the Ensembl client's software_release reports (it version-keys + the skip). The /info/software call itself is the client's, covered by the library.""" + with patch("mavedb.lib.vep.EnsemblRestClient.software_release", return_value="116"): + assert await get_ensembl_release() == "116" -# --------------------------------------------------------------------------- -# get_functional_consequence -# --------------------------------------------------------------------------- +### Tests for resolve_consequences (the delegation adapter) ### @pytest.mark.asyncio @pytest.mark.unit -class TestGetFunctionalConsequence: - async def test_extracts_most_severe_consequence(self): - """Parses input and most_severe_consequence from the top-level VEP response structure.""" - with patch("mavedb.lib.vep.request_with_backoff", return_value=_mock_response(VEP_RESPONSE_SINGLE)): - result = await get_functional_consequence(["NM_007294.4:c.5G>A"]) - - assert result == {"NM_007294.4:c.5G>A": "missense_variant"} - - async def test_maps_none_when_consequence_absent(self): - """When most_severe_consequence is missing from an entry, the HGVS maps to None.""" - response = [{"input": "NM_007294.4:c.5G>A"}] - with patch("mavedb.lib.vep.request_with_backoff", return_value=_mock_response(response)): - result = await get_functional_consequence(["NM_007294.4:c.5G>A"]) - - assert result == {"NM_007294.4:c.5G>A": None} - - async def test_skips_entries_without_input_key(self): - """Entries that have no 'input' key are skipped entirely.""" - response = [{"most_severe_consequence": "missense_variant"}] - with patch("mavedb.lib.vep.request_with_backoff", return_value=_mock_response(response)): - result = await get_functional_consequence(["NM_007294.4:c.5G>A"]) +class TestResolveConsequences: + """resolve_consequences is a thin adapter over the shared library orchestration: it runs the + library's resolve_consequences over an Ensembl client and re-keys the result by input HGVS. The + resolution flow itself is the library's, covered by its own tests.""" + + async def test_delegates_and_keys_results_by_hgvs(self): + """The library's list-in-input-order result comes back re-keyed by each input's HGVS.""" + a = VepInput(hgvs="NM_1:c.5G>A") + b = VepInput(hgvs="NM_1:c.10C>T") + library_result = [_resolved(a.hgvs, "missense_variant"), _absent(b.hgvs)] + with patch("mavedb.lib.vep.resolve_consequences_kernel", MagicMock(return_value=library_result)) as kernel: + result = await resolve_consequences([a, b]) + + kernel.assert_called_once() # delegated to the library, not re-implemented here + assert set(result) == {a.hgvs, b.hgvs} + assert result[a.hgvs].outcome is ConsequenceOutcome.RESOLVED + assert result[b.hgvs].outcome is ConsequenceOutcome.ABSENT + + async def test_empty_input_short_circuits_without_calling_the_library(self): + """No inputs -> no client built and no library call, just an empty result.""" + with patch("mavedb.lib.vep.resolve_consequences_kernel", MagicMock()) as kernel: + result = await resolve_consequences([]) assert result == {} - - async def test_handles_multiple_variants(self): - """All variants in a batch response are extracted correctly.""" - with patch("mavedb.lib.vep.request_with_backoff", return_value=_mock_response(VEP_RESPONSE_MULTIPLE)): - result = await get_functional_consequence(["NM_007294.4:c.5G>A", "NM_007294.4:c.10C>T"]) - - assert result == { - "NM_007294.4:c.5G>A": "missense_variant", - "NM_007294.4:c.10C>T": "synonymous_variant", - } - - async def test_raises_if_more_than_200_variants(self): - """Passing more than 200 HGVS strings raises ValueError before any HTTP call.""" - with pytest.raises(ValueError, match="maximum of 200"): - await get_functional_consequence(["NM_007294.4:c.1A>T"] * 201) - - -### Tests for get_ensembl_release function ### - - -@pytest.mark.asyncio -async def test_get_ensembl_release_returns_release_as_string(): - """The /info/software release integer is returned as a string for use as source_version.""" - with patch("mavedb.lib.vep.request_with_backoff", return_value=_mock_response({"release": 116})): - assert await get_ensembl_release() == "116" + kernel.assert_not_called() ### Tests for link_vep_consequences_to_alleles function ### @@ -288,11 +126,22 @@ def _all_rows_for(session, allele_id): def test_link_vep_creates_new_consequence(session): - """A consequence for an allele with no live row creates a single live row and is reported changed.""" + """A consequence for an allele with no live row creates a single live row carrying the headline term + and its full resolution provenance, and is reported changed.""" allele = _make_allele(session, vrs_digest="vrs-1") verdicts = link_vep_consequences_to_alleles( - session, {allele.id: "missense_variant"}, source_version="116", access_date=date.today() + session, + { + allele.id: _resolved( + "NM_007294.4:c.5G>A", + "missense_variant", + terms=["missense_variant", "splice_region_variant"], + matched="NM_007294.4", + ) + }, + source_version="116", + access_date=date.today(), ) session.commit() @@ -302,12 +151,18 @@ def test_link_vep_creates_new_consequence(session): assert live[0].functional_consequence == "missense_variant" assert live[0].source_version == "116" assert live[0].access_date == date.today() - - -def test_link_vep_unchanged_bumps_version_and_date_in_place(session): - """Re-confirming an unchanged consequence at a new release advances source_version and access_date - in place — no supersede, no new valid-time boundary. The allele is reported UNCHANGED (status - preexisting) so the caller need not re-query consequence state.""" + # Full resolution provenance is persisted, not just the headline term (#772). + assert live[0].consequence_terms == ["missense_variant", "splice_region_variant"] + assert live[0].consequence_source == VepConsequenceSource.transcript + assert live[0].matched_transcript == "NM_007294.4" + # The resolution-rule version is stamped so the skip can tell a rule-fixed row from a stale one. + assert live[0].resolver_version == RESOLVER_VERSION + + +def test_link_vep_unchanged_bumps_version_and_fills_provenance_in_place(session): + """Re-confirming an unchanged headline term at a new release advances source_version/access_date in + place — no supersede — and fills in the resolution provenance a pre-#772 row was missing. The allele + is reported UNCHANGED (status preexisting) so the caller need not re-query consequence state.""" allele = _make_allele(session, vrs_digest="vrs-1") session.add( VepAlleleConsequence( @@ -320,21 +175,66 @@ def test_link_vep_unchanged_bumps_version_and_date_in_place(session): session.commit() verdicts = link_vep_consequences_to_alleles( - session, {allele.id: "missense_variant"}, source_version="116", access_date=date.today() + session, + { + allele.id: _resolved( + "NM_1:c.5G>A", "missense_variant", source=ConsequenceSource.TRANSCRIPT, matched="NM_1.4" + ) + }, + source_version="116", + access_date=date.today(), ) session.commit() assert verdicts == {allele.id: VepLinkVerdict.UNCHANGED} - # One row, still live, never retired — version and access_date advanced in place. + # One row, still live, never retired — version, date, and provenance advanced in place. all_rows = _all_rows_for(session, allele.id) assert len(all_rows) == 1 assert all_rows[0].valid_to is None assert all_rows[0].source_version == "116" assert all_rows[0].access_date == date.today() + assert all_rows[0].consequence_source == VepConsequenceSource.transcript + assert all_rows[0].matched_transcript == "NM_1.4" + # A pre-column row (NULL resolver_version) is filled in place, not superseded. + assert all_rows[0].resolver_version == RESOLVER_VERSION + + +def test_link_vep_source_change_same_term_advances_in_place(session): + """A better provenance for the *same* headline term (most_severe -> transcript) is not a value + change: it advances in place rather than churning a new history row. History records term changes.""" + allele = _make_allele(session, vrs_digest="vrs-1") + session.add( + VepAlleleConsequence( + allele_id=allele.id, + functional_consequence="missense_variant", + consequence_source=VepConsequenceSource.most_severe, + source_version="116", + access_date=date.today() - timedelta(days=1), + ) + ) + session.commit() + + verdicts = link_vep_consequences_to_alleles( + session, + { + allele.id: _resolved( + "NM_1:c.5G>A", "missense_variant", source=ConsequenceSource.TRANSCRIPT, matched="NM_1.4" + ) + }, + source_version="116", + access_date=date.today(), + ) + session.commit() + + assert verdicts == {allele.id: VepLinkVerdict.UNCHANGED} + rows = _all_rows_for(session, allele.id) + assert len(rows) == 1 # no supersede + assert rows[0].consequence_source == VepConsequenceSource.transcript + assert rows[0].matched_transcript == "NM_1.4" def test_link_vep_changed_consequence_supersedes(session): - """A changed consequence retires the live row and inserts the successor — exactly one live row, + """A changed headline term retires the live row and inserts the successor — exactly one live row, keyed on allele_id, with the old one preserved as retired history.""" allele = _make_allele(session, vrs_digest="vrs-1") session.add( @@ -348,7 +248,10 @@ def test_link_vep_changed_consequence_supersedes(session): session.commit() verdicts = link_vep_consequences_to_alleles( - session, {allele.id: "missense_variant"}, source_version="116", access_date=date.today() + session, + {allele.id: _resolved("NM_1:c.5G>A", "missense_variant")}, + source_version="116", + access_date=date.today(), ) session.commit() @@ -363,10 +266,10 @@ def test_link_vep_changed_consequence_supersedes(session): assert len([r for r in all_rows if r.valid_to is not None]) == 1 -def test_link_vep_none_leaves_live_row_untouched(session): - """A transient None result must not overwrite a held consequence: the live row is left intact - (value, version, and date). The held consequence is reported UNCHANGED (status preexisting) — the - allele still has a live consequence, it just was not re-confirmed this run.""" +def test_link_vep_absent_leaves_live_row_untouched(session): + """A transient absent result must not overwrite a held consequence: the live row is left intact + (value, version, and date). The held consequence is reported RETAINED_ON_ABSENCE (status + preexisting) — the allele still has a live consequence, it just was not re-confirmed this run.""" allele = _make_allele(session, vrs_digest="vrs-1") session.add( VepAlleleConsequence( @@ -379,11 +282,13 @@ def test_link_vep_none_leaves_live_row_untouched(session): session.commit() verdicts = link_vep_consequences_to_alleles( - session, {allele.id: None}, source_version="116", access_date=date.today() + session, {allele.id: _absent("NM_1:c.5G>A")}, source_version="116", access_date=date.today() ) session.commit() - assert verdicts == {allele.id: VepLinkVerdict.UNCHANGED} + # RETAINED_ON_ABSENCE, not UNCHANGED: the prior value was held because VEP found nothing this run, + # a case the caller surfaces distinctly (job metadata + a loud warning). + assert verdicts == {allele.id: VepLinkVerdict.RETAINED_ON_ABSENCE} live = _live_rows_for(session, allele.id) assert len(live) == 1 assert live[0].functional_consequence == "missense_variant" @@ -392,14 +297,35 @@ def test_link_vep_none_leaves_live_row_untouched(session): assert live[0].access_date == date.today() - timedelta(days=90) -def test_link_vep_none_with_no_live_row_writes_nothing(session): - """A None result for an allele with no live row writes nothing and leaves the allele out of the +def test_link_vep_absent_with_no_live_row_writes_nothing(session): + """An absent result for an allele with no live row writes nothing and leaves the allele out of the verdict map (the caller reads that as a no-result and re-queries next run), mirroring gnomAD's no-match handling.""" allele = _make_allele(session, vrs_digest="vrs-1") verdicts = link_vep_consequences_to_alleles( - session, {allele.id: None}, source_version="116", access_date=date.today() + session, {allele.id: _absent("NM_1:c.5G>A")}, source_version="116", access_date=date.today() + ) + session.commit() + + assert verdicts == {} + assert len(_all_rows_for(session, allele.id)) == 0 + + +def test_link_vep_errored_is_never_linked(session): + """An ERRORED resolution must never reach the value table (that would overwrite a held consequence + with a failure). It is ignored defensively and leaves no verdict — failures live in the event stream.""" + allele = _make_allele(session, vrs_digest="vrs-1") + + verdicts = link_vep_consequences_to_alleles( + session, + { + allele.id: ConsequenceResolution( + input=VepInput(hgvs="NM_1:c.5G>A"), outcome=ConsequenceOutcome.ERRORED, error="boom" + ) + }, + source_version="116", + access_date=date.today(), ) session.commit() diff --git a/tests/worker/jobs/external_services/network/test_vep.py b/tests/worker/jobs/external_services/network/test_vep.py index 61ba8dfd4..52ed791d2 100644 --- a/tests/worker/jobs/external_services/network/test_vep.py +++ b/tests/worker/jobs/external_services/network/test_vep.py @@ -6,10 +6,11 @@ from sqlalchemy import select +from mavedb.models.annotation_event import AnnotationEvent from mavedb.models.enums.annotation_type import AnnotationType from mavedb.models.enums.disposition import Disposition from mavedb.models.enums.job_pipeline import JobStatus, PipelineStatus -from mavedb.models.annotation_event import AnnotationEvent +from mavedb.models.enums.vep import VepConsequenceSource from mavedb.models.vep_allele_consequence import VepAlleleConsequence pytestmark = pytest.mark.usefixtures("patch_db_session_ctxmgr") @@ -52,53 +53,12 @@ async def test_populate_vep_e2e( ).one() assert live.functional_consequence is not None assert live.access_date is not None + # The coding allele's consequence is read from its own transcript (#772), not from VEP's + # cross-transcript headline. + assert live.consequence_source == VepConsequenceSource.transcript + assert live.matched_transcript is not None - # Events are allele-keyed now; the variant resolves its status through the live link. - event = session.scalars( - select(AnnotationEvent).where( - AnnotationEvent.allele_id == allele.id, - AnnotationEvent.annotation_type == AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, - ) - ).one() - assert event.disposition == Disposition.PRESENT - - async def test_populate_vep_e2e_with_recoder_path( - self, - session, - arq_redis, - arq_worker, - sample_populate_vep_run_pipeline, - sample_populate_vep_pipeline, - setup_sample_protein_allele_for_vep, - ): - """VEP job uses Variant Recoder for a protein HGVS (NP_ accession) that VEP cannot resolve directly. - - NP_009225.1:p.Val1696His is a BRCA1 protein variant that VEP's /vep/human/hgvs endpoint - does not return a consequence for. The job must fall back to Variant Recoder, recode it - to a genomic HGVS, and then re-query VEP with the recoded string. - """ - variant, allele = setup_sample_protein_allele_for_vep - - await arq_redis.enqueue_job("populate_vep_for_score_set", sample_populate_vep_run_pipeline.id) - await arq_worker.async_run() - await arq_worker.run_check() - - session.refresh(sample_populate_vep_run_pipeline) - assert sample_populate_vep_run_pipeline.status == JobStatus.SUCCEEDED - - session.refresh(sample_populate_vep_pipeline) - assert sample_populate_vep_pipeline.status == PipelineStatus.SUCCEEDED - - live = session.scalars( - select(VepAlleleConsequence).where( - VepAlleleConsequence.allele_id == allele.id, - VepAlleleConsequence.current, - ) - ).one() - assert live.functional_consequence is not None - assert live.access_date is not None - - # Events are allele-keyed now; the variant resolves its status through the live link. + # Events are allele-keyed; the variant resolves its status through the live link. event = session.scalars( select(AnnotationEvent).where( AnnotationEvent.allele_id == allele.id, diff --git a/tests/worker/jobs/external_services/test_vep.py b/tests/worker/jobs/external_services/test_vep.py index 19587f5c2..cc24e213c 100644 --- a/tests/worker/jobs/external_services/test_vep.py +++ b/tests/worker/jobs/external_services/test_vep.py @@ -9,21 +9,48 @@ from sqlalchemy import select +from variant_annotation.lib.vep import ( + NO_CHANGE_TERM, + RESOLVER_VERSION, + ConsequenceOutcome, + ConsequenceResolution, + ConsequenceSource, + VepInput, +) + from mavedb.lib.types.workflow import JobExecutionOutcome from mavedb.models.enums.job_pipeline import JobStatus, PipelineStatus +from mavedb.models.enums.vep import VepConsequenceSource from mavedb.models.annotation_event import AnnotationEvent from mavedb.models.vep_allele_consequence import VepAlleleConsequence -from mavedb.lib.vep import VepResolution from mavedb.worker.jobs.external_services.vep import populate_vep_for_score_set from mavedb.worker.lib.managers.job_manager import JobManager pytestmark = pytest.mark.usefixtures("patch_db_session_ctxmgr") -_RESOLVE = "mavedb.worker.jobs.external_services.vep._resolve_consequences" +# resolve_consequences lives in mavedb.lib.vep; the worker imports it, so we patch the worker's binding. +# The orchestration itself is unit-tested in tests/lib/test_vep.py::TestResolveConsequences. +_RESOLVE = "mavedb.worker.jobs.external_services.vep.resolve_consequences" _RELEASE = "mavedb.worker.jobs.external_services.vep.get_ensembl_release" _ENSEMBL_RELEASE = "116" +def _resolved(hgvs: str, term: str) -> ConsequenceResolution: + """A RESOLVED resolution keyed by hgvs — the shape resolve_consequences returns per input.""" + return ConsequenceResolution( + input=VepInput(hgvs=hgvs), + outcome=ConsequenceOutcome.RESOLVED, + consequence_terms=[term], + most_severe_consequence=term, + source=ConsequenceSource.TRANSCRIPT, + ) + + +def _errored(hgvs: str) -> ConsequenceResolution: + """An ERRORED resolution — the request failed, result unknown (must be retried, never linked).""" + return ConsequenceResolution(input=VepInput(hgvs=hgvs), outcome=ConsequenceOutcome.ERRORED, error="boom") + + @pytest.fixture(autouse=True) def mock_ensembl_release(): """Stamp every job run with a fixed Ensembl release so tests version-key deterministically without @@ -68,27 +95,6 @@ async def test_no_alleles_with_hgvs( assert result.data["absent_allele_count"] == 0 assert result.data["errored_allele_count"] == 0 - async def test_calls_resolver_when_alleles_present( - self, - session, - with_populated_domain_data, - with_populate_vep_job, - mock_worker_ctx, - sample_populate_vep_run, - setup_sample_alleles_for_vep, - ): - """The VEP resolver is invoked once when the score set has HGVS-bearing alleles to query.""" - with patch(_RESOLVE, return_value=VepResolution({}, set())) as mock_resolve: - result = await populate_vep_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_vep_run.id), - ) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - mock_resolve.assert_called_once() - async def test_propagates_exceptions( self, session, @@ -169,10 +175,10 @@ async def test_no_consequence_resolved( setup_sample_alleles_for_vep, ): """A genuine empty (VEP queried successfully, found nothing) writes no consequence row and an - ABSENT/no_record event — a trustworthy negative, now that empty is split from a failed request.""" + ABSENT/no_record event — a trustworthy negative, distinct from a failed request.""" _, allele = setup_sample_alleles_for_vep - with patch(_RESOLVE, return_value=VepResolution({}, set())): + with patch(_RESOLVE, return_value={}): result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) assert result.status == JobStatus.SUCCEEDED @@ -186,6 +192,28 @@ async def test_no_consequence_resolved( assert events[0].annotation_type == "vep_functional_consequence" assert events[0].allele_id == allele.id and events[0].variant_id is None + async def test_skips_protein_level_allele( + self, + session, + with_populated_domain_data, + with_populate_vep_job, + mock_worker_ctx, + sample_populate_vep_run, + setup_sample_protein_allele_for_vep, + ): + """A protein-level allele is never submitted to VEP (#772 routing) — its consequence is carried + by the coding alleles reverse translation enumerates. With only a protein allele present the job + has nothing eligible to do: no external query, and no consequence row or event for it.""" + _, allele = setup_sample_protein_allele_for_vep + + with patch(_RESOLVE) as mock_resolve: + result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) + + assert result.status == JobStatus.SUCCEEDED + mock_resolve.assert_not_called() # protein allele routed out before any query + assert len(_live_consequences_for(session, allele.id)) == 0 + assert session.scalars(select(AnnotationEvent).where(AnnotationEvent.allele_id == allele.id)).all() == [] + async def test_request_failure_recorded_as_errored( self, session, @@ -200,7 +228,7 @@ async def test_request_failure_recorded_as_errored( _, allele = setup_sample_alleles_for_vep # Resolver reports the allele's HGVS as errored (request failed), not as a hit or an empty. - with patch(_RESOLVE, return_value=VepResolution({}, {allele.hgvs_c})): + with patch(_RESOLVE, return_value={allele.hgvs_c: _errored(allele.hgvs_c)}): result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) assert result.status == JobStatus.SUCCEEDED @@ -227,7 +255,7 @@ async def test_successful_linking_independent( """A resolved consequence creates a single live VepAlleleConsequence and a SUCCESS VAS row.""" _, allele = setup_sample_alleles_for_vep - with patch(_RESOLVE, return_value=VepResolution({allele.hgvs_c: "missense_variant"}, set())): + with patch(_RESOLVE, return_value={allele.hgvs_c: _resolved(allele.hgvs_c, "missense_variant")}): result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) assert result.status == JobStatus.SUCCEEDED @@ -264,7 +292,7 @@ async def test_links_and_annotates_rt_derived_allele( # VEP resolves only the RT-derived allele's genomic HGVS; the authoritative allele's coding # HGVS is queried successfully but yields no consequence this run (a genuine empty, not errored). - with patch(_RESOLVE, return_value=VepResolution({rt_allele.hgvs_g: "missense_variant"}, set())): + with patch(_RESOLVE, return_value={rt_allele.hgvs_g: _resolved(rt_allele.hgvs_g, "missense_variant")}): result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) assert result.status == JobStatus.SUCCEEDED @@ -286,6 +314,81 @@ async def test_links_and_annotates_rt_derived_allele( assert by_allele[authoritative_allele.id].disposition == "absent" assert by_allele[authoritative_allele.id].reason == "no_record" + async def test_reference_identical_control_is_labelled( + self, + session, + with_populated_domain_data, + with_populate_vep_job, + mock_worker_ctx, + sample_populate_vep_run, + setup_sample_alleles_for_vep, + ): + """A wild-type control describes no sequence change, so VEP cannot resolve it. When the reference + port settles it as reference-identical, it is stored as a live consequence labelled + ``reference_identical`` (headline term ``no_change``) rather than dropped to ABSENT — so a control + can be told apart from an unannotated allele. The event is present/created: a result was reached.""" + _, allele = setup_sample_alleles_for_vep + hgvs = allele.hgvs_c or allele.hgvs_g + + reference_identical = ConsequenceResolution( + input=VepInput(hgvs=hgvs), + outcome=ConsequenceOutcome.RESOLVED, + consequence_terms=[NO_CHANGE_TERM], + most_severe_consequence=NO_CHANGE_TERM, + source=ConsequenceSource.REFERENCE_IDENTICAL, + ) + with patch(_RESOLVE, return_value={hgvs: reference_identical}): + result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) + + assert result.status == JobStatus.SUCCEEDED + assert result.data["created_allele_count"] == 1 + + live = _live_consequences_for(session, allele.id) + assert len(live) == 1 + assert live[0].functional_consequence == NO_CHANGE_TERM + assert live[0].consequence_source == VepConsequenceSource.reference_identical + assert live[0].matched_transcript is None + + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 1 + assert events[0].disposition == "present" + assert events[0].reason == "created" + assert events[0].allele_id == allele.id and events[0].variant_id is None + + async def test_resolves_in_chunks_and_merges_results( + self, + session, + with_populated_domain_data, + with_populate_vep_job, + mock_worker_ctx, + sample_populate_vep_run, + setup_rt_derived_allele_for_vep, + ): + """Inputs are resolved in chunks (so progress can be reported between them), and the results from + every chunk are merged — not just the last chunk's. With the chunk size forced to one, each of the + two alleles resolves in its own call and both are still linked; a chunk whose result was dropped + would leave one allele unlinked.""" + _, authoritative_allele, rt_allele = setup_rt_derived_allele_for_vep + + def _resolve_each(inputs, **kwargs): + # One input per chunk here; each chunk returns only its own input's resolution, so a chunk + # whose result the job failed to merge would drop that allele's link. + return {inp.hgvs: _resolved(inp.hgvs, "missense_variant") for inp in inputs} + + with ( + patch("mavedb.worker.jobs.external_services.vep._RESOLUTION_CHUNK_SIZE", 1), + patch(_RESOLVE, side_effect=_resolve_each) as mock_resolve, + ): + result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) + + assert result.status == JobStatus.SUCCEEDED + # Two unique HGVS at chunk size one -> two resolution calls, each a single-input chunk. + assert mock_resolve.call_count == 2 + assert all(len(call.args[0]) == 1 for call in mock_resolve.call_args_list) + # Both alleles linked -> results from both chunks were merged, not overwritten by the last. + assert len(_live_consequences_for(session, authoritative_allele.id)) == 1 + assert len(_live_consequences_for(session, rt_allele.id)) == 1 + async def test_skips_allele_already_at_current_release( self, session, @@ -295,16 +398,18 @@ async def test_skips_allele_already_at_current_release( sample_populate_vep_run, setup_sample_alleles_for_vep, ): - """An allele with a live consequence already at the current Ensembl release is skipped: no VEP - query, the status reports SUCCESS/preexisting, and the existing row is not churned.""" + """An allele with a live consequence already at the current Ensembl release *and* resolver + version is skipped: no VEP query, the status reports SUCCESS/preexisting, and the row is not + churned.""" _, allele = setup_sample_alleles_for_vep - # Simulate a prior run at the current release. + # Simulate a prior run at the current release and resolver version (current on both axes). session.add( VepAlleleConsequence( allele_id=allele.id, functional_consequence="missense_variant", source_version=_ENSEMBL_RELEASE, + resolver_version=RESOLVER_VERSION, access_date=date.today(), ) ) @@ -327,6 +432,47 @@ async def test_skips_allele_already_at_current_release( assert events[0].disposition == "present" assert events[0].reason == "preexisting" + async def test_stale_resolver_version_is_requeried( + self, + session, + with_populated_domain_data, + with_populate_vep_job, + mock_worker_ctx, + sample_populate_vep_run, + setup_sample_alleles_for_vep, + ): + """An allele live at the *current* Ensembl release but resolved under an older resolver version is + NOT skipped — a resolution-rule fix (RESOLVER_VERSION bump) must re-query without waiting for an + Ensembl release bump. Re-resolving the same term reports preexisting and advances the resolver + version on the live row in place (no churn).""" + _, allele = setup_sample_alleles_for_vep + + # Current Ensembl release but a stale resolver version -> only the resolver axis is out of date. + session.add( + VepAlleleConsequence( + allele_id=allele.id, + functional_consequence="missense_variant", + source_version=_ENSEMBL_RELEASE, + resolver_version=f"{RESOLVER_VERSION}-stale", + access_date=date.today() - timedelta(days=1), + ) + ) + session.commit() + + with patch( + _RESOLVE, return_value={allele.hgvs_c: _resolved(allele.hgvs_c, "missense_variant")} + ) as mock_resolve: + result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) + + mock_resolve.assert_called_once() # stale resolver version -> re-queried despite the current release + assert result.data["preexisting_allele_count"] == 1 + assert result.data["created_allele_count"] == 0 + + # The live row is refreshed to the current resolver version (advanced in place, still one live row). + live = _live_consequences_for(session, allele.id) + assert len(live) == 1 + assert live[0].resolver_version == RESOLVER_VERSION + async def test_new_release_same_consequence_bumps_in_place( self, session, @@ -336,9 +482,9 @@ async def test_new_release_same_consequence_bumps_in_place( sample_populate_vep_run, setup_sample_alleles_for_vep, ): - """An allele live at an older release is re-queried; an unchanged consequence advances - source_version (and access_date) in place — no supersede, so a new release does not churn - history for a categorical value that did not change.""" + """An allele live at an *older* release is not skipped: the job re-queries it and reports it + preexisting (the unchanged consequence advances in place). The row-level in-place-bump semantics + are the linker's, covered by test_link_vep_unchanged_bumps_version_and_fills_provenance_in_place.""" _, allele = setup_sample_alleles_for_vep session.add( @@ -351,20 +497,15 @@ async def test_new_release_same_consequence_bumps_in_place( ) session.commit() - with patch(_RESOLVE, return_value=VepResolution({allele.hgvs_c: "missense_variant"}, set())) as mock_resolve: + with patch( + _RESOLVE, return_value={allele.hgvs_c: _resolved(allele.hgvs_c, "missense_variant")} + ) as mock_resolve: result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) mock_resolve.assert_called_once() # older release -> not skipped, re-queried assert result.data["preexisting_allele_count"] == 1 assert result.data["created_allele_count"] == 0 - # Unchanged value -> no supersede: one row, still live, version + date advanced in place. - rows = session.scalars(select(VepAlleleConsequence).where(VepAlleleConsequence.allele_id == allele.id)).all() - assert len(rows) == 1 - assert rows[0].valid_to is None - assert rows[0].source_version == _ENSEMBL_RELEASE - assert rows[0].access_date == date.today() - async def test_force_requeries_unchanged_without_churn( self, session, @@ -374,17 +515,18 @@ async def test_force_requeries_unchanged_without_churn( sample_populate_vep_run, setup_sample_alleles_for_vep, ): - """force bypasses the current-release skip and re-queries, but the linker only supersedes on a - value change: a forced re-run that resolves the same consequence reports preexisting and does - not churn the row (version/access_date advanced in place).""" + """force bypasses the current-release skip: an allele already at the current release, which would + normally be skipped, is re-queried instead. The no-churn-on-unchanged-value guarantee is the + linker's, covered by test_link_vep_unchanged_bumps_version_and_fills_provenance_in_place.""" _, allele = setup_sample_alleles_for_vep - # Prior live consequence already at the current release (would be skipped without force). + # Prior live consequence already current on both axes (would be skipped without force). session.add( VepAlleleConsequence( allele_id=allele.id, functional_consequence="missense_variant", source_version=_ENSEMBL_RELEASE, + resolver_version=RESOLVER_VERSION, access_date=date.today() - timedelta(days=1), ) ) @@ -393,19 +535,15 @@ async def test_force_requeries_unchanged_without_churn( sample_populate_vep_run.job_params = {**sample_populate_vep_run.job_params, "force": True} session.commit() - with patch(_RESOLVE, return_value=VepResolution({allele.hgvs_c: "missense_variant"}, set())) as mock_resolve: + with patch( + _RESOLVE, return_value={allele.hgvs_c: _resolved(allele.hgvs_c, "missense_variant")} + ) as mock_resolve: result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) mock_resolve.assert_called_once() # force bypassed the current-release skip assert result.data["preexisting_allele_count"] == 1 assert result.data["created_allele_count"] == 0 - # Unchanged consequence -> no supersede: one row, still live, access_date touched in place. - rows = session.scalars(select(VepAlleleConsequence).where(VepAlleleConsequence.allele_id == allele.id)).all() - assert len(rows) == 1 - assert rows[0].valid_to is None - assert rows[0].access_date == date.today() - async def test_new_release_changed_consequence_supersedes( self, session, @@ -415,8 +553,9 @@ async def test_new_release_changed_consequence_supersedes( sample_populate_vep_run, setup_sample_alleles_for_vep, ): - """An allele live at an older release is re-queried; a changed result supersedes it — one live - row carrying the new consequence/release and one retired row preserving the old.""" + """An allele live at an *older* release is not skipped: the job re-queries it and a changed + result is reported created. The supersede/retire row semantics are the linker's, covered by + test_link_vep_changed_consequence_supersedes.""" _, allele = setup_sample_alleles_for_vep # Prior live consequence at an older release -> not skipped, eligible for re-query. @@ -430,22 +569,53 @@ async def test_new_release_changed_consequence_supersedes( ) session.commit() - with patch(_RESOLVE, return_value=VepResolution({allele.hgvs_c: "missense_variant"}, set())): + with patch( + _RESOLVE, return_value={allele.hgvs_c: _resolved(allele.hgvs_c, "missense_variant")} + ) as mock_resolve: result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) + mock_resolve.assert_called_once() # older release -> not skipped, re-queried assert result.data["created_allele_count"] == 1 + async def test_retained_on_absence_surfaced_in_outcome( + self, + session, + with_populated_domain_data, + with_populate_vep_job, + mock_worker_ctx, + sample_populate_vep_run, + setup_sample_alleles_for_vep, + ): + """An allele that had a consequence at an older release but for which VEP now returns nothing + keeps its prior value (not superseded) and is counted in ``retained_on_absence_count`` — the + job-metadata signal for a possible genuine disappearance.""" + _, allele = setup_sample_alleles_for_vep + + # Prior live consequence at an older release -> re-queried, but VEP returns nothing this run. + session.add( + VepAlleleConsequence( + allele_id=allele.id, + functional_consequence="missense_variant", + source_version="115", + access_date=date.today() - timedelta(days=90), + ) + ) + session.commit() + + with patch(_RESOLVE, return_value={}): + result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) + + assert result.status == JobStatus.SUCCEEDED + # Surfaced in the outcome metadata, and counted as preexisting (the allele still has a value). + assert result.data["retained_on_absence_count"] == 1 + assert result.data["preexisting_allele_count"] == 1 + assert result.data["absent_allele_count"] == 0 + + # The prior consequence was held in place, not superseded or dropped. live = _live_consequences_for(session, allele.id) assert len(live) == 1 assert live[0].functional_consequence == "missense_variant" - assert live[0].source_version == _ENSEMBL_RELEASE - - # Old consequence retired, not deleted. - all_rows = session.scalars( - select(VepAlleleConsequence).where(VepAlleleConsequence.allele_id == allele.id) - ).all() - assert len(all_rows) == 2 - assert len([r for r in all_rows if r.valid_to is not None]) == 1 + assert live[0].source_version == "115" async def test_successful_linking_pipeline( self, @@ -459,7 +629,7 @@ async def test_successful_linking_pipeline( """End-to-end successful linking within a pipeline updates both job and pipeline status.""" _, allele = setup_sample_alleles_for_vep - with patch(_RESOLVE, return_value=VepResolution({allele.hgvs_c: "missense_variant"}, set())): + with patch(_RESOLVE, return_value={allele.hgvs_c: _resolved(allele.hgvs_c, "missense_variant")}): result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run_pipeline.id) assert result.status == JobStatus.SUCCEEDED @@ -520,7 +690,7 @@ async def test_populate_vep_with_arq_context_independent( """The VEP job links a consequence and records a SUCCESS annotation through the ARQ worker.""" _, allele = setup_sample_alleles_for_vep - with patch(_RESOLVE, return_value=VepResolution({allele.hgvs_c: "missense_variant"}, set())): + with patch(_RESOLVE, return_value={allele.hgvs_c: _resolved(allele.hgvs_c, "missense_variant")}): await arq_redis.enqueue_job("populate_vep_for_score_set", sample_populate_vep_run.id) await arq_worker.async_run() await arq_worker.run_check() @@ -550,7 +720,7 @@ async def test_populate_vep_with_arq_context_pipeline( """The VEP job completes and advances the pipeline through the ARQ worker.""" _, allele = setup_sample_alleles_for_vep - with patch(_RESOLVE, return_value=VepResolution({allele.hgvs_c: "missense_variant"}, set())): + with patch(_RESOLVE, return_value={allele.hgvs_c: _resolved(allele.hgvs_c, "missense_variant")}): await arq_redis.enqueue_job("populate_vep_for_score_set", sample_populate_vep_run_pipeline.id) await arq_worker.async_run() await arq_worker.run_check() From 720c6b4a0e2923d4ebc309d833b98f5e146d1230 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Fri, 28 Aug 2026 13:14:36 -0700 Subject: [PATCH 125/252] feat(pipeline): re-enable VEP annotation now that consequences are transcript-matched populate_vep_for_score_set was disabled (#772) because its consequences came from VEP's top-level most_severe_consequence, the worst call across every transcript overlapping a position rather than the one the variant was mapped to. Now that the worker job resolves against the allele's own transcript via the shared resolution kernel, re-add the job to the pipeline definition. Depend on reverse_translate_variants_for_score_set rather than the old submit_score_set_mappings_to_car: VEP needs the coding/genomic HGVS reverse translation produces, not confirmation that mappings were submitted to CAR. --- src/mavedb/lib/workflow/definitions.py | 24 ++++++++----------- tests/scripts/test_run_score_set_pipelines.py | 9 ++----- 2 files changed, 12 insertions(+), 21 deletions(-) diff --git a/src/mavedb/lib/workflow/definitions.py b/src/mavedb/lib/workflow/definitions.py index 5b1a4cda8..4b7b1e178 100644 --- a/src/mavedb/lib/workflow/definitions.py +++ b/src/mavedb/lib/workflow/definitions.py @@ -96,20 +96,16 @@ def annotation_pipeline_job_definitions( }, "dependencies": [("warm_clingen_cache", DependencyType.SUCCESS_REQUIRED)], }, - # VEP annotation is intentionally not scheduled: its consequences came from VEP's top-level - # most_severe_consequence, the worst call across all overlapping transcripts rather than the - # one the variant was mapped to. - # TODO(#772): Re-enable once VEP annotates Allele rows directly. - # { - # "key": "populate_vep_for_score_set", - # "function": "populate_vep_for_score_set", - # "type": JobType.MAPPED_VARIANT_ANNOTATION, - # "params": { - # "correlation_id": None, # Required param to be filled in at runtime - # "score_set_id": None, # Required param to be filled in at runtime - # }, - # "dependencies": [("submit_score_set_mappings_to_car", DependencyType.SUCCESS_REQUIRED)], - # }, + { + "key": "populate_vep_for_score_set", + "function": "populate_vep_for_score_set", + "type": JobType.MAPPED_VARIANT_ANNOTATION, + "params": { + "correlation_id": None, # Required param to be filled in at runtime + "score_set_id": None, # Required param to be filled in at runtime + }, + "dependencies": [("reverse_translate_variants_for_score_set", DependencyType.SUCCESS_REQUIRED)], + }, ] diff --git a/tests/scripts/test_run_score_set_pipelines.py b/tests/scripts/test_run_score_set_pipelines.py index f2d1f2722..49498df11 100644 --- a/tests/scripts/test_run_score_set_pipelines.py +++ b/tests/scripts/test_run_score_set_pipelines.py @@ -339,14 +339,12 @@ def test_fast_annotate_leaf_resolves_against_map_annotate_score_set(self): "poll_uniprot_mapping_jobs_for_score_set", } - # TODO(#772) - @pytest.mark.skip(reason="vep currently disabled") def test_vep_leaf_resolves_against_map_annotate_score_set(self): jobs = PIPELINE_DEFINITIONS["map_annotate_score_set"]["job_definitions"] subset = resolve_job_subset(jobs, frozenset({"populate_vep_for_score_set"})) assert {j["key"] for j in subset} == { "map_variants_for_score_set", - "submit_score_set_mappings_to_car", + "reverse_translate_variants_for_score_set", "populate_vep_for_score_set", } @@ -362,8 +360,7 @@ def test_vep_leaf_resolves_against_map_annotate_score_set(self): "poll_uniprot_mapping_jobs_for_score_set", } ), - # TODO(#772) - # frozenset({"populate_vep_for_score_set"}), + frozenset({"populate_vep_for_score_set"}), ], ) def test_presets_against_annotate_score_set_exclude_mapping_job(self, leaf): @@ -376,8 +373,6 @@ def test_missing_leaf_key_raises(self): with pytest.raises(ValueError): resolve_job_subset(jobs, frozenset({"populate_vep_for_score_set"})) - # TODO(#772) - @pytest.mark.skip(reason="vep currently disabled") def test_preserves_base_pipeline_order(self): jobs = PIPELINE_DEFINITIONS["map_annotate_score_set"]["job_definitions"] subset = resolve_job_subset(jobs, frozenset({"populate_vep_for_score_set"})) From 918aed2edfd32a10e8dd707bfbd56d41e040cff0 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Fri, 28 Aug 2026 15:58:37 -0700 Subject: [PATCH 126/252] feat(reverse-translation): record non-translatable consequences as a skip, not a failure (#767) Split the reverse-translation error loop on the library's typed reason: NOT_TRANSLATABLE maps to (not_applicable, not_translatable) and is counted as skipped; every other error keeps the failed path. Adds the NOT_TRANSLATABLE event reason. This keeps benign "nothing to translate" variants out of the FAILED tally, which the retroactive backfill (#747) depends on for a trustworthy failure count. --- src/mavedb/models/enums/event_reason.py | 1 + .../variant_processing/reverse_translation.py | 32 ++++- .../test_reverse_translation.py | 123 +++++++++++++++++- 3 files changed, 148 insertions(+), 8 deletions(-) diff --git a/src/mavedb/models/enums/event_reason.py b/src/mavedb/models/enums/event_reason.py index 2082a6163..ece15e554 100644 --- a/src/mavedb/models/enums/event_reason.py +++ b/src/mavedb/models/enums/event_reason.py @@ -43,6 +43,7 @@ class EventReason(str, Enum): "protein_level_allele" # protein allele excluded from nucleotide-level annotation (ClinVar, gnomAD) ) NO_ASSAY_LEVEL_HGVS = "no_assay_level_hgvs" # no assay-level HGVS to translate (RT) + NOT_TRANSLATABLE = "not_translatable" # protein consequence's edit type has no DNA equivalence class (RT) # failed — errored API_ERROR = "api_error" # network/timeout/upstream error (ClinVar, VEP, LDH, CAR no-response) diff --git a/src/mavedb/worker/jobs/variant_processing/reverse_translation.py b/src/mavedb/worker/jobs/variant_processing/reverse_translation.py index a6cdfc257..ea1d311a6 100644 --- a/src/mavedb/worker/jobs/variant_processing/reverse_translation.py +++ b/src/mavedb/worker/jobs/variant_processing/reverse_translation.py @@ -29,7 +29,12 @@ from variant_annotation import __version__ as variant_annotation_version from variant_annotation.lib.accessions import looks_like_refseq_protein_accession from variant_annotation.lib.translation import construct_equivalent_variants -from variant_annotation.lib.translation.types import TranslationConfig, VariantInput, WtCodonMode +from variant_annotation.lib.translation.types import ( + TranslationConfig, + TranslationErrorReason, + VariantInput, + WtCodonMode, +) from mavedb.lib.annotation_status_manager import AnnotationStatusManager from mavedb.lib.hgvs import extract_accession, strip_protein_prediction_parens @@ -501,12 +506,25 @@ async def reverse_translate_variants_for_score_set( MappingRecordAllele.mapping_record_id.in_(current_record_ids), ) - # TODO#767: some non-substitution consequences (del/ins/delins/fs/ext) have no synonymous - # equivalence class, so they arrive here as TranslationErrors and are miscounted as - # FAILED. Classify by the protein consequence's edit type up front and map these to - # SKIPPED instead of pattern-matching engine error strings. + # The library types each error's reason: NOT_TRANSLATABLE is a benign structural gap (the + # protein consequence's edit type — del/ins/delins/fs/ext/stop-loss — has no DNA equivalence + # class to construct), everything else is a genuine failure. Split on that typed reason rather + # than pattern-matching the engine's error text. for error in errors: _rec, variant = variant_input_map[id(error.input)] + if error.reason is TranslationErrorReason.NOT_TRANSLATABLE: + # A structural gap ("we could not ask"), not a failure — count it as a skip and record + # NOT_APPLICABLE so it never pollutes the failed tally (mirrors NO_ASSAY_LEVEL_HGVS). + annotation_counts["skipped"] += 1 + _annotate_translation( + annotation_manager, + variant_id=variant.id, + disposition=Disposition.NOT_APPLICABLE, + reason=EventReason.NOT_TRANSLATABLE, + metadata={"hgvs_input": error.input.hgvs, "error_message": error.error}, + ) + continue + annotation_counts["failed"] += 1 _annotate_translation( annotation_manager, @@ -516,7 +534,9 @@ async def reverse_translate_variants_for_score_set( metadata={"hgvs_input": error.input.hgvs, "error_message": error.error}, ) - annotation_counts["skipped"] = len(skipped_variants) + # skipped already holds the NOT_TRANSLATABLE skips counted above; add the transcript-unresolved + # skips (skipped_variants) rather than overwriting. + annotation_counts["skipped"] += len(skipped_variants) for p in skipped_variants: category = target_category_by_gene.get(p.target_gene_id) if p.target_gene_id is not None else None reason, disposition = _classify_skip(p, category) diff --git a/tests/worker/jobs/variant_processing/test_reverse_translation.py b/tests/worker/jobs/variant_processing/test_reverse_translation.py index d3ffb004e..b32f18d2d 100644 --- a/tests/worker/jobs/variant_processing/test_reverse_translation.py +++ b/tests/worker/jobs/variant_processing/test_reverse_translation.py @@ -12,6 +12,7 @@ from variant_annotation.lib.translation.types import ( ProjectionPair, TranslationError, + TranslationErrorReason, TranslationResult, WtCodonMode, ) @@ -61,7 +62,11 @@ def model_dump(self, **kwargs) -> dict: return {"type": self.vrs_type, "id": self.id} -def fake_construct(results_by_hgvs: dict, errors_by_hgvs: dict | None = None): +def fake_construct( + results_by_hgvs: dict, + errors_by_hgvs: dict | None = None, + not_translatable_by_hgvs: dict | None = None, +): """Build a stand-in for ``construct_equivalent_variants``. The job correlates each TranslationResult/TranslationError back to its originating @@ -77,13 +82,26 @@ def fake_construct(results_by_hgvs: dict, errors_by_hgvs: dict | None = None): Each pair becomes one ProjectionPair, mirroring the position-aligned coding↔genomic rows the real reverse-translate CLI emits (with ``--one-row-per-input``). + + ``errors_by_hgvs`` maps an assay HGVS to a genuine-failure message (reason ``FAILED``); + ``not_translatable_by_hgvs`` maps one to a benign non-translatable message (reason + ``NOT_TRANSLATABLE``), standing in for an edit type the library screens out up front. """ errors_by_hgvs = errors_by_hgvs or {} + not_translatable_by_hgvs = not_translatable_by_hgvs or {} def _construct(inputs, *, transcripts, coordinates, config): results, errors = [], [] for inp in inputs: - if inp.hgvs in errors_by_hgvs: + if inp.hgvs in not_translatable_by_hgvs: + errors.append( + TranslationError( + input=inp, + error=not_translatable_by_hgvs[inp.hgvs], + reason=TranslationErrorReason.NOT_TRANSLATABLE, + ) + ) + elif inp.hgvs in errors_by_hgvs: errors.append(TranslationError(input=inp, error=errors_by_hgvs[inp.hgvs])) elif inp.hgvs in results_by_hgvs: spec = results_by_hgvs[inp.hgvs] @@ -964,6 +982,107 @@ async def test_partial_success_succeeds_with_mixed_annotations( assert len(_cross_level_events(session, sample_score_set.id, disposition="failed")) == 1 assert len(_non_authoritative_links(session)) == 1 + async def test_non_translatable_edit_is_skipped_not_failed( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """A protein consequence whose edit type has no DNA equivalence class (the library screens it + out as NOT_TRANSLATABLE) is a benign structural gap, not a failure: it is counted as skipped + and recorded as (not_applicable, not_translatable) — never in the failed tally. This is the + #767 fix: such variants must not pollute FAILED for the retroactive backfill.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + assay_hgvs = "NM_000000.1:c.1A>G" + construct = fake_construct( + {}, + not_translatable_by_hgvs={ + assay_hgvs: "Protein edit type has no DNA equivalence class to reverse-translate" + }, + ) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", fake_translate({})), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + # A whole score set of non-translatable variants is a benign no-op, not a job failure. + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 0, "failed": 0, "skipped": 1, "alleles_created": 0} + + # No alleles/links, and no failed event at all. + assert _non_authoritative_links(session) == [] + assert _cross_level_events(session, sample_score_set.id, disposition="failed") == [] + + # Recorded as a structural gap: not_applicable / not_translatable, carrying the input HGVS. + events = _cross_level_events(session, sample_score_set.id, reason="not_translatable") + assert len(events) == 1 + assert events[0].disposition == "not_applicable" + assert events[0].event_metadata["hgvs_input"] == assay_hgvs + + async def test_non_translatable_skip_coexists_with_success_and_failure( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """Across three variants — one translatable, one non-translatable, one genuinely erroring — + each lands in its own bucket: translated / skipped / failed. The genuine error still records + FAILED (translation_error); the non-translatable one is a not_translatable skip.""" + variant_ok = Variant(score_set_id=sample_score_set.id, urn="variant:1", hgvs_nt="NM_000000.1:c.1A>G", data={}) + variant_skip = Variant(score_set_id=sample_score_set.id, urn="variant:2", hgvs_nt="NM_000000.1:c.2G>T", data={}) + variant_fail = Variant(score_set_id=sample_score_set.id, urn="variant:3", hgvs_nt="NM_000000.1:c.3T>A", data={}) + session.add_all([variant_ok, variant_skip, variant_fail]) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + c_candidate = "NM_000001.1:c.5A>G" + construct = fake_construct( + {"NM_000000.1:c.1A>G": [(c_candidate, None)]}, + errors_by_hgvs={"NM_000000.1:c.3T>A": "forward translation failed"}, + not_translatable_by_hgvs={"NM_000000.1:c.2G>T": "no DNA equivalence class"}, + ) + translate = fake_translate({c_candidate: "ga4gh:VA.coding"}) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 1, "failed": 1, "skipped": 1, "alleles_created": 1} + + # The genuine error is FAILED with reason translation_error — distinct from the benign skip. + failed = _cross_level_events(session, sample_score_set.id, disposition="failed") + assert len(failed) == 1 + assert failed[0].reason == "translation_error" + assert failed[0].event_metadata["error_message"] == "forward translation failed" + + # The non-translatable one is not_applicable / not_translatable. + skipped = _cross_level_events(session, sample_score_set.id, reason="not_translatable") + assert len(skipped) == 1 + assert skipped[0].disposition == "not_applicable" + async def test_partial_candidate_translation_failure_keeps_success_with_metadata( self, session, From d9668d00a2463cb61bcaebffc2caafc7b045208c Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 2 Sep 2026 10:18:38 -0700 Subject: [PATCH 127/252] fix(dependencies): add moto-init dependency for app and worker services --- docker-compose-dev.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 925c30f0d..e7054721e 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -10,6 +10,7 @@ services: depends_on: - db - redis + - moto-init env_file: - settings/.env.dev environment: @@ -54,6 +55,7 @@ services: depends_on: - db - redis + - moto-init dcd-mapping: build: ../dcd_mapping From cc762ce608593f84f31541688b5052374b8f75f5 Mon Sep 17 00:00:00 2001 From: David Reinhart Date: Tue, 1 Sep 2026 13:12:24 -0700 Subject: [PATCH 128/252] feat(urns): forward URNs retired by publication to the published record publish_score_set overwrites the tmp: URN of an experiment set, an experiment and a score set in place, and refresh_variant_urns rebuilds every variant URN from the score set's. Nothing recorded the old value, so a link already shared to the unpublished record began returning 404 under a name the caller had no way to guess. Reloading a score set page after publishing it was enough to hit this. Record what each URN became in a new urn_redirects table, and resolve it in forward_retired_urns, an application-wide dependency: a read naming a retired URN is answered 308 to the same path under the record's current URN. One implementation covers every route that takes a URN, sub-resources included, and since substitution operates on the URN substring, a variant follows its score set without a row of its own. A dependency rather than ASGI middleware, because it needs the request's session; middleware runs outside dependency resolution, so it would open a session of its own that no dependency_overrides could redirect. Reads only: an owner is permitted to publish a published score set, so forwarding a stale POST .../publish would rename a live public record. And only onto a target confirmed public, since a Location header names its target to an anonymous caller before any route checks a permission. That check also keeps a deleted record's surviving row from answering a permanent redirect with a 404. The dependency reads the path from the ASGI scope. request.url.path truncates at the '#' in a variant URN, because Starlette rebuilds that URL by re-parsing it, which turns everything after the '#' into a fragment and drops the variant number, the sub-resource and the query string. lib/logging/context.py has the same pattern and is left for a separate change. Forwarding is one hop, which is all that can arise while nothing renames a published record. URNs retired before this are unrecoverable, since publication overwrote them and kept no history, so the table is not backfilled and links to records published earlier stay broken. --- .../c4b18d0f7a92_add_urn_redirects.py | 35 ++++ src/mavedb/lib/urn_redirects.py | 187 ++++++++++++++++++ src/mavedb/models/__init__.py | 1 + src/mavedb/models/urn_redirect.py | 36 ++++ src/mavedb/routers/score_sets.py | 8 + src/mavedb/server_main.py | 7 +- tests/lib/test_urn_redirects.py | 82 ++++++++ tests/routers/test_urn_redirects.py | 154 +++++++++++++++ 8 files changed, 508 insertions(+), 2 deletions(-) create mode 100644 alembic/versions/c4b18d0f7a92_add_urn_redirects.py create mode 100644 src/mavedb/lib/urn_redirects.py create mode 100644 src/mavedb/models/urn_redirect.py create mode 100644 tests/lib/test_urn_redirects.py create mode 100644 tests/routers/test_urn_redirects.py diff --git a/alembic/versions/c4b18d0f7a92_add_urn_redirects.py b/alembic/versions/c4b18d0f7a92_add_urn_redirects.py new file mode 100644 index 000000000..9da5e8dc7 --- /dev/null +++ b/alembic/versions/c4b18d0f7a92_add_urn_redirects.py @@ -0,0 +1,35 @@ +"""Add urn_redirects, forwarding URNs that publication has retired + +Revision ID: c4b18d0f7a92 +Revises: a7f3c2e9b104 +Create Date: 2026-09-01 00:00:00.000000 + +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "c4b18d0f7a92" +down_revision = "a7f3c2e9b104" +branch_labels = None +depends_on = None + + +def upgrade(): + # Not backfilled: publication overwrote each record's temporary URN in place and kept no history of + # it, so the URNs retired before this table existed cannot be recovered. + op.create_table( + "urn_redirects", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("old_urn", sa.String(length=64), nullable=False), + sa.Column("new_urn", sa.String(length=64), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index("ix_urn_redirects_old_urn", "urn_redirects", ["old_urn"], unique=True) + + +def downgrade(): + op.drop_index("ix_urn_redirects_old_urn", table_name="urn_redirects") + op.drop_table("urn_redirects") diff --git a/src/mavedb/lib/urn_redirects.py b/src/mavedb/lib/urn_redirects.py new file mode 100644 index 000000000..584a0c232 --- /dev/null +++ b/src/mavedb/lib/urn_redirects.py @@ -0,0 +1,187 @@ +"""Forwarding of URNs that publication has retired. + +A dataset is created with a ``tmp:`` URN and keeps it until it is published, at which point +:func:`mavedb.routers.score_sets.publish_score_set` overwrites the URN in place with a permanent one. +Any link already shared to the unpublished record then stops resolving, which is what +https://github.com/VariantEffect/mavedb-ui/issues/617 reports: the record is still there, under a name +the caller has no way to guess. + +Publication records what each retired URN became, and every read is checked here before it reaches its +route. A request naming a retired URN is answered with ``308 Permanent Redirect`` to the same path +under the record's current URN. + +Three limits: + + - Only a public target is forwarded to. A ``Location`` header names the record it points at, and + forwarding happens before any route checks a permission, so the header would disclose that URN to + an anonymous caller. See _target_is_public. + - Forwarding is one hop. Nothing in the application renames a published record, so a chain cannot + arise; if one ever could, resolution here would need to follow it. + - URNs retired before this was added are unrecoverable. Publication overwrote them and no history of + them was kept, so links to records published earlier stay broken. +""" + +import logging +from typing import Optional +from urllib.parse import quote + +from fastapi import Depends, HTTPException +from sqlalchemy import select +from sqlalchemy.orm import Session +from starlette.requests import Request + +from mavedb.deps import get_db +from mavedb.lib.logging.context import logging_context, save_to_logging_context +from mavedb.lib.validation.urn_re import ( + MAVEDB_EXPERIMENT_SET_URN_RE, + MAVEDB_EXPERIMENT_URN_RE, + MAVEDB_SCORE_SET_URN_RE, + MAVEDB_TMP_URN_RE, +) +from mavedb.models.experiment import Experiment +from mavedb.models.experiment_set import ExperimentSet +from mavedb.models.score_set import ScoreSet +from mavedb.models.urn_redirect import UrnRedirect + +logger = logging.getLogger(__name__) + +# Methods whose requests are forwarded. See forward_retired_urns for why a write is not. +SAFE_METHODS = frozenset({"GET", "HEAD"}) + +# The record kinds publication renames, recognized by the shape of the URN it gave them. Matched with +# fullmatch, under which the three patterns are mutually exclusive. +FORWARDING_TARGET_MODELS = ( + (MAVEDB_SCORE_SET_URN_RE, ScoreSet), + (MAVEDB_EXPERIMENT_URN_RE, Experiment), + (MAVEDB_EXPERIMENT_SET_URN_RE, ExperimentSet), +) + + +def record_urn_redirect(db: Session, old_urn: Optional[str], new_urn: str) -> None: + """ + Record that a record's URN has changed, so that requests naming the old one can be forwarded. + + Staged on the session rather than committed, so that a caller which reassigns several URNs -- as + publication does, across an experiment set, an experiment and a score set -- commits the redirects + together with the renames they describe. + + :param db: An active database session. + :param old_urn: The URN being retired. A record that never had one, or a rename that is not a + change, is not worth a row and is ignored. + :param new_urn: The URN replacing it. + """ + if not old_urn or old_urn == new_urn: + return + + db.add(UrnRedirect(old_urn=old_urn, new_urn=new_urn)) # type: ignore[call-arg] + + +def _target_is_public(db: Session, urn: str) -> bool: + """ + Report whether the record a redirect points to is one that may be named to any caller. + + A ``Location`` header discloses the URN it carries, to whoever asked -- including an anonymous + caller, since forwarding happens before a route checks anything. Publication only ever records a + redirect onto a record it is making public, and nothing in the application returns a published + record to private, so a private target should not arise; a row written out of band, or by some + later feature, would be enough for one to. Withholding on anything but a confirmed public record + keeps that from becoming a disclosure. + + A target that no longer exists is likewise not public: a deleted record leaves its redirect row + behind, and forwarding to it would answer a permanent redirect with a 404. + + :param db: An active database session. + :param urn: The URN a redirect points to. + :return: True only if a record under this URN exists and is public. + """ + for urn_re, model in FORWARDING_TARGET_MODELS: + if urn_re.fullmatch(urn): + private = db.execute(select(model.private).where(model.urn == urn)).scalar_one_or_none() + return private is False + + return False + + +def forwarded_path(db: Session, path: str) -> Optional[str]: + """ + Rewrite a request path so that any retired URN in it names the record's current URN instead. + + Substitution is by substring, not by path segment, so a variant URN -- ``{score_set_urn}#{n}`` -- + is carried along by its score set's redirect. + + :param db: An active database session. + :param path: The decoded request path. + :return: The rewritten path, or None if the path should be served as it is: it names no retired URN, + or one whose target this caller must not be told about. See _target_is_public. + """ + # A live temporary URN belongs to an unpublished record and matches nothing in the table, so the + # lookup below distinguishes the two cases and no separate check for publication is needed. + candidate_urns = set(MAVEDB_TMP_URN_RE.findall(path)) + if not candidate_urns: + return None + + redirects = db.execute( + select(UrnRedirect.old_urn, UrnRedirect.new_urn).where(UrnRedirect.old_urn.in_(candidate_urns)) + ).all() + if not redirects: + return None + + forwarded = path + for old_urn, new_urn in redirects: + if not _target_is_public(db, new_urn): + return None + forwarded = forwarded.replace(old_urn, new_urn) + + return forwarded + + +def forward_retired_urns(request: Request, db: Session = Depends(get_db)) -> None: + """ + Forward a request that names a retired URN to the same resource under its current URN. + + Installed as an application-wide dependency in :mod:`mavedb.server_main`, which is what makes one + implementation cover every route that takes a URN, sub-resources included: a stale link to a score + set's scores CSV or mapped variants is forwarded on the same terms as a link to the score set. + + A dependency rather than ASGI middleware, though it sits at the same single point in the request + path, because it needs the request's database session. Middleware runs outside dependency + resolution, so it would have to open a session of its own, which no ``dependency_overrides`` could + redirect and which would therefore reach past the test database. + + ``308`` rather than ``301``: the redirect is permanent, and 308 forbids a client from rewriting the + request to a GET on the way, which is what makes the header safe to emit for any method. + + Only reads are forwarded. What the issue asks for is that shared *links* keep working, and a write + is a different proposition: the caller addressed a private draft, and the record now under that URN + is published, with different rules and a wider audience. ``POST .../publish`` is the sharp case -- + an owner is permitted to publish a published score set, so forwarding a stale one would rename a + live public record. A write to a retired URN keeps getting the 404 it gets today, which tells the + client to look the record up again. + """ + if request.scope["method"] not in SAFE_METHODS: + return + + # The ASGI scope rather than request.url: Starlette builds request.url by re-parsing the decoded + # path, so a variant URN's '#' starts a fragment there and everything after it -- the variant + # number, the sub-resource, the query -- is silently dropped. + path = request.scope["path"] + query = request.scope.get("query_string", b"").decode("ascii") + + forwarded = forwarded_path(db, path) + if forwarded is None: + return + + # The scope's path is percent-decoded, so a '#' in it has to be re-encoded or it would open a + # fragment in the header. Relative, so that a proxy's scheme and host survive. + location = quote(forwarded, safe="/:") + if query: + location = f"{location}?{query}" + + save_to_logging_context({"requested_resource": path, "forwarded_to": forwarded}) + logger.info(msg="Forwarding a request that named a retired URN.", extra=logging_context()) + + raise HTTPException( + status_code=308, + detail="This URN was replaced when the record was published; the record has moved permanently.", + headers={"Location": location}, + ) diff --git a/src/mavedb/models/__init__.py b/src/mavedb/models/__init__.py index 2f0d65b48..5be50ec13 100644 --- a/src/mavedb/models/__init__.py +++ b/src/mavedb/models/__init__.py @@ -35,6 +35,7 @@ "taxonomy", "uniprot_identifier", "uniprot_offset", + "urn_redirect", "user", "variant_annotation_status", "variant", diff --git a/src/mavedb/models/urn_redirect.py b/src/mavedb/models/urn_redirect.py new file mode 100644 index 000000000..5cd24f38a --- /dev/null +++ b/src/mavedb/models/urn_redirect.py @@ -0,0 +1,36 @@ +""" +SQLAlchemy model for URNs that publication has retired. +""" + +from datetime import datetime + +from sqlalchemy import DateTime, Integer, String, func +from sqlalchemy.orm import Mapped, mapped_column + +from mavedb.db.base import Base + + +class UrnRedirect(Base): + """ + Records that a record's URN was replaced, and by what. + + Publishing a dataset overwrites the ``tmp:`` URN it was created with, so every link already + shared to the unpublished record stops resolving. One row is written here per URN publication + retires, and requests naming a retired URN are forwarded to its replacement. + + Only experiment sets, experiments and score sets get rows. A variant's URN is + ``{score_set_urn}#{n}``, so forwarding replaces the retired URN wherever it appears in a request + path and a variant follows its score set without a row of its own. + """ + + __tablename__ = "urn_redirects" + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + + old_urn: Mapped[str] = mapped_column(String(64), nullable=False, unique=True, index=True) + new_urn: Mapped[str] = mapped_column(String(64), nullable=False) + + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now()) + + def __repr__(self) -> str: + return f"" diff --git a/src/mavedb/routers/score_sets.py b/src/mavedb/routers/score_sets.py index aff713429..d38988695 100644 --- a/src/mavedb/routers/score_sets.py +++ b/src/mavedb/routers/score_sets.py @@ -77,6 +77,7 @@ from mavedb.lib.target_genes import find_or_create_target_gene_by_accession, find_or_create_target_gene_by_sequence from mavedb.lib.taxonomies import find_or_create_taxonomy from mavedb.lib.types.authentication import UserData +from mavedb.lib.urn_redirects import record_urn_redirect from mavedb.lib.urns import ( generate_experiment_set_urn, generate_experiment_urn, @@ -2563,7 +2564,9 @@ async def publish_score_set( published_date = date.today() if item.experiment.experiment_set.private or not item.experiment.experiment_set.published_date: + retired_experiment_set_urn = item.experiment.experiment_set.urn item.experiment.experiment_set.urn = generate_experiment_set_urn(db) + record_urn_redirect(db, retired_experiment_set_urn, item.experiment.experiment_set.urn) item.experiment.experiment_set.private = False item.experiment.experiment_set.published_date = published_date db.add(item.experiment.experiment_set) @@ -2571,18 +2574,23 @@ async def publish_score_set( save_to_logging_context({"experiment_set": item.experiment.experiment_set.urn}) if item.experiment.private or not item.experiment.published_date: + retired_experiment_urn = item.experiment.urn item.experiment.urn = generate_experiment_urn( db, item.experiment.experiment_set, experiment_is_meta_analysis=len(item.meta_analyzes_score_sets) > 0, ) + record_urn_redirect(db, retired_experiment_urn, item.experiment.urn) item.experiment.private = False item.experiment.published_date = published_date db.add(item.experiment) save_to_logging_context({"experiment": item.experiment.urn}) + retired_score_set_urn = item.urn item.urn = generate_score_set_urn(db, item.experiment) + # Variant URNs are rewritten below from the score set's, so this one redirect forwards them too. + record_urn_redirect(db, retired_score_set_urn, item.urn) item.private = False item.published_date = published_date refresh_variant_urns(db, item) diff --git a/src/mavedb/server_main.py b/src/mavedb/server_main.py index 880bfcfe1..7f2233e28 100644 --- a/src/mavedb/server_main.py +++ b/src/mavedb/server_main.py @@ -3,7 +3,7 @@ import uvicorn from eutils._internal.exceptions import EutilsRequestError # type: ignore -from fastapi import FastAPI +from fastapi import Depends, FastAPI from fastapi.encoders import jsonable_encoder from fastapi.exceptions import RequestValidationError from fastapi.middleware.cors import CORSMiddleware @@ -37,6 +37,7 @@ from mavedb.lib.middleware import CatchAllErrorMiddleware from mavedb.lib.permissions.exceptions import PermissionException from mavedb.lib.slack import send_slack_error +from mavedb.lib.urn_redirects import forward_retired_urns from mavedb.models import * # noqa: F403 from mavedb.routers import ( access_keys, @@ -75,7 +76,9 @@ # an instance of the related class has been created. configure_mappers() -app = FastAPI() +# forward_retired_urns is applied to every route, so that one implementation forwards a read of a URN +# publication has retired wherever that URN points: a record, or any of its sub-resources. +app = FastAPI(dependencies=[Depends(forward_retired_urns)]) # `add_middleware` inserts at the head of the stack, so the *first* call here is the innermost layer. # CatchAllErrorMiddleware must sit inside both CORSMiddleware and the context middleware: CORS has to # decorate the 500 it produces, and the correlation id it returns comes from the context. diff --git a/tests/lib/test_urn_redirects.py b/tests/lib/test_urn_redirects.py new file mode 100644 index 000000000..9f5efb116 --- /dev/null +++ b/tests/lib/test_urn_redirects.py @@ -0,0 +1,82 @@ +"""Tests for forwarding URNs that publication has retired.""" + +import pytest + +from mavedb.lib.urn_redirects import forwarded_path, record_urn_redirect +from mavedb.models.urn_redirect import UrnRedirect + +from tests.helpers.constants import VALID_SCORE_SET_URN + +RETIRED_URN = "tmp:00000000-0000-4000-8000-000000000001" +PUBLISHED_URN = VALID_SCORE_SET_URN + + +@pytest.mark.integration +class TestRecordUrnRedirect: + def test_records_a_rename(self, session): + record_urn_redirect(session, RETIRED_URN, PUBLISHED_URN) + session.commit() + + redirect = session.query(UrnRedirect).one() + assert redirect.old_urn == RETIRED_URN + assert redirect.new_urn == PUBLISHED_URN + + def test_ignores_a_record_that_had_no_urn(self, session): + record_urn_redirect(session, None, PUBLISHED_URN) + session.commit() + + assert session.query(UrnRedirect).count() == 0 + + def test_ignores_a_rename_that_changes_nothing(self, session): + record_urn_redirect(session, PUBLISHED_URN, PUBLISHED_URN) + session.commit() + + assert session.query(UrnRedirect).count() == 0 + + +@pytest.mark.integration +class TestForwardedPath: + @pytest.fixture + def retired(self, session, setup_lib_db_with_score_set): + """Retire a temporary URN onto a real, public score set. + + A real record is needed, not just a row in the table: forwarding withholds a target it cannot + confirm is public, so a redirect pointing at nothing forwards nowhere. + """ + setup_lib_db_with_score_set.private = False + record_urn_redirect(session, RETIRED_URN, PUBLISHED_URN) + session.commit() + + def test_withholds_a_target_that_is_private(self, session, retired, setup_lib_db_with_score_set): + setup_lib_db_with_score_set.private = True + session.commit() + + assert forwarded_path(session, f"/api/v1/score-sets/{RETIRED_URN}") is None + + def test_withholds_a_target_that_does_not_exist(self, session): + """A deleted record leaves its redirect row behind.""" + record_urn_redirect(session, RETIRED_URN, PUBLISHED_URN) + session.commit() + + assert forwarded_path(session, f"/api/v1/score-sets/{RETIRED_URN}") is None + + def test_forwards_a_retired_urn(self, session, retired): + assert forwarded_path(session, f"/api/v1/score-sets/{RETIRED_URN}") == f"/api/v1/score-sets/{PUBLISHED_URN}" + + def test_forwards_a_sub_resource_of_a_retired_urn(self, session, retired): + assert ( + forwarded_path(session, f"/api/v1/score-sets/{RETIRED_URN}/scores") + == f"/api/v1/score-sets/{PUBLISHED_URN}/scores" + ) + + def test_forwards_a_variant_of_a_retired_score_set(self, session, retired): + """A variant URN is built from its score set's, so the score set's redirect carries it.""" + assert forwarded_path(session, f"/api/v1/variants/{RETIRED_URN}#4") == f"/api/v1/variants/{PUBLISHED_URN}#4" + + def test_leaves_a_path_naming_no_temporary_urn_alone(self, session, retired): + assert forwarded_path(session, f"/api/v1/score-sets/{PUBLISHED_URN}") is None + + def test_leaves_a_live_temporary_urn_alone(self, session, retired): + """An unpublished record still answers to its temporary URN, and has no row in the table.""" + live_urn = "tmp:00000000-0000-4000-8000-000000000002" + assert forwarded_path(session, f"/api/v1/score-sets/{live_urn}") is None diff --git a/tests/routers/test_urn_redirects.py b/tests/routers/test_urn_redirects.py new file mode 100644 index 000000000..cdeeb095a --- /dev/null +++ b/tests/routers/test_urn_redirects.py @@ -0,0 +1,154 @@ +# ruff: noqa: E402 + +from urllib.parse import quote, unquote + +import pytest + +arq = pytest.importorskip("arq") +cdot = pytest.importorskip("cdot") +fastapi = pytest.importorskip("fastapi") + +from mavedb.models.experiment import Experiment as ExperimentDbModel +from mavedb.models.experiment_set import ExperimentSet as ExperimentSetDbModel +from mavedb.models.score_set import ScoreSet as ScoreSetDbModel + +from tests.helpers.util.experiment import create_experiment +from tests.helpers.util.score_set import ( + create_seq_score_set, + create_seq_score_set_with_variants, + publish_score_set, +) + +UNKNOWN_TMP_URN = "tmp:00000000-0000-4000-8000-00000000ffff" + +COLLECTIONS = {"experiment_set": "experiment-sets", "experiment": "experiments", "score_set": "score-sets"} + + +@pytest.fixture +def published(session, data_provider, client, setup_router_db, data_files): + """Publish a score set, and report what each of its records was called before and after.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + published_score_set = publish_score_set(client, score_set["urn"]) + + return { + "experiment_set": (experiment["experimentSetUrn"], published_score_set["experiment"]["experimentSetUrn"]), + "experiment": (experiment["urn"], published_score_set["experiment"]["urn"]), + "score_set": (score_set["urn"], published_score_set["urn"]), + } + + +@pytest.mark.integration +class TestForwardingRetiredUrns: + @pytest.mark.parametrize("record", ["experiment_set", "experiment", "score_set"]) + def test_retired_urn_is_forwarded_to_the_published_record(self, client, published, record): + """Publication renames all three records, so a stale link to any of them has to resolve.""" + retired_urn, published_urn = published[record] + collection = COLLECTIONS[record] + + response = client.get(f"/api/v1/{collection}/{retired_urn}", follow_redirects=False) + + assert response.status_code == 308 + assert response.headers["location"] == f"/api/v1/{collection}/{published_urn}" + + @pytest.mark.parametrize("record", ["experiment_set", "experiment", "score_set"]) + def test_a_client_following_the_forward_reaches_the_record(self, client, published, record): + retired_urn, published_urn = published[record] + collection = COLLECTIONS[record] + + response = client.get(f"/api/v1/{collection}/{retired_urn}") + + assert response.status_code == 200 + assert response.json()["urn"] == published_urn + + def test_a_sub_resource_of_a_retired_urn_is_forwarded(self, client, published): + """The complaint in mavedb-ui#617 is about a page, which loads more than the record itself.""" + retired_urn, published_urn = published["score_set"] + + response = client.get(f"/api/v1/score-sets/{retired_urn}/scores", follow_redirects=False) + + assert response.status_code == 308 + assert response.headers["location"] == f"/api/v1/score-sets/{published_urn}/scores" + + def test_a_variant_of_a_retired_score_set_is_forwarded(self, client, published): + """A variant URN is derived from its score set's, and needs no redirect of its own.""" + retired_urn, published_urn = published["score_set"] + + variant_path = f"/api/v1/variants/{quote(f'{retired_urn}#1', safe='')}/csv-namespaces" + + response = client.get(variant_path, follow_redirects=False) + + assert response.status_code == 308 + location = response.headers["location"] + assert unquote(location) == f"/api/v1/variants/{published_urn}#1/csv-namespaces" + # The '#' has to come back encoded, or a client reads the rest of the location as a fragment. + assert "%23" in location + assert client.get(variant_path).status_code == 200 + + def test_a_query_string_survives_forwarding(self, client, published): + retired_urn, published_urn = published["score_set"] + + response = client.get(f"/api/v1/score-sets/{retired_urn}/scores?start=0&limit=1", follow_redirects=False) + + assert response.status_code == 308 + assert response.headers["location"] == f"/api/v1/score-sets/{published_urn}/scores?start=0&limit=1" + + def test_a_live_temporary_urn_is_served_not_forwarded(self, client, setup_router_db): + """An unpublished record still answers to the temporary URN it was created with.""" + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + + response = client.get(f"/api/v1/score-sets/{score_set['urn']}", follow_redirects=False) + + assert response.status_code == 200 + assert response.json()["urn"] == score_set["urn"] + + def test_an_unknown_temporary_urn_is_still_a_404(self, client, setup_router_db): + response = client.get(f"/api/v1/score-sets/{UNKNOWN_TMP_URN}", follow_redirects=False) + + assert response.status_code == 404 + + @pytest.mark.parametrize( + "record,model", + [ + ("score_set", ScoreSetDbModel), + ("experiment", ExperimentDbModel), + ("experiment_set", ExperimentSetDbModel), + ], + ) + def test_a_retired_urn_is_not_forwarded_to_a_private_record(self, session, client, published, record, model): + """A Location header names its target, to an anonymous caller, before any route checks anything. + + Publication only records a redirect onto a record it is making public, and nothing in the + application returns a published record to private, so this state is reached here the only way + it could be reached in production: out of band. + """ + retired_urn, published_urn = published[record] + session.query(model).filter(model.urn == published_urn).one().private = True + session.commit() + + response = client.get(f"/api/v1/{COLLECTIONS[record]}/{retired_urn}", follow_redirects=False) + + assert response.status_code == 404 + assert published_urn not in response.text + + def test_a_retired_urn_is_not_forwarded_to_a_deleted_record(self, session, client, published): + """A deleted record leaves its redirect row behind, pointing at a URN that resolves to nothing.""" + retired_urn, published_urn = published["score_set"] + session.delete(session.query(ScoreSetDbModel).filter(ScoreSetDbModel.urn == published_urn).one()) + session.commit() + + response = client.get(f"/api/v1/score-sets/{retired_urn}", follow_redirects=False) + + assert response.status_code == 404 + assert published_urn not in response.text + + def test_a_write_to_a_retired_urn_is_not_forwarded(self, client, published): + """Forwarding a stale publish request would rename the live public record it reached.""" + retired_urn, _ = published["score_set"] + + response = client.post(f"/api/v1/score-sets/{retired_urn}/publish", follow_redirects=False) + + assert response.status_code == 404 From 613a177d098d4308e39aee654bd8d61527310985 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 2 Sep 2026 11:25:13 -0700 Subject: [PATCH 129/252] feat(deprecation): unify RFC 8594 signalling and usage recording across retired MappedVariant surfaces Add a shared deprecation helper and route every retired surface through it, so the /mapped-variants 301 redirects, the /score-sets/{urn}/mapped-variants 410 tombstone, and the deprecated CSV query parameters all emit consistent Deprecation / Link / Warning headers and record each hit under one queryable deprecation_marker. The redirects name their 1:1 successor as rel=successor-version; the tombstone (no wire-compatible successor) points at variant-details via Warning. Sunset stays unset until a removal date is ratified. The usage marker makes 'who is still on a deprecated surface' answerable before that date is chosen. --- src/mavedb/lib/csv/deprecated_params.py | 18 +----- src/mavedb/lib/deprecation.py | 81 +++++++++++++++++++++++++ src/mavedb/routers/mapped_variant.py | 15 ++++- src/mavedb/routers/score_sets.py | 13 +++- tests/lib/csv/test_deprecated_params.py | 1 + tests/lib/test_deprecation.py | 44 ++++++++++++++ tests/routers/test_mapped_variant.py | 12 ++++ 7 files changed, 165 insertions(+), 19 deletions(-) create mode 100644 src/mavedb/lib/deprecation.py create mode 100644 tests/lib/test_deprecation.py diff --git a/src/mavedb/lib/csv/deprecated_params.py b/src/mavedb/lib/csv/deprecated_params.py index 3199fce5d..e4c692f1d 100644 --- a/src/mavedb/lib/csv/deprecated_params.py +++ b/src/mavedb/lib/csv/deprecated_params.py @@ -10,14 +10,11 @@ TODO(#XXX): remove this module once clients have migrated. """ -import logging from dataclasses import dataclass, field from typing import List, Optional from mavedb.lib.csv.namespaces import CsvNamespace -from mavedb.lib.logging.context import save_to_logging_context - -logger = logging.getLogger(__name__) +from mavedb.lib.deprecation import deprecation_headers, record_deprecated_usage DROP_NA_COLUMNS_DESCRIPTION = ( @@ -48,12 +45,7 @@ class ResolvedCsvParams: def _record(self, name: str, replacement: str) -> None: self.deprecations[name] = replacement - save_to_logging_context({"deprecated_query_parameters": sorted(self.deprecations), "deprecation_marker": True}) - logger.warning( - msg=f"Request used the deprecated query parameter '{name}'; it will be removed in a future" - f" release. Use '{replacement}' instead.", - extra={"deprecated_query_parameter": name, "replacement_query_parameter": replacement}, - ) + record_deprecated_usage(name, successor=replacement) @property def response_headers(self) -> dict[str, str]: @@ -64,11 +56,7 @@ def response_headers(self) -> dict[str, str]: warnings = "; ".join( f"{name} is deprecated, use {replacement}" for name, replacement in sorted(self.deprecations.items()) ) - return { - # RFC 8594. No Sunset header: the removal release is not scheduled. - "Deprecation": "true", - "Warning": f'299 - "{warnings}"', - } + return deprecation_headers(successor=None, warning=warnings) def resolve_deprecated_csv_params( diff --git a/src/mavedb/lib/deprecation.py b/src/mavedb/lib/deprecation.py new file mode 100644 index 000000000..b99eed244 --- /dev/null +++ b/src/mavedb/lib/deprecation.py @@ -0,0 +1,81 @@ +"""Build RFC 8594 deprecation headers for retired routes, parameters, and artifacts. + +RFC 8594 defines ``Deprecation`` (the resource is deprecated) and ``Sunset`` (when it will stop working). +:func:`deprecation_headers` emits ``Deprecation: true`` and, where a replacement exists, a ``Link`` with +``rel="successor-version"``; a ``Warning: 299`` carries human-readable guidance for clients that do not +parse the structured headers. ``Sunset`` is included only when a removal date is supplied. +""" + +import logging +from typing import Optional + +from mavedb.lib.logging.context import logging_context, save_to_logging_context + +logger = logging.getLogger(__name__) + +# The removal date for the MappedVariant-retirement surfaces (the /mapped-variants redirects and the +# score-set tombstone), as an IMF-fixdate string (e.g. "Wed, 01 Jul 2026 00:00:00 GMT"), or None to emit no +# Sunset header. Unset until the team ratifies a date. Scoped to this one campaign — a different deprecation +# should not inherit this date, so pass it explicitly at each MappedVariant-retirement call site rather than +# relying on a builder-wide default. +MAPPED_VARIANT_SUNSET: Optional[str] = None + +# Where an integrator should read about the migration. Points at the dump/API changelog channel. +DEPRECATION_DOCS_URL = "https://mavedb.org/docs/mavedb/changelog.html" + + +def deprecation_headers( + *, + successor: Optional[str] = None, + warning: Optional[str] = None, + sunset: Optional[str] = None, + docs_url: Optional[str] = DEPRECATION_DOCS_URL, +) -> dict[str, str]: + """Build the RFC 8594 header set for a deprecated resource. + + Args: + successor: absolute path or URL of the 1:1 replacement resource, emitted as a + ``Link: ; rel="successor-version"``. Omit when no wire-compatible successor exists + (a cardinality change), and rely on ``warning`` to point the caller at the nearest resource. + warning: human-readable guidance, emitted as ``Warning: 299 - ""``. + sunset: IMF-fixdate removal date, emitted as ``Sunset``. No default — each deprecation campaign has + its own removal timeline, so callers pass their own date constant explicitly. + docs_url: a documentation link, emitted as ``Link: ; rel="deprecation"``. Pass None to omit. + """ + headers: dict[str, str] = {"Deprecation": "true"} + + links = [] + if successor: + links.append(f'<{successor}>; rel="successor-version"') + if docs_url: + links.append(f'<{docs_url}>; rel="deprecation"') + if links: + headers["Link"] = ", ".join(links) + + if sunset: + headers["Sunset"] = sunset + if warning: + headers["Warning"] = f'299 - "{warning}"' + + return headers + + +def record_deprecated_usage(surface: str, *, successor: Optional[str] = None) -> None: + """Record that a request touched a deprecated surface, under one queryable log marker. + + Every deprecated surface — a retired route, a deprecated query parameter, a future one — should show up + the same way, so "who is still on a deprecated surface?" is a single query rather than a per-surface hunt. + + Stamps ``deprecation_marker: True`` into the logging context (the aggregation key), accumulates ``surface`` + into ``deprecated_surfaces`` so several deprecations on one request all show, and logs a warning naming the + surface and its successor. ``surface`` is the thing used (a route path or a parameter name); ``successor`` + is its replacement, if one exists. + """ + surfaces = sorted({*logging_context().get("deprecated_surfaces", []), surface}) + save_to_logging_context({"deprecated_surfaces": surfaces, "deprecation_marker": True}) + logger.warning( + "Request used the deprecated surface %r; it will be removed in a future release.%s", + surface, + f" Use {successor} instead." if successor else "", + extra={"deprecated_surface": surface, "deprecation_successor": successor}, + ) diff --git a/src/mavedb/routers/mapped_variant.py b/src/mavedb/routers/mapped_variant.py index a42fa2503..39417224e 100644 --- a/src/mavedb/routers/mapped_variant.py +++ b/src/mavedb/routers/mapped_variant.py @@ -6,6 +6,7 @@ from fastapi.responses import RedirectResponse from ga4gh.core.identifiers import GA4GH_IR_REGEXP +from mavedb.lib.deprecation import MAPPED_VARIANT_SUNSET, deprecation_headers, record_deprecated_usage from mavedb.lib.logging import LoggedRoute from mavedb.routers.shared import ROUTER_BASE_PREFIX @@ -21,8 +22,20 @@ def _redirect(request: Request, target: str) -> RedirectResponse: + """Redirect to the successor, tagging the response with RFC 8594 deprecation headers. + + The redirect status already moves a well-behaved client, but the headers announce the deprecation to + tooling that follows the 301 silently: ``Deprecation``, a ``Link`` naming this exact successor, and a + ``Warning`` a human sees in logs. The successor here is a genuine 1:1 replacement, so it is safe to name. + """ url = f"{target}?{request.url.query}" if request.url.query else target - return RedirectResponse(url=url, status_code=301) + record_deprecated_usage(request.url.path, successor=target) + headers = deprecation_headers( + successor=target, + warning=f"This resource has moved permanently to {target}; update your integration.", + sunset=MAPPED_VARIANT_SUNSET, + ) + return RedirectResponse(url=url, status_code=301, headers=headers) @router.get( diff --git a/src/mavedb/routers/score_sets.py b/src/mavedb/routers/score_sets.py index 94910cb57..752c51ceb 100644 --- a/src/mavedb/routers/score_sets.py +++ b/src/mavedb/routers/score_sets.py @@ -29,6 +29,7 @@ variant_study_result, ) from mavedb.lib.annotation.context import variant_annotation_context +from mavedb.lib.deprecation import MAPPED_VARIANT_SUNSET, deprecation_headers, record_deprecated_usage from mavedb.lib.annotation.exceptions import EXPECTED_ABSENCE_EXCEPTIONS from mavedb.lib.authorization import ( get_current_user, @@ -1116,11 +1117,17 @@ def get_score_set_mapped_variants_removed(*, urn: str) -> Any: record), so the two are not wire-compatible and this route does not redirect. Use ``GET /score-sets/{urn}/variant-details`` instead. """ + successor = f"/score-sets/{urn}/variant-details" + record_deprecated_usage(f"/score-sets/{urn}/mapped-variants", successor=successor) + # No wire-compatible successor (the response shape and cardinality changed), so no rel="successor-version" + # Link — the Warning points at the nearest replacement instead. raise HTTPException( status_code=410, - detail=( - f"GET /score-sets/{urn}/mapped-variants has been removed. " - f"Use GET /score-sets/{urn}/variant-details instead." + detail=(f"GET /score-sets/{urn}/mapped-variants has been removed. Use GET {successor} instead."), + headers=deprecation_headers( + successor=None, + warning=f"Removed; use GET {successor} (NDJSON, different field shape).", + sunset=MAPPED_VARIANT_SUNSET, ), ) diff --git a/tests/lib/csv/test_deprecated_params.py b/tests/lib/csv/test_deprecated_params.py index fa63fb30e..6f47c2e57 100644 --- a/tests/lib/csv/test_deprecated_params.py +++ b/tests/lib/csv/test_deprecated_params.py @@ -132,6 +132,7 @@ def test_a_single_deprecation_is_announced(self): assert resolved.response_headers == { "Deprecation": "true", + "Link": '; rel="deprecation"', "Warning": '299 - "drop_na_columns is deprecated, use drop_unused_hgvs_columns"', } diff --git a/tests/lib/test_deprecation.py b/tests/lib/test_deprecation.py new file mode 100644 index 000000000..027e26404 --- /dev/null +++ b/tests/lib/test_deprecation.py @@ -0,0 +1,44 @@ +# ruff: noqa: E402 +"""Tests for the shared RFC 8594 deprecation-header builder.""" + +import pytest + +from mavedb.lib import deprecation +from mavedb.lib.deprecation import deprecation_headers + + +@pytest.mark.unit +def test_minimal_headers_are_deprecation_and_docs_link(): + headers = deprecation_headers() + assert headers["Deprecation"] == "true" + assert headers["Link"] == f'<{deprecation.DEPRECATION_DOCS_URL}>; rel="deprecation"' + assert "Sunset" not in headers # unset by default: no ratified removal date + assert "Warning" not in headers + + +@pytest.mark.unit +def test_successor_is_emitted_as_a_link_alongside_the_docs_link(): + headers = deprecation_headers(successor="/api/v1/variants/x", warning="moved") + link = headers["Link"] + assert '; rel="successor-version"' in link + assert 'rel="deprecation"' in link + assert headers["Warning"] == '299 - "moved"' + + +@pytest.mark.unit +def test_sunset_is_emitted_when_supplied(): + headers = deprecation_headers(sunset="Wed, 01 Jul 2026 00:00:00 GMT") + assert headers["Sunset"] == "Wed, 01 Jul 2026 00:00:00 GMT" + + +@pytest.mark.unit +def test_docs_link_can_be_omitted(): + headers = deprecation_headers(successor="/x", docs_url=None) + assert headers["Link"] == '; rel="successor-version"' + + +@pytest.mark.unit +def test_no_links_at_all_omits_the_link_header(): + headers = deprecation_headers(docs_url=None) + assert "Link" not in headers + assert headers == {"Deprecation": "true"} diff --git a/tests/routers/test_mapped_variant.py b/tests/routers/test_mapped_variant.py index e118ba192..17438b5f4 100644 --- a/tests/routers/test_mapped_variant.py +++ b/tests/routers/test_mapped_variant.py @@ -25,6 +25,18 @@ def test_redirect_mapped_variant(client): assert response.headers["location"] == f"/api/v1/variants/{quote_plus(TEST_URN)}" +def test_redirect_carries_rfc8594_deprecation_headers(client): + """A client that follows the 301 silently still learns the resource is deprecated, and where it went.""" + response = client.get(f"/api/v1/mapped-variants/{quote_plus(TEST_URN)}", follow_redirects=False) + + assert response.headers["Deprecation"] == "true" + assert 'rel="successor-version"' in response.headers["Link"] + assert f"/api/v1/variants/{quote_plus(TEST_URN)}" in response.headers["Link"] + assert response.headers["Warning"].startswith("299 - ") + # Sunset stays unset until a removal date is ratified (deprecation.DEFAULT_SUNSET is None). + assert "Sunset" not in response.headers + + @pytest.mark.parametrize( "suffix", ["va/study-result", "va/functional-statement", "va/pathogenicity-statement"], From 33e381add25895c653abd87f675a420784993130 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 2 Sep 2026 11:25:24 -0700 Subject: [PATCH 130/252] feat(export): make the legacy mapped-variants dump artifact switchable for its deprecation window mapped/{urn}.mapped-variants.json is the last dump artifact sourced from the frozen MappedVariant table; vrs/{urn}.vrs.ndjson supersedes it with a re-shaped (correct) representation. Gate its emission on EXPORT_LEGACY_MAPPED_VARIANTS (default on) so it drops cleanly at the chosen version boundary -- one operational change, not a code edit -- and removes with the table drop. Record the deprecation and the re-shape rationale in the dump changelog. --- src/mavedb/scripts/export_public_data.py | 12 +++++++++++- src/mavedb/scripts/resources/CHANGELOG.md | 9 +++++++-- tests/scripts/test_export_public_data.py | 15 +++++++++++++++ 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/src/mavedb/scripts/export_public_data.py b/src/mavedb/scripts/export_public_data.py index aa68c5eb8..c4c868819 100644 --- a/src/mavedb/scripts/export_public_data.py +++ b/src/mavedb/scripts/export_public_data.py @@ -66,6 +66,16 @@ """The only license whose data the dump may carry.""" +def _emit_legacy_mapped_variants_artifact() -> bool: + """Whether to emit the deprecated ``mapped/{urn}.mapped-variants.json`` artifact. + + Superseded by ``vrs/{urn}.vrs.ndjson`` and removed with the MappedVariant table drop; kept for a + deprecation window until then. Set ``EXPORT_LEGACY_MAPPED_VARIANTS=0`` (or ``false``/``no``/``off``) to + stop emitting it before the boundary. Defaults to on. + """ + return os.getenv("EXPORT_LEGACY_MAPPED_VARIANTS", "1").strip().lower() not in {"0", "false", "no", "off"} + + def annotation_export_namespaces(db: Session, score_set: ScoreSet, viewer: ScoreCalibrationViewer) -> list[str]: """The namespaces the public annotations CSV should carry for this score set. @@ -267,7 +277,7 @@ def score_set_artifacts(db: Session, score_set: ScoreSet, principal: Principal) yield f"vrs/{base}.vrs.ndjson", vrs_ndjson(db, score_set) yield f"va/{base}.va.ndjson", va_ndjson(db, score_set, principal) - if score_set_has_legacy_mapped_variants(db, score_set): + if _emit_legacy_mapped_variants_artifact() and score_set_has_legacy_mapped_variants(db, score_set): yield f"mapped/{base}.mapped-variants.json", mapped_variants_json(db, score_set) counts = counts_csv(db, score_set) diff --git a/src/mavedb/scripts/resources/CHANGELOG.md b/src/mavedb/scripts/resources/CHANGELOG.md index 792b245a4..7e9739524 100644 --- a/src/mavedb/scripts/resources/CHANGELOG.md +++ b/src/mavedb/scripts/resources/CHANGELOG.md @@ -26,8 +26,13 @@ given archive was generated. - **`mapped/{urn}.mapped-variants.json`** — superseded by `vrs/{urn}.vrs.ndjson`. It is sourced from the pre-allele-graph mapping store, which no longer receives - writes, so it now appears only for score sets mapped before that migration. It will - be removed in a future version. + writes, so it now appears only for score sets mapped before that migration. It is + kept for one deprecation window so existing consumers are not broken by a single + release, but will be removed in the next major release. The old single-mapping + shape encodes one representation per variant, a cardinality the allele graph does + not have (one variant maps to many alleles across coordinate levels), so + `vrs/{urn}.vrs.ndjson` is a re-shape, not a drop-in rename: migrate to it rather + than expecting the old file to keep the old shape. ### Changed diff --git a/tests/scripts/test_export_public_data.py b/tests/scripts/test_export_public_data.py index 4c8919e15..3af75f583 100644 --- a/tests/scripts/test_export_public_data.py +++ b/tests/scripts/test_export_public_data.py @@ -231,6 +231,21 @@ def test_a_superseded_score_set_contributes_scores_alone(self, session, make_dum assert set(artifacts) == {f"csv/{archive_path_base(score_set.urn)}.scores.csv"} + def test_legacy_mapped_variants_artifact_switches_off( + self, session, make_dump_score_set, anonymous_principal, monkeypatch + ): + """The deprecated legacy artifact drops cleanly at a version boundary via one env flag, without + touching the substrate-sourced artifacts that replace it.""" + score_set = make_dump_score_set(mapped=True, current=True) + base = archive_path_base(score_set.urn) + + monkeypatch.setenv("EXPORT_LEGACY_MAPPED_VARIANTS", "0") + artifacts = _artifacts(session, score_set, anonymous_principal) + + assert f"mapped/{base}.mapped-variants.json" not in artifacts + # The substrate replacement still ships. + assert f"vrs/{base}.vrs.ndjson" in artifacts + def test_count_columns_add_the_counts_file(self, session, make_dump_score_set, anonymous_principal): score_set = make_dump_score_set(mapped=False, count_columns=("c_0",)) base = archive_path_base(score_set.urn) From df3b0985bc5d97d4bdd13692220cd1e27d2f6381 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 2 Sep 2026 11:25:24 -0700 Subject: [PATCH 131/252] feat(scripts): add backfill campaign coverage map and safe-to-drop reconcile gate backfill_campaign.py adds two read-only commands for the MappedVariant -> allele-substrate backfill. 'coverage' reports each published score set's furthest lifecycle state (none -> partial_reshape -> reshaped -> rt -> enriched) with a roll-up. 'reconcile' is the gate that blocks dropping the frozen mapped_variants tables: it asserts measured-level coverage parity, zero per-variant gnomAD/ClinVar/VEP annotation regression, and no query reader of the legacy tables outside the migration/export allowlist, exiting non-zero unless the drop is safe. --- src/mavedb/scripts/backfill_campaign.py | 799 ++++++++++++++++++++++++ tests/scripts/test_backfill_campaign.py | 359 +++++++++++ 2 files changed, 1158 insertions(+) create mode 100644 src/mavedb/scripts/backfill_campaign.py create mode 100644 tests/scripts/test_backfill_campaign.py diff --git a/src/mavedb/scripts/backfill_campaign.py b/src/mavedb/scripts/backfill_campaign.py new file mode 100644 index 000000000..61f8ba8be --- /dev/null +++ b/src/mavedb/scripts/backfill_campaign.py @@ -0,0 +1,799 @@ +"""Read-only campaign map and safe-to-drop gate for the MappedVariant → allele-substrate backfill. + +The backfill has two halves (see ``alembic/manual_migrations/migrate_mapped_variants_to_allele_substrate.py`` +for the reshape and ``run_score_set_pipelines.py`` for the enrichment): a deterministic *reshape* that +reconstructs the measured-level spine and source-annotation links from the frozen ``mapped_variants`` rows, +and a decoupled *enrichment* that runs reverse translation (the cross-level fan-out) and re-runs the source +annotations against the substrate. This script does not move data; it reports where every published score set +sits between those two states, and whether the frozen ``mapped_variants`` tables are safe to drop. + +Commands +-------- +coverage + The campaign's map of what's left. For each score set: reshaped? RT fan-out present? gnomAD / ClinVar / + ClinGen / VEP enrichment present (and, when versions are supplied, at the current version)? Roll-up + counts show how many score sets are in each state. + +reconcile + The gate that should block dropping the ``mapped_variants`` tables. Reports measured-level coverage + parity between the frozen tables and the substrate, the expected divergence the substrate adds (RT + fan-out, advanced annotation versions), and a static scan confirming no code outside the known + migration/export allowlist still queries the legacy tables. Exits non-zero unless the drop is safe, so it + can gate a release step in CI or a runbook. + +Usage: + poetry run python -m mavedb.scripts.backfill_campaign coverage --published-only + poetry run python -m mavedb.scripts.backfill_campaign coverage --published-only --incomplete-only --json + poetry run python -m mavedb.scripts.backfill_campaign reconcile --published-only + poetry run python -m mavedb.scripts.backfill_campaign reconcile --published-only \\ + --vep-source-version 116 --vep-resolver-version 3 +""" + +import dataclasses +import json +import logging +import pathlib +import re +from typing import Optional, Sequence + +import asyncclick as click +from sqlalchemy import and_, distinct, func, select +from sqlalchemy.orm import Session + +from mavedb.db.session import SessionLocal +from mavedb.models.allele import Allele +from mavedb.models.clinvar_allele_link import ClinvarAlleleLink +from mavedb.models.gnomad_allele_link import GnomadAlleleLink +from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.score_set import ScoreSet +from mavedb.models.variant import Variant +from mavedb.models.vep_allele_consequence import VepAlleleConsequence +from mavedb.scripts.environment import script_environment + +logger = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# Coverage model +# --------------------------------------------------------------------------- + + +# The lifecycle states a score set moves through during the campaign, in order. classify_state returns the +# furthest state a score set has reached; the roll-up counts how many sit at each. +STATE_NONE = "none" # no live mapping_record: reshape has not run +STATE_PARTIAL_RESHAPE = "partial_reshape" # some but not all legacy-current variants have a live record +STATE_RESHAPED = "reshaped" # measured-level spine complete, no RT fan-out yet +STATE_RT = "rt" # reverse-translation fan-out present, but enrichment incomplete +STATE_ENRICHED = "enriched" # RT + gnomAD + ClinVar + ClinGen + VEP all present (at current version if pinned) + +STATE_ORDER = [STATE_NONE, STATE_PARTIAL_RESHAPE, STATE_RESHAPED, STATE_RT, STATE_ENRICHED] + + +@dataclasses.dataclass +class ScoreSetCoverage: + """Substrate state of one score set, as counts over its variants and their live alleles.""" + + score_set_id: int + urn: Optional[str] + n_variants: int = 0 + n_legacy_variants: int = 0 # variants with a live (current) legacy MappedVariant — the reshape denominator + n_reshaped_variants: int = 0 # variants with a live MappingRecord + n_live_alleles: int = 0 + n_nonauthoritative_alleles: int = 0 # RT fan-out breadth; reshape mints only authoritative alleles + has_projection_group: bool = False + n_clingen_alleles: int = 0 + n_gnomad_alleles: int = 0 + n_clinvar_alleles: int = 0 + n_vep_alleles: int = 0 + n_vep_current_alleles: int = 0 # only meaningful when VEP versions were pinned on the command line + + @property + def is_reshaped(self) -> bool: + """Every legacy-served variant has a live MappingRecord (the measured level is complete).""" + return self.n_legacy_variants > 0 and self.n_reshaped_variants >= self.n_legacy_variants + + @property + def is_partially_reshaped(self) -> bool: + return 0 < self.n_reshaped_variants < self.n_legacy_variants + + @property + def has_rt(self) -> bool: + """Reverse translation ran: it is the only producer of non-authoritative sibling alleles.""" + return self.n_nonauthoritative_alleles > 0 + + @property + def has_gnomad(self) -> bool: + return self.n_gnomad_alleles > 0 + + @property + def has_clinvar(self) -> bool: + return self.n_clinvar_alleles > 0 + + @property + def has_clingen(self) -> bool: + return self.n_clingen_alleles > 0 + + def has_vep(self, *, require_current: bool) -> bool: + return (self.n_vep_current_alleles if require_current else self.n_vep_alleles) > 0 + + +def classify_state(cov: ScoreSetCoverage, *, require_current_vep: bool) -> str: + """The furthest lifecycle state a score set has reached. + + ``enriched`` is defined by having all four source annotations present — NOT by the RT fan-out breadth. + Reverse translation legitimately produces zero sibling alleles for a single-level score set (nothing to + project across coordinate frames), so gating ``enriched`` on non-authoritative alleles would strand such a + score set at ``reshaped`` forever even after it is fully annotated. VEP depends on reverse translation + (SUCCESS_REQUIRED), so a live VEP consequence already implies RT ran; the fan-out count is reported as its + own column rather than used as a state gate. + + A score set legitimately carrying no ClinVar or gnomAD overlap would never reach ``enriched`` by the + presence rule; the campaign treats presence as the signal and relies on the reconcile parity report, not + this roll-up, to catch a score set that *should* have had a link and doesn't. + """ + if cov.n_reshaped_variants == 0: + return STATE_NONE + if cov.is_partially_reshaped: + return STATE_PARTIAL_RESHAPE + fully_enriched = ( + cov.has_gnomad and cov.has_clinvar and cov.has_clingen and cov.has_vep(require_current=require_current_vep) + ) + if fully_enriched: + return STATE_ENRICHED + # RT has run (fan-out siblings exist, or a VEP consequence — which depends on RT — is present) but at least + # one source annotation is still missing. + if cov.has_rt or cov.has_vep(require_current=False): + return STATE_RT + return STATE_RESHAPED + + +# --------------------------------------------------------------------------- +# Coverage queries +# --------------------------------------------------------------------------- + + +def resolve_cohort_ids( + db: Session, + *, + published_only: bool, + include_private: bool, + collection_urn: Optional[str], + explicit_urns: Optional[Sequence[str]], +) -> list[int]: + """Score-set ids in scope. Filters AND together; ``--published-only`` covers accession-based score sets + identically to sequence-based ones (it discriminates only on ``published_date``), so the historical + taxonomy-join pitfall that silently dropped accession-based score sets cannot recur here.""" + query = select(ScoreSet.id).where(ScoreSet.urn.is_not(None)) + if published_only: + query = query.where(ScoreSet.published_date.is_not(None)) + if not include_private and not published_only: + query = query.where(ScoreSet.private.is_(False)) + if explicit_urns: + query = query.where(ScoreSet.urn.in_(list(explicit_urns))) + if collection_urn: + from mavedb.models.collection import Collection + from mavedb.models.collection_score_set_association import CollectionScoreSetAssociation + + query = ( + query.join(CollectionScoreSetAssociation, CollectionScoreSetAssociation.score_set_id == ScoreSet.id) + .join(Collection, Collection.id == CollectionScoreSetAssociation.collection_id) + .where(Collection.urn == collection_urn) + ) + return list(db.scalars(query).all()) + + +def compute_coverage( + db: Session, + score_set_ids: Sequence[int], + *, + vep_source_version: Optional[str] = None, + vep_resolver_version: Optional[str] = None, +) -> dict[int, ScoreSetCoverage]: + """Substrate coverage for each score set id, as a handful of grouped aggregate passes joined in Python. + + Every count is over *live* substrate rows (``valid_to IS NULL``); a superseded record or link does not + count as coverage. Allele-level counts run over the live authoritative-and-RT alleles reachable from each + score set's live mapping records. + """ + ids = list(score_set_ids) + coverage = {sid: ScoreSetCoverage(score_set_id=sid, urn=None) for sid in ids} + if not ids: + return coverage + + for sid, urn in db.execute(select(ScoreSet.id, ScoreSet.urn).where(ScoreSet.id.in_(ids))): + coverage[sid].urn = urn + + for sid, n in db.execute( + select(Variant.score_set_id, func.count(Variant.id)) + .where(Variant.score_set_id.in_(ids)) + .group_by(Variant.score_set_id) + ): + coverage[sid].n_variants = n + + for sid, n in db.execute( + select(Variant.score_set_id, func.count(distinct(MappedVariant.variant_id))) + .join(Variant, Variant.id == MappedVariant.variant_id) + .where(Variant.score_set_id.in_(ids), MappedVariant.current.is_(True)) + .group_by(Variant.score_set_id) + ): + coverage[sid].n_legacy_variants = n + + for sid, n in db.execute( + select(Variant.score_set_id, func.count(distinct(MappingRecord.variant_id))) + .join(Variant, Variant.id == MappingRecord.variant_id) + .where(Variant.score_set_id.in_(ids), MappingRecord.valid_to.is_(None)) + .group_by(Variant.score_set_id) + ): + coverage[sid].n_reshaped_variants = n + + # Allele-level aggregates over live records -> live record-allele links -> alleles. + live_allele_join = ( + select( + Variant.score_set_id.label("sid"), + func.count(distinct(MappingRecordAllele.allele_id)).label("n_alleles"), + func.count(distinct(MappingRecordAllele.allele_id)) + .filter(MappingRecordAllele.is_authoritative.is_(False)) + .label("n_nonauth"), + func.bool_or(MappingRecordAllele.projection_group.is_not(None)).label("has_pg"), + func.count(distinct(MappingRecordAllele.allele_id)) + .filter(Allele.clingen_allele_id.is_not(None)) + .label("n_clingen"), + ) + .select_from(Variant) + .join(MappingRecord, and_(MappingRecord.variant_id == Variant.id, MappingRecord.valid_to.is_(None))) + .join( + MappingRecordAllele, + and_( + MappingRecordAllele.mapping_record_id == MappingRecord.id, + MappingRecordAllele.valid_to.is_(None), + ), + ) + .join(Allele, Allele.id == MappingRecordAllele.allele_id) + .where(Variant.score_set_id.in_(ids)) + .group_by(Variant.score_set_id) + ) + for sid, n_alleles, n_nonauth, has_pg, n_clingen in db.execute(live_allele_join): + cov = coverage[sid] + cov.n_live_alleles = n_alleles + cov.n_nonauthoritative_alleles = n_nonauth + cov.has_projection_group = bool(has_pg) + cov.n_clingen_alleles = n_clingen + + _fill_link_counts(db, ids, coverage, GnomadAlleleLink, "n_gnomad_alleles") + _fill_link_counts(db, ids, coverage, ClinvarAlleleLink, "n_clinvar_alleles") + _fill_vep_counts(db, ids, coverage, vep_source_version, vep_resolver_version) + + return coverage + + +def _live_allele_ids_for_scoreset(): + """A correlated live-allele subquery keyed by score set, shared by the per-source link counters.""" + return ( + select(Variant.score_set_id.label("sid"), MappingRecordAllele.allele_id.label("allele_id")) + .select_from(Variant) + .join(MappingRecord, and_(MappingRecord.variant_id == Variant.id, MappingRecord.valid_to.is_(None))) + .join( + MappingRecordAllele, + and_( + MappingRecordAllele.mapping_record_id == MappingRecord.id, + MappingRecordAllele.valid_to.is_(None), + ), + ) + ) + + +def _fill_link_counts(db: Session, ids: Sequence[int], coverage: dict, link_model, attr: str) -> None: + live_alleles = _live_allele_ids_for_scoreset().where(Variant.score_set_id.in_(ids)).subquery() + query = ( + select(live_alleles.c.sid, func.count(distinct(live_alleles.c.allele_id))) + .join(link_model, and_(link_model.allele_id == live_alleles.c.allele_id, link_model.valid_to.is_(None))) + .group_by(live_alleles.c.sid) + ) + for sid, n in db.execute(query): + setattr(coverage[sid], attr, n) + + +def _fill_vep_counts( + db: Session, + ids: Sequence[int], + coverage: dict, + vep_source_version: Optional[str], + vep_resolver_version: Optional[str], +) -> None: + live_alleles = _live_allele_ids_for_scoreset().where(Variant.score_set_id.in_(ids)).subquery() + base = ( + select(live_alleles.c.sid, func.count(distinct(live_alleles.c.allele_id))) + .join( + VepAlleleConsequence, + and_(VepAlleleConsequence.allele_id == live_alleles.c.allele_id, VepAlleleConsequence.valid_to.is_(None)), + ) + .group_by(live_alleles.c.sid) + ) + for sid, n in db.execute(base): + coverage[sid].n_vep_alleles = n + + if vep_source_version is None: + return + # "Current" mirrors the VEP job's own skip predicate: both the Ensembl release and the resolver version + # must match. A pinned resolver version of "" is treated as "match the source version only". + conds = [VepAlleleConsequence.source_version == vep_source_version] + if vep_resolver_version: + conds.append(VepAlleleConsequence.resolver_version == vep_resolver_version) + current = ( + select(live_alleles.c.sid, func.count(distinct(live_alleles.c.allele_id))) + .join( + VepAlleleConsequence, + and_(VepAlleleConsequence.allele_id == live_alleles.c.allele_id, VepAlleleConsequence.valid_to.is_(None)), + ) + .where(*conds) + .group_by(live_alleles.c.sid) + ) + for sid, n in db.execute(current): + coverage[sid].n_vep_current_alleles = n + + +# --------------------------------------------------------------------------- +# coverage command +# --------------------------------------------------------------------------- + + +def _coverage_row(cov: ScoreSetCoverage, state: str) -> dict: + return { + "score_set_urn": cov.urn or "(no urn)", + "state": state, + "n_variants": cov.n_variants, + "n_legacy": cov.n_legacy_variants, + "n_reshaped": cov.n_reshaped_variants, + "reshaped": cov.is_reshaped, + "rt": cov.has_rt, + "n_rt_alleles": cov.n_nonauthoritative_alleles, + "gnomad": cov.has_gnomad, + "clinvar": cov.has_clinvar, + "clingen": cov.has_clingen, + "vep": cov.has_vep(require_current=False), + "vep_current": cov.n_vep_current_alleles, + } + + +@script_environment.command(name="coverage") +@click.option("--published-only", is_flag=True, help="Only published score sets (covers accession-based sets).") +@click.option("--include-private", is_flag=True, help="Include private score sets (ignored with --published-only).") +@click.option("--collection-urn", default=None, help="Only score sets in this collection.") +@click.option("--score-set-urn", "score_set_urns", multiple=True, help="Restrict to these URNs (repeatable).") +@click.option("--vep-source-version", default=None, help="Ensembl release counted as current for VEP (e.g. 116).") +@click.option("--vep-resolver-version", default=None, help="Resolver version counted as current for VEP.") +@click.option("--incomplete-only", is_flag=True, help="Only score sets not yet fully enriched.") +@click.option("--limit", type=int, default=None, help="Cap rows (after filtering); the roll-up still counts all.") +@click.option("--json", "as_json", is_flag=True, help="Emit rows as JSON.") +async def coverage( + published_only: bool, + include_private: bool, + collection_urn: Optional[str], + score_set_urns: tuple[str, ...], + vep_source_version: Optional[str], + vep_resolver_version: Optional[str], + incomplete_only: bool, + limit: Optional[int], + as_json: bool, +) -> None: + """The campaign's map of what's left: substrate state of every in-scope score set.""" + require_current_vep = vep_source_version is not None + with SessionLocal() as db: + ids = resolve_cohort_ids( + db, + published_only=published_only, + include_private=include_private, + collection_urn=collection_urn, + explicit_urns=score_set_urns or None, + ) + coverage_by_id = compute_coverage( + db, ids, vep_source_version=vep_source_version, vep_resolver_version=vep_resolver_version + ) + + covs = sorted(coverage_by_id.values(), key=lambda c: c.urn or "") + states = {cov.score_set_id: classify_state(cov, require_current_vep=require_current_vep) for cov in covs} + + rollup = {state: 0 for state in STATE_ORDER} + for state in states.values(): + rollup[state] += 1 + + rows = [_coverage_row(cov, states[cov.score_set_id]) for cov in covs] + if incomplete_only: + rows = [r for r in rows if r["state"] != STATE_ENRICHED] + + if as_json: + click.echo( + json.dumps({"rollup": rollup, "total": len(covs), "score_sets": rows[:limit]}, indent=2, default=str) + ) + return + + click.echo(f"Cohort: {len(covs)} score set(s)") + click.echo("Campaign state roll-up (furthest state reached):") + for state in STATE_ORDER: + click.echo(f" {state:<16} {rollup[state]:>6}") + if require_current_vep: + click.echo(f" (VEP 'current' pinned to source={vep_source_version} resolver={vep_resolver_version or 'any'})") + click.echo() + + shown = rows[:limit] + if not shown: + click.echo("No score sets to show.") + return + + header = ( + f"{'URN':<26} {'STATE':<16} {'VARS':>7} {'LEGACY':>7} {'RESHAP':>7} " + f"{'RT':>3} {'RTALL':>6} {'GNO':>3} {'CLV':>3} {'CLG':>3} {'VEP':>3}" + ) + click.echo(header) + click.echo("-" * len(header)) + for r in shown: + click.echo( + f"{r['score_set_urn']:<26} {r['state']:<16} {r['n_variants']:>7} {r['n_legacy']:>7} " + f"{r['n_reshaped']:>7} {_yn(r['rt']):>3} {r['n_rt_alleles']:>6} {_yn(r['gnomad']):>3} " + f"{_yn(r['clinvar']):>3} {_yn(r['clingen']):>3} {_yn(r['vep']):>3}" + ) + if limit is not None and len(rows) > limit: + click.echo(f"... {len(rows) - limit} more row(s) not shown (--limit {limit}).") + + +def _yn(value: bool) -> str: + return "yes" if value else "-" + + +# --------------------------------------------------------------------------- +# reconcile command — the safe-to-drop gate +# --------------------------------------------------------------------------- + + +# Files allowed to query the legacy MappedVariant tables. The reshape migration reads them by design; the +# export script's legacy artifact and this campaign tool query them intentionally, and are retired/removed with +# the drop. Any other file issuing a query against the frozen tables would be stranded by the drop, so the +# reader scan fails on a query idiom found outside this set. +# +# Deliberately NOT in scope: ORM relationship declarations, imports, type aliases, and docstrings that merely +# name MappedVariant. Those are definitions, not readers — dropping the tables breaks them at mapper-config +# time, but they are removed as part of the drop changeset itself (the runbook enumerates them), not cleared +# ahead of it. The scan matches executable query idioms so those declarations do not produce noise. +LEGACY_READER_ALLOWLIST = { + "src/mavedb/scripts/export_public_data.py", + "src/mavedb/scripts/mapped_gene_from_mapped_variant.py", + "src/mavedb/scripts/backfill_campaign.py", +} + +# A reader is a SQLAlchemy query construct against the model or its association tables — a select/query +# targeting it, a join onto it, or an eager-loader pulling it. This intentionally does not match +# ``relationship(... "MappedVariant")``, ``secondary=..._association_table``, ``delete(MappedVariant)`` (a +# writer, removed with the drop), imports, ``Union[..., MappedVariant, ...]``, or prose. +_LEGACY_READER_PATTERNS = [ + re.compile(r"\bselect\(\s*MappedVariant\b"), + re.compile(r"\.query\(\s*MappedVariant\b"), + re.compile(r"\bjoin\(\s*MappedVariant\b"), + re.compile(r"\b(?:joinedload|selectinload|contains_eager|subqueryload)\([^)]*MappedVariant\b"), + # The association tables are only ever referenced as ``secondary=..._association_table`` in a relationship + # (a declaration, ignored) or in a real query, where a call paren sits immediately before the raw table + # name: ``select_from(gnomad_variants_mapped_variants)``, ``join(mapped_variants_clinical_controls, ...)``. + # The trailing ``\b`` before ``_association_table`` fails to match, so the ``secondary=`` symbol is skipped. + re.compile(r"\((?:gnomad_variants_mapped_variants|mapped_variants_clinical_controls)\b"), +] + + +@dataclasses.dataclass +class AnnotationParity: + """Per-variant annotation coverage for one source, legacy vs substrate. + + ``regressed`` is the load-bearing number: variants the frozen tables annotated for this source that have + NO live substrate annotation of it on any of their live alleles. Dropping the frozen tables while this is + non-zero silently regresses that source's serving. It is expected to be non-zero after reshape-only for a + protein-authoritative score set — the reshape force-closes gnomAD/ClinVar links on a protein allele, and + only the RT fan-out's genomic/coding sibling plus re-annotation makes them live again — which is exactly + why enrichment must complete before the drop. + """ + + source: str + legacy_variants: int + substrate_variants: int + regressed_variants: int + + +@dataclasses.dataclass +class ReconcileReport: + cohort_size: int + legacy_current_variants: int + reshaped_variants: int + missing_reshape_variants: int # legacy-current variants with no live MappingRecord — the coverage gate + rt_fanout_alleles: int # expected divergence: substrate breadth the legacy tables never had + annotation_parity: list[AnnotationParity] + unexpected_legacy_readers: list[str] + + @property + def measured_parity_ok(self) -> bool: + return self.missing_reshape_variants == 0 + + @property + def annotation_parity_ok(self) -> bool: + """No source lost live per-variant coverage the frozen tables had.""" + return all(parity.regressed_variants == 0 for parity in self.annotation_parity) + + @property + def readers_clear(self) -> bool: + return not self.unexpected_legacy_readers + + @property + def safe_to_drop(self) -> bool: + return self.measured_parity_ok and self.annotation_parity_ok and self.readers_clear + + +def scan_legacy_readers(repo_root: pathlib.Path) -> list[str]: + """Repo-relative ``path:line`` sites, outside the allowlist, that *query* the frozen MappedVariant tables. + + This is the static half of the drop gate: the reconcile DB parity proves the substrate *covers* the data, + and this proves nothing outside the migration/export allowlist still *reads* the frozen tables. A new query + reader added after the campaign lands trips this even if the data is fully backfilled. + """ + src = repo_root / "src" / "mavedb" + hits: list[str] = [] + for path in sorted(src.rglob("*.py")): + rel = path.relative_to(repo_root).as_posix() + if rel in LEGACY_READER_ALLOWLIST: + continue + try: + text = path.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError): + continue + for lineno, line in enumerate(text.splitlines(), start=1): + if line.lstrip().startswith("#"): + continue + if any(pattern.search(line) for pattern in _LEGACY_READER_PATTERNS): + hits.append(f"{rel}:{lineno}") + return hits + + +def compute_reconcile(db: Session, score_set_ids: Sequence[int], *, repo_root: pathlib.Path) -> ReconcileReport: + ids = list(score_set_ids) + id_filter = Variant.score_set_id.in_(ids) if ids else None + + def scoped(query): + return query.where(id_filter) if id_filter is not None else query + + legacy_current_variants = db.scalar( + scoped( + select(func.count(distinct(MappedVariant.variant_id))) + .select_from(MappedVariant) + .join(Variant, Variant.id == MappedVariant.variant_id) + .where(MappedVariant.current.is_(True)) + ) + ) + + reshaped_variants = db.scalar( + scoped( + select(func.count(distinct(MappingRecord.variant_id))) + .select_from(MappingRecord) + .join(Variant, Variant.id == MappingRecord.variant_id) + .where(MappingRecord.valid_to.is_(None)) + ) + ) + + # The gate: legacy-current variants with no live MappingRecord. Dropping the frozen tables while this is + # non-zero strands those variants — they would have no substrate row to serve from. + live_record_exists = ( + select(MappingRecord.id) + .where(MappingRecord.variant_id == MappedVariant.variant_id, MappingRecord.valid_to.is_(None)) + .exists() + ) + missing_reshape_variants = db.scalar( + scoped( + select(func.count(distinct(MappedVariant.variant_id))) + .select_from(MappedVariant) + .join(Variant, Variant.id == MappedVariant.variant_id) + .where(MappedVariant.current.is_(True), ~live_record_exists) + ) + ) + + rt_fanout_alleles = db.scalar( + scoped( + select(func.count(distinct(MappingRecordAllele.allele_id))) + .select_from(Variant) + .join(MappingRecord, and_(MappingRecord.variant_id == Variant.id, MappingRecord.valid_to.is_(None))) + .join( + MappingRecordAllele, + and_( + MappingRecordAllele.mapping_record_id == MappingRecord.id, + MappingRecordAllele.valid_to.is_(None), + MappingRecordAllele.is_authoritative.is_(False), + ), + ) + ) + ) + + annotation_parity = _annotation_parity(db, id_filter) + + return ReconcileReport( + cohort_size=len(ids), + legacy_current_variants=legacy_current_variants or 0, + reshaped_variants=reshaped_variants or 0, + missing_reshape_variants=missing_reshape_variants or 0, + rt_fanout_alleles=rt_fanout_alleles or 0, + annotation_parity=annotation_parity, + unexpected_legacy_readers=scan_legacy_readers(repo_root), + ) + + +def _substrate_link_exists(variant_id_col, link_model): + """A correlated EXISTS: the variant has a live *link_model* row on a live allele of its live record.""" + return ( + select(1) + .select_from(MappingRecord) + .join( + MappingRecordAllele, + and_(MappingRecordAllele.mapping_record_id == MappingRecord.id, MappingRecordAllele.valid_to.is_(None)), + ) + .join(link_model, and_(link_model.allele_id == MappingRecordAllele.allele_id, link_model.valid_to.is_(None))) + .where(MappingRecord.variant_id == variant_id_col, MappingRecord.valid_to.is_(None)) + .exists() + ) + + +def _annotation_parity(db: Session, id_filter) -> list["AnnotationParity"]: + """Per-variant gnomAD / ClinVar / VEP coverage, legacy vs substrate, with the regression that gates the drop. + + For each source: how many current-mapped variants the frozen tables annotate, how many the substrate + annotates live, and — the gate — how many the frozen tables annotate that the substrate does NOT annotate + live. A non-zero regression means dropping the frozen tables would lose that source's serving for those + variants, so it must reach zero (via enrichment) before the drop. + """ + from mavedb.models.clinical_control_mapped_variant import ( + mapped_variants_clinical_controls_association_table as clinvar_assoc, + ) + from mavedb.models.gnomad_variant_mapped_variant import ( + gnomad_variants_mapped_variants_association_table as gnomad_assoc, + ) + + def legacy_variant_ids(join_or_predicate): + query = ( + select(distinct(MappedVariant.variant_id).label("variant_id")) + .select_from(MappedVariant) + .join(Variant, Variant.id == MappedVariant.variant_id) + .where(MappedVariant.current.is_(True)) + ) + query = join_or_predicate(query) + if id_filter is not None: + query = query.where(id_filter) + return query.subquery() + + def substrate_variant_count(link_model): + query = ( + select(func.count(distinct(Variant.id))) + .select_from(Variant) + .join(MappingRecord, and_(MappingRecord.variant_id == Variant.id, MappingRecord.valid_to.is_(None))) + .join( + MappingRecordAllele, + and_(MappingRecordAllele.mapping_record_id == MappingRecord.id, MappingRecordAllele.valid_to.is_(None)), + ) + .join( + link_model, and_(link_model.allele_id == MappingRecordAllele.allele_id, link_model.valid_to.is_(None)) + ) + ) + if id_filter is not None: + query = query.where(id_filter) + return db.scalar(query) or 0 + + def parity(source, legacy_ids_subq, link_model) -> AnnotationParity: + legacy = db.scalar(select(func.count()).select_from(legacy_ids_subq)) or 0 + regressed = ( + db.scalar( + select(func.count()) + .select_from(legacy_ids_subq) + .where(~_substrate_link_exists(legacy_ids_subq.c.variant_id, link_model)) + ) + or 0 + ) + return AnnotationParity( + source=source, + legacy_variants=legacy, + substrate_variants=substrate_variant_count(link_model), + regressed_variants=regressed, + ) + + return [ + parity( + "gnomad", + legacy_variant_ids(lambda q: q.join(gnomad_assoc, gnomad_assoc.c.mapped_variant_id == MappedVariant.id)), + GnomadAlleleLink, + ), + parity( + "clinvar", + legacy_variant_ids(lambda q: q.join(clinvar_assoc, clinvar_assoc.c.mapped_variant_id == MappedVariant.id)), + ClinvarAlleleLink, + ), + parity( + "vep", + legacy_variant_ids(lambda q: q.where(MappedVariant.vep_functional_consequence.is_not(None))), + VepAlleleConsequence, + ), + ] + + +@script_environment.command(name="reconcile") +@click.option("--published-only", is_flag=True, help="Only published score sets.") +@click.option("--include-private", is_flag=True, help="Include private score sets (ignored with --published-only).") +@click.option("--collection-urn", default=None, help="Only score sets in this collection.") +@click.option("--score-set-urn", "score_set_urns", multiple=True, help="Restrict to these URNs (repeatable).") +@click.option("--json", "as_json", is_flag=True, help="Emit the report as JSON.") +async def reconcile( + published_only: bool, + include_private: bool, + collection_urn: Optional[str], + score_set_urns: tuple[str, ...], + as_json: bool, +) -> None: + """Safe-to-drop gate: measured-level parity + expected divergence + a static legacy-reader scan. + + Exits non-zero unless the drop is safe (every legacy-served variant is reshaped and no code outside the + migration/export allowlist still reads the frozen tables), so it can gate a runbook or CI step. + """ + repo_root = pathlib.Path(__file__).resolve().parents[3] + with SessionLocal() as db: + ids = resolve_cohort_ids( + db, + published_only=published_only, + include_private=include_private, + collection_urn=collection_urn, + explicit_urns=score_set_urns or None, + ) + report = compute_reconcile(db, ids, repo_root=repo_root) + + if as_json: + click.echo( + json.dumps(dataclasses.asdict(report) | {"safe_to_drop": report.safe_to_drop}, indent=2, default=str) + ) + else: + _print_reconcile(report) + + raise SystemExit(0 if report.safe_to_drop else 1) + + +def _print_reconcile(report: ReconcileReport) -> None: + click.echo(f"Reconcile over {report.cohort_size or 'ALL'} score set(s)") + click.echo() + click.echo("Measured-level parity (frozen mapped_variants -> substrate):") + click.echo(f" legacy current variants {report.legacy_current_variants:>10}") + click.echo(f" reshaped variants (live record) {report.reshaped_variants:>10}") + click.echo(f" MISSING reshape (gate) {report.missing_reshape_variants:>10} (must be 0 to drop)") + click.echo() + click.echo(f"RT fan-out sibling alleles the substrate adds (legacy had none): {report.rt_fanout_alleles}") + click.echo() + click.echo("Per-variant annotation parity (legacy -> substrate live; REGRESSED gates the drop):") + click.echo(f" {'source':<8} {'legacy':>8} {'substrate':>9} {'REGRESSED':>9}") + for parity in report.annotation_parity: + click.echo( + f" {parity.source:<8} {parity.legacy_variants:>8} " + f"{parity.substrate_variants:>9} {parity.regressed_variants:>9}" + ) + click.echo(" (a variant is 'regressed' if the frozen tables annotate it and the live substrate does not;") + click.echo(" reshape force-closes gnomAD/ClinVar on protein alleles, so enrichment must clear this to 0)") + click.echo() + click.echo("Legacy-table readers outside the allowlist:") + if report.unexpected_legacy_readers: + for hit in report.unexpected_legacy_readers: + click.echo(f" ! {hit}") + else: + click.echo(" none") + click.echo() + verdict = "SAFE TO DROP" if report.safe_to_drop else "NOT SAFE TO DROP" + click.echo(f"==> {verdict}") + if not report.safe_to_drop: + if not report.measured_parity_ok: + click.echo(f" - {report.missing_reshape_variants} legacy-served variant(s) are not reshaped.") + if not report.annotation_parity_ok: + regressed = ", ".join( + f"{p.source}={p.regressed_variants}" for p in report.annotation_parity if p.regressed_variants + ) + click.echo(f" - annotation coverage regressed for: {regressed} (run enrichment).") + if not report.readers_clear: + click.echo(f" - {len(report.unexpected_legacy_readers)} unexpected legacy reader(s) remain.") + + +if __name__ == "__main__": + script_environment() diff --git a/tests/scripts/test_backfill_campaign.py b/tests/scripts/test_backfill_campaign.py new file mode 100644 index 000000000..0b28d6e23 --- /dev/null +++ b/tests/scripts/test_backfill_campaign.py @@ -0,0 +1,359 @@ +# ruff: noqa: E402 +"""Tests for the backfill campaign map (coverage) and safe-to-drop gate (reconcile).""" + +import pytest + +pytest.importorskip("arq") + +from mavedb.models.clinical_control import ClinvarControl +from mavedb.models.gnomad_variant import GnomADVariant +from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.variant import Variant +from mavedb.scripts.backfill_campaign import ( + STATE_ENRICHED, + STATE_NONE, + STATE_PARTIAL_RESHAPE, + STATE_RESHAPED, + STATE_RT, + ScoreSetCoverage, + classify_state, + compute_coverage, + compute_reconcile, + resolve_cohort_ids, + scan_legacy_readers, +) +from tests.helpers.constants import TEST_MINIMAL_MAPPED_VARIANT +from tests.helpers.util.annotation import AlleleSpec, seed_mapping_record + + +# Pure: state classification + + +def _cov(**kw) -> ScoreSetCoverage: + base = dict(score_set_id=1, urn="urn:mavedb:x-1", n_variants=10, n_legacy_variants=10, n_reshaped_variants=10) + base.update(kw) + return ScoreSetCoverage(**base) + + +@pytest.mark.unit +class TestClassifyState: + def test_none_when_no_live_record(self): + assert classify_state(_cov(n_reshaped_variants=0), require_current_vep=False) == STATE_NONE + + def test_partial_reshape(self): + cov = _cov(n_legacy_variants=10, n_reshaped_variants=4) + assert classify_state(cov, require_current_vep=False) == STATE_PARTIAL_RESHAPE + + def test_reshaped_without_rt(self): + cov = _cov(n_nonauthoritative_alleles=0) + assert classify_state(cov, require_current_vep=False) == STATE_RESHAPED + + def test_rt_when_fanout_present_but_enrichment_incomplete(self): + cov = _cov(n_nonauthoritative_alleles=5, n_gnomad_alleles=1) # missing clinvar/clingen/vep + assert classify_state(cov, require_current_vep=False) == STATE_RT + + def test_enriched_requires_all_sources(self): + cov = _cov( + n_nonauthoritative_alleles=5, + n_gnomad_alleles=1, + n_clinvar_alleles=1, + n_clingen_alleles=1, + n_vep_alleles=1, + ) + assert classify_state(cov, require_current_vep=False) == STATE_ENRICHED + + def test_current_vep_gate_downgrades_when_only_stale_vep_present(self): + cov = _cov( + n_nonauthoritative_alleles=5, + n_gnomad_alleles=1, + n_clinvar_alleles=1, + n_clingen_alleles=1, + n_vep_alleles=1, # some VEP, but... + n_vep_current_alleles=0, # ...none at the pinned current version + ) + assert classify_state(cov, require_current_vep=True) == STATE_RT + assert classify_state(cov, require_current_vep=False) == STATE_ENRICHED + + def test_fully_annotated_without_fanout_is_enriched(self): + # Reverse translation can legitimately produce zero siblings (single-level score set). Such a set, + # once fully annotated, must reach 'enriched' rather than being stranded at 'reshaped'. VEP presence + # (VEP depends on RT) confirms RT ran even with no fan-out. + cov = _cov( + n_nonauthoritative_alleles=0, + n_gnomad_alleles=1, + n_clinvar_alleles=1, + n_clingen_alleles=1, + n_vep_alleles=1, + ) + assert classify_state(cov, require_current_vep=False) == STATE_ENRICHED + + def test_vep_present_without_fanout_reads_as_rt_when_incomplete(self): + # VEP live but a source still missing: RT ran (VEP implies it), so this is 'rt', not 'reshaped'. + cov = _cov(n_nonauthoritative_alleles=0, n_vep_alleles=1) # missing gnomad/clinvar/clingen + assert classify_state(cov, require_current_vep=False) == STATE_RT + + def test_is_reshaped_needs_a_legacy_denominator(self): + # A score set with no legacy rows can't be "reshaped" against a zero denominator. + assert not _cov(n_legacy_variants=0, n_reshaped_variants=5).is_reshaped + + +# Pure: static legacy-reader scan + + +def _write(root, rel, text): + path = root / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + + +@pytest.mark.unit +class TestScanLegacyReaders: + def test_flags_a_query_reader_outside_the_allowlist(self, tmp_path): + _write(tmp_path, "src/mavedb/routers/rogue.py", "rows = db.scalars(select(MappedVariant)).all()\n") + hits = scan_legacy_readers(tmp_path) + assert hits == ["src/mavedb/routers/rogue.py:1"] + + def test_flags_join_and_loader_idioms(self, tmp_path): + _write( + tmp_path, + "src/mavedb/lib/x.py", + "q = select(Variant).join(MappedVariant)\no = joinedload(Foo.bar, MappedVariant)\n", + ) + assert scan_legacy_readers(tmp_path) == ["src/mavedb/lib/x.py:1", "src/mavedb/lib/x.py:2"] + + def test_flags_association_table_query(self, tmp_path): + _write( + tmp_path, "src/mavedb/lib/y.py", "n = select(func.count()).select_from(gnomad_variants_mapped_variants)\n" + ) + assert scan_legacy_readers(tmp_path) == ["src/mavedb/lib/y.py:1"] + + def test_ignores_declarations_imports_aliases_and_prose(self, tmp_path): + _write( + tmp_path, + "src/mavedb/models/variant.py", + ( + "from mavedb.models.mapped_variant import MappedVariant\n" + 'mapped_variants: Mapped[list["MappedVariant"]] = relationship("MappedVariant")\n' + "Alias = Union[ScoreSet, MappedVariant, Variant]\n" + "# a comment naming MappedVariant\n" + '"""a docstring naming MappedVariant."""\n' + "cleanup = delete(MappedVariant)\n" # a writer, removed with the drop — not a reader + ), + ) + assert scan_legacy_readers(tmp_path) == [] + + def test_allowlisted_files_are_skipped(self, tmp_path): + _write( + tmp_path, + "src/mavedb/scripts/export_public_data.py", + "rows = db.scalars(select(MappedVariant)).all()\n", + ) + assert scan_legacy_readers(tmp_path) == [] + + +# DB-backed: coverage and reconcile + + +def _add_variants(session, score_set, n): + variants = [] + for i in range(n): + v = Variant(urn=f"{score_set.urn}#{i}", score_set_id=score_set.id, hgvs_nt=f"c.{i + 1}A>G", data={}) + session.add(v) + variants.append(v) + session.commit() + return variants + + +def _add_legacy_mapped_variant(session, variant, *, current=True): + session.add(MappedVariant(**{**TEST_MINIMAL_MAPPED_VARIANT, "current": current}, variant_id=variant.id)) + session.commit() + + +def _published(session, make_score_set, urn): + ss = make_score_set(published=True) + ss.urn = urn + session.commit() + return ss + + +@pytest.mark.integration +class TestComputeCoverage: + def test_not_reshaped_score_set_is_none(self, session, make_score_set): + ss = _published(session, make_score_set, "urn:mavedb:00000900-a-1") + for v in _add_variants(session, ss, 3): + _add_legacy_mapped_variant(session, v) + + cov = compute_coverage(session, [ss.id])[ss.id] + assert cov.n_variants == 3 + assert cov.n_legacy_variants == 3 + assert cov.n_reshaped_variants == 0 + assert classify_state(cov, require_current_vep=False) == STATE_NONE + + def test_reshaped_only_score_set(self, session, make_score_set): + ss = _published(session, make_score_set, "urn:mavedb:00000901-a-1") + for i, v in enumerate(_add_variants(session, ss, 2)): + _add_legacy_mapped_variant(session, v) + seed_mapping_record( + session, + v, + assay_level="genomic", + alleles=[AlleleSpec(digest=f"resh-{i}", level="genomic", is_authoritative=True)], + ) + + cov = compute_coverage(session, [ss.id])[ss.id] + assert cov.n_reshaped_variants == 2 + assert cov.is_reshaped + assert not cov.has_rt + assert classify_state(cov, require_current_vep=False) == STATE_RESHAPED + + def test_rt_and_enriched_score_set(self, session, make_score_set): + ss = _published(session, make_score_set, "urn:mavedb:00000902-a-1") + gnomad = GnomADVariant( + db_name="gnomAD", + db_identifier="1-1-A-G", + db_version="4", + allele_count=1, + allele_number=2, + allele_frequency=0.5, + ) + control = ClinvarControl( + db_identifier="1", + gene_symbol="X", + clinical_significance="Pathogenic", + clinical_review_status="ok", + db_name="ClinVar", + db_version="2026-01", + ) + session.add_all([gnomad, control]) + session.commit() + + v = _add_variants(session, ss, 1)[0] + _add_legacy_mapped_variant(session, v) + seed_mapping_record( + session, + v, + assay_level="genomic", + alleles=[ + AlleleSpec(digest="auth", level="genomic", is_authoritative=True, clingen_allele_id="CA1"), + # RT fan-out sibling carrying every source annotation: + AlleleSpec( + digest="sib", + level="cdna", + is_authoritative=False, + projection_group=1, + vep_consequence="missense_variant", + clinvar_control_ids=(control.id,), + gnomad_variant_ids=(gnomad.id,), + ), + ], + ) + + cov = compute_coverage(session, [ss.id])[ss.id] + assert cov.has_rt + assert cov.n_nonauthoritative_alleles == 1 + assert cov.has_gnomad and cov.has_clinvar and cov.has_clingen and cov.has_vep(require_current=False) + assert classify_state(cov, require_current_vep=False) == STATE_ENRICHED + # Pinning VEP to a version the seed did not write (seed writes "116") downgrades to rt. + cov_pinned = compute_coverage(session, [ss.id], vep_source_version="999")[ss.id] + assert cov_pinned.n_vep_current_alleles == 0 + assert classify_state(cov_pinned, require_current_vep=True) == STATE_RT + # And pinning to the version the seed wrote counts it current again. + cov_match = compute_coverage(session, [ss.id], vep_source_version="116")[ss.id] + assert cov_match.n_vep_current_alleles == 1 + + +@pytest.mark.integration +class TestComputeReconcile: + def test_not_safe_when_a_legacy_variant_is_unreshaped(self, session, make_score_set, tmp_path): + ss = _published(session, make_score_set, "urn:mavedb:00000903-a-1") + for v in _add_variants(session, ss, 2): + _add_legacy_mapped_variant(session, v) + + report = compute_reconcile(session, [ss.id], repo_root=tmp_path) + assert report.legacy_current_variants == 2 + assert report.missing_reshape_variants == 2 + assert not report.measured_parity_ok + assert not report.safe_to_drop + + def test_safe_when_every_legacy_variant_is_reshaped_and_readers_clear(self, session, make_score_set, tmp_path): + ss = _published(session, make_score_set, "urn:mavedb:00000904-a-1") + for i, v in enumerate(_add_variants(session, ss, 2)): + _add_legacy_mapped_variant(session, v) + seed_mapping_record( + session, + v, + assay_level="genomic", + alleles=[AlleleSpec(digest=f"ok-{i}", level="genomic", is_authoritative=True)], + ) + + report = compute_reconcile(session, [ss.id], repo_root=tmp_path) # empty tmp repo -> no readers + assert report.missing_reshape_variants == 0 + assert report.measured_parity_ok + assert report.annotation_parity_ok + assert report.readers_clear + assert report.safe_to_drop + + def test_annotation_regression_blocks_the_drop(self, session, make_score_set, tmp_path): + # A variant the frozen tables annotate for ClinVar, reshaped but with NO live substrate ClinVar link + # (the protein force-close case), must block the drop until enrichment restores it. + ss = _published(session, make_score_set, "urn:mavedb:00000906-a-1") + control = ClinvarControl( + db_identifier="9", + gene_symbol="X", + clinical_significance="Pathogenic", + clinical_review_status="ok", + db_name="ClinVar", + db_version="2026-01", + ) + session.add(control) + session.commit() + v = _add_variants(session, ss, 1)[0] + # Legacy MappedVariant carrying a ClinVar association... + mv = MappedVariant(**{**TEST_MINIMAL_MAPPED_VARIANT, "current": True}, variant_id=v.id) + mv.clinical_controls = [control] + session.add(mv) + session.commit() + # ...reshaped to the measured level, but with no live ClinVar link on the substrate. + seed_mapping_record( + session, v, assay_level="protein", alleles=[AlleleSpec(digest="p", level="protein", is_authoritative=True)] + ) + + report = compute_reconcile(session, [ss.id], repo_root=tmp_path) + assert report.measured_parity_ok # measured level is complete... + clinvar = next(p for p in report.annotation_parity if p.source == "clinvar") + assert clinvar.legacy_variants == 1 + assert clinvar.regressed_variants == 1 # ...but ClinVar serving would regress. + assert not report.annotation_parity_ok + assert not report.safe_to_drop + + def test_unexpected_reader_blocks_even_with_full_parity(self, session, make_score_set, tmp_path): + ss = _published(session, make_score_set, "urn:mavedb:00000905-a-1") + v = _add_variants(session, ss, 1)[0] + _add_legacy_mapped_variant(session, v) + seed_mapping_record( + session, v, assay_level="genomic", alleles=[AlleleSpec(digest="ok", level="genomic", is_authoritative=True)] + ) + _write(tmp_path, "src/mavedb/routers/rogue.py", "select(MappedVariant)\n") + + report = compute_reconcile(session, [ss.id], repo_root=tmp_path) + assert report.measured_parity_ok + assert not report.readers_clear + assert not report.safe_to_drop + + +@pytest.mark.integration +class TestResolveCohortIds: + def test_published_only_includes_accession_based(self, session, make_score_set): + # The accession-based-skip pitfall: --published-only must not discriminate on target type. + seq = make_score_set(published=True, gene_names=("BRCA1",)) + acc = make_score_set(published=True, gene_names=(), accession_gene_names=("TP53",)) + private = make_score_set(published=False, gene_names=("EGFR",)) + seq.urn, acc.urn, private.urn = "urn:mavedb:00000910-a-1", "urn:mavedb:00000911-a-1", "urn:mavedb:00000912-a-1" + session.commit() + + ids = resolve_cohort_ids( + session, published_only=True, include_private=False, collection_urn=None, explicit_urns=None + ) + assert seq.id in ids + assert acc.id in ids + assert private.id not in ids From 7e7beb9ffbff9ccbffcab3ae7b4b1267d073faa1 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 2 Sep 2026 11:33:10 -0700 Subject: [PATCH 132/252] fix(reverse-translation): anchor cdna transcript resolution to the mapping run (#763) Reverse translation resolved a variant's coding transcript from cdna TargetGeneMappings keyed by (target_gene_id, mapped_date). mapped_date is day-granular, so two mapping runs of one score set on the same calendar day were indistinguishable: a later same-day remap that emitted no cdna row would bind the earlier run's stale transcript instead of correctly skipping. Record the producing JobRun on each TargetGeneMapping (nullable FK, ON DELETE SET NULL) and resolve a record's transcript by its own run via (job_run_id, target_gene_id), so a run with no cdna row resolves to nothing and the variant is skipped as transcript-unresolved. Rows with no job_run_id (pre-column, or reshaped from legacy) fall back to the day-granular key. The resolution is contained in a single _cdna_transcript_resolver factory. Adds same-day two-run and mapped_date-fallback regression tests. --- ..._add_job_run_id_to_target_gene_mappings.py | 45 +++++++ src/mavedb/models/target_gene_mapping.py | 4 + .../worker/jobs/variant_processing/mapping.py | 1 + .../variant_processing/reverse_translation.py | 80 +++++++----- .../test_reverse_translation.py | 116 ++++++++++++++++-- 5 files changed, 208 insertions(+), 38 deletions(-) create mode 100644 alembic/versions/d1c9b7a3e2f4_add_job_run_id_to_target_gene_mappings.py diff --git a/alembic/versions/d1c9b7a3e2f4_add_job_run_id_to_target_gene_mappings.py b/alembic/versions/d1c9b7a3e2f4_add_job_run_id_to_target_gene_mappings.py new file mode 100644 index 000000000..744f3f72d --- /dev/null +++ b/alembic/versions/d1c9b7a3e2f4_add_job_run_id_to_target_gene_mappings.py @@ -0,0 +1,45 @@ +"""add job_run_id to target_gene_mappings + +Revision ID: d1c9b7a3e2f4 +Revises: a3f9c2e1b7d6 +Create Date: 2026-09-02 + +Adds a nullable ``job_run_id`` (FK ``job_runs``, ``ON DELETE SET NULL``) recording the mapping job that +produced each row. Reverse translation uses it to bind a variant's cross-level transcript to its *own* run's +cdna alignment rather than to any same-day run's (#763): the day-granular ``mapped_date`` key it previously +used cannot tell two runs on one calendar day apart, so a same-day remap that emits no cdna row would bind the +earlier run's stale transcript instead of correctly skipping. + +Nullable, no backfill: rows written before this migration keep NULL and fall back to the coarser +``(target_gene_id, mapped_date)`` match. ``SET NULL`` so a pruned job_run degrades to that fallback rather +than dangling. +""" + +import sqlalchemy as sa + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "d1c9b7a3e2f4" +down_revision = "a3f9c2e1b7d6" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column("target_gene_mappings", sa.Column("job_run_id", sa.Integer(), nullable=True)) + op.create_foreign_key( + "fk_target_gene_mappings_job_run_id", + "target_gene_mappings", + "job_runs", + ["job_run_id"], + ["id"], + ondelete="SET NULL", + ) + op.create_index("ix_target_gene_mappings_job_run_id", "target_gene_mappings", ["job_run_id"]) + + +def downgrade() -> None: + op.drop_index("ix_target_gene_mappings_job_run_id", table_name="target_gene_mappings") + op.drop_constraint("fk_target_gene_mappings_job_run_id", "target_gene_mappings", type_="foreignkey") + op.drop_column("target_gene_mappings", "job_run_id") diff --git a/src/mavedb/models/target_gene_mapping.py b/src/mavedb/models/target_gene_mapping.py index 5ead4a887..a128d9e14 100644 --- a/src/mavedb/models/target_gene_mapping.py +++ b/src/mavedb/models/target_gene_mapping.py @@ -73,6 +73,10 @@ class TargetGeneMapping(Base): # ``modification_date``, which track this row's lifecycle in MaveDB. mapped_date = Column(Date, nullable=True) + # The mapping job (JobRun) that produced this row. Nullable: rows written before this column, + # and rows whose job was pruned (ON DELETE SET NULL), carry no run anchor. + job_run_id = Column(Integer, ForeignKey("job_runs.id", ondelete="SET NULL"), nullable=True, index=True) + creation_date = Column(Date, nullable=False, default=date.today) modification_date = Column(Date, nullable=False, default=date.today, onupdate=date.today) diff --git a/src/mavedb/worker/jobs/variant_processing/mapping.py b/src/mavedb/worker/jobs/variant_processing/mapping.py index 19a76e224..063094ff5 100644 --- a/src/mavedb/worker/jobs/variant_processing/mapping.py +++ b/src/mavedb/worker/jobs/variant_processing/mapping.py @@ -253,6 +253,7 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan alignment_metadata=tm.get("alignment_metadata"), vrs_version=tm.get("vrs_version"), mapped_date=mapping_results["mapped_date"], + job_run_id=job_manager.job_id, ) job_manager.db.add(target_gene_mapping) target_gene_mapping_by_key[(target_gene_identifier, level_value)] = target_gene_mapping diff --git a/src/mavedb/worker/jobs/variant_processing/reverse_translation.py b/src/mavedb/worker/jobs/variant_processing/reverse_translation.py index ea1d311a6..dcaaef642 100644 --- a/src/mavedb/worker/jobs/variant_processing/reverse_translation.py +++ b/src/mavedb/worker/jobs/variant_processing/reverse_translation.py @@ -22,10 +22,11 @@ import logging from collections import Counter from datetime import date -from typing import Any, NamedTuple, Sequence +from typing import Any, Callable, NamedTuple, Sequence from ga4gh.vrs.extras.translator import AlleleTranslator from sqlalchemy import select +from sqlalchemy.orm import Session from variant_annotation import __version__ as variant_annotation_version from variant_annotation.lib.accessions import looks_like_refseq_protein_accession from variant_annotation.lib.translation import construct_equivalent_variants @@ -116,6 +117,45 @@ def _coding_transcripts_for_proteins(protein_accessions: set[str]) -> dict[str, } +def _cdna_transcript_resolver(db: Session, score_set_id: int) -> Callable[[int | None, int, date | None], str | None]: + """Return a resolver from a record's ``(job_run_id, target_gene_id, mapped_date)`` to its cdna transcript. + + A record binds to *its own run's* cdna row via ``job_run_id``. Records with no ``job_run_id`` (pre-column, + or reshaped from legacy) fall back to the day-granular ``(target_gene_id, mapped_date)`` key, which cannot + separate two runs on one calendar day. Within either key, highest ``TargetGeneMapping.id`` wins. The two + lookup tables are built once per score set here and closed over, so per-record resolution is a dict hit. + """ + by_job_run: dict[tuple[int, int], str | None] = {} + by_run_date: dict[tuple[int, date | None], str | None] = {} + for target_gene_id, mapped_date, job_run_id, reference_accession in ( + db.execute( + select( + TargetGeneMapping.target_gene_id, + TargetGeneMapping.mapped_date, + TargetGeneMapping.job_run_id, + TargetGeneMapping.reference_accession, + ) + .join(TargetGene, TargetGene.id == TargetGeneMapping.target_gene_id) + .where(TargetGene.score_set_id == score_set_id) + .where(TargetGeneMapping.alignment_level == SequenceLevel.cdna) + .where(TargetGeneMapping.reference_accession.isnot(None)) + .order_by(TargetGeneMapping.id) + ) + .tuples() + .all() + ): + if job_run_id is not None: + by_job_run[(job_run_id, target_gene_id)] = reference_accession + by_run_date[(target_gene_id, mapped_date)] = reference_accession + + def resolve(job_run_id: int | None, target_gene_id: int, mapped_date: date | None) -> str | None: + if job_run_id is not None: + return by_job_run.get((job_run_id, target_gene_id)) + return by_run_date.get((target_gene_id, mapped_date)) + + return resolve + + def _build_translation_config(overrides: dict[str, Any] | None) -> TranslationConfig: """Build a TranslationConfig from optional job-param overrides merged with job defaults. @@ -208,41 +248,19 @@ async def reverse_translate_variants_for_score_set( job_manager.update_progress(0, 100, "Starting reverse translation job.") logger.info(msg="Started reverse translation job.", extra=job_manager.logging_context()) - # Build {(target_gene_id, run date) -> NM_ transcript} from cdna TargetGeneMappings. - # TargetGeneMapping rows accumulate across re-maps (retired records still FK them), so we - # key by (target_gene_id, mapped_date) to bind each record to its own run's cdna row. - # All TargetGeneMappings in one job share a mapped_date; within a key, highest id wins. - # - # TODO#763: mapped_date is day-granular, so two re-maps on the same calendar day collide. - # A later same-day run that emits no cdna row can still bind the earlier run's stale - # transcript. Only a versioned run anchor closes this fully. - cdna_transcript_by_run: dict[tuple[int, date | None], str | None] = { - (target_gene_id, mapped_date): reference_accession - for target_gene_id, mapped_date, reference_accession in job_manager.db.execute( - select( - TargetGeneMapping.target_gene_id, - TargetGeneMapping.mapped_date, - TargetGeneMapping.reference_accession, - ) - .join(TargetGene, TargetGene.id == TargetGeneMapping.target_gene_id) - .where(TargetGene.score_set_id == score_set_id) - .where(TargetGeneMapping.alignment_level == SequenceLevel.cdna) - .where(TargetGeneMapping.reference_accession.isnot(None)) - .order_by(TargetGeneMapping.id) - ) - .tuples() - .all() - } + resolve_cdna_transcript = _cdna_transcript_resolver(job_manager.db, score_set_id) - # Load current authoritative MappingRecords with their Variant and TargetGeneMapping. - # mapped_date from the joined (genomic) TargetGeneMapping anchors the cdna transcript lookup. - rows: Sequence[tuple[MappingRecord, Variant, int, date | None]] = ( + # Load current authoritative MappingRecords with their Variant and TargetGeneMapping. The record's own + # TargetGeneMapping (its measured alignment) carries the run's job_run_id and mapped_date, which anchor + # the cdna transcript lookup below. + rows: Sequence[tuple[MappingRecord, Variant, int, date | None, int | None]] = ( job_manager.db.execute( select( MappingRecord, Variant, TargetGeneMapping.target_gene_id, TargetGeneMapping.mapped_date, + TargetGeneMapping.job_run_id, ) .join(MappingRecordAllele, MappingRecord.id == MappingRecordAllele.mapping_record_id) .join(Variant, MappingRecord.variant_id == Variant.id) @@ -270,8 +288,8 @@ async def reverse_translate_variants_for_score_set( # cdna reference_accession, so collect their NP_ accessions for a batched NP_→NM_ UTA lookup. transcript_resolutions: list[_TranscriptResolution] = [] protein_accessions: set[str] = set() - for rec, variant, target_gene_id, mapped_date in rows: - coding_accession = cdna_transcript_by_run.get((target_gene_id, mapped_date)) + for rec, variant, target_gene_id, mapped_date, record_job_run_id in rows: + coding_accession = resolve_cdna_transcript(record_job_run_id, target_gene_id, mapped_date) protein_accession = None if not coding_accession and rec.hgvs_assay_level is not None: raw_accession = extract_accession(rec.hgvs_assay_level) diff --git a/tests/worker/jobs/variant_processing/test_reverse_translation.py b/tests/worker/jobs/variant_processing/test_reverse_translation.py index b32f18d2d..cea59c676 100644 --- a/tests/worker/jobs/variant_processing/test_reverse_translation.py +++ b/tests/worker/jobs/variant_processing/test_reverse_translation.py @@ -298,6 +298,107 @@ async def test_no_mapping_records_is_a_noop( assert _non_authoritative_links(session) == [] assert _cross_level_events(session, sample_score_set.id) == [] + async def test_same_day_remap_without_a_cdna_row_skips_rather_than_binding_a_stale_transcript( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + map_variants_sample_params, + ): + """#763: a record binds its transcript to its own run via job_run_id. When a later same-day remap + emits no cdna row, reverse translation must resolve no transcript and skip -- not fall back to an + earlier same-day run's cdna transcript, which the day-granular key alone could not tell apart.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + + # Run 1 (all layers) writes a cdna TargetGeneMapping; its transcript must not leak to run 2. + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + # Run 2 -- same score set, same calendar day -- emits genomic + protein only, no cdna row. Its + # records supersede run 1's and are the current ones reverse translation reads. + second_mapping_run = JobRun( + urn="test:map_variants_for_score_set:2", + job_type="map_variants_for_score_set", + job_function="map_variants_for_score_set", + max_retries=3, + retry_count=0, + job_params=map_variants_sample_params, + ) + session.add(second_mapping_run) + session.commit() + await _map_variants(session, mock_worker_ctx, second_mapping_run, sample_score_set, with_layers={"g", "p"}) + + # construct_equivalent_variants is never reached for a skipped variant. + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", fake_construct({})), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", fake_translate({})), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + assert result.data["translated"] == 0 + assert result.data["skipped"] == 1 + assert result.data["alleles_created"] == 0 + assert _non_authoritative_links(session) == [] + assert len(_cross_level_events(session, sample_score_set.id, reason="transcript_unresolved")) == 1 + + async def test_falls_back_to_mapped_date_when_the_record_has_no_job_run_id( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """A record with no job_run_id (pre-column, or reshaped from legacy) resolves its cdna transcript + via the day-granular (target_gene_id, mapped_date) fallback rather than skipping.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + # Simulate the pre-job_run_id / reshaped state: no TargetGeneMapping carries a run anchor, so + # resolution has only the day-granular key to work with. + session.query(TargetGeneMapping).filter( + TargetGeneMapping.target_gene_id == sample_score_set.target_genes[0].id + ).update({TargetGeneMapping.job_run_id: None}, synchronize_session=False) + session.commit() + + assay_hgvs = "NM_000000.1:c.1A>G" + c_candidate = "NM_000001.1:c.5A>G" + g_candidate = "NC_000001.11:g.1000A>G" + construct = fake_construct({assay_hgvs: [(c_candidate, g_candidate)]}) + translate = fake_translate({g_candidate: "ga4gh:VA.genomic", c_candidate: "ga4gh:VA.coding"}) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + # The transcript resolved via the fallback, so the variant translated instead of skipping. + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 2} + async def test_creates_genomic_and_coding_candidate_alleles( self, session, @@ -1296,7 +1397,7 @@ async def test_uses_latest_cdna_row_within_the_run( sample_independent_reverse_translation_run, sample_score_set, ): - """When a run has more than one cdna TargetGeneMapping for a target (same run date), + """When a run has more than one cdna TargetGeneMapping for a target (same job_run_id), RT reverse-translates against the newest (highest id), not an arbitrary one.""" variant = Variant( score_set_id=sample_score_set.id, @@ -1306,7 +1407,7 @@ async def test_uses_latest_cdna_row_within_the_run( ) session.add(variant) session.commit() - # Mapping emits a cdna TargetGeneMapping (NM_999999.1) stamped with the run date. + # Mapping emits a cdna TargetGeneMapping (NM_999999.1) stamped with this run's job_run_id. await _map_variants( session, mock_worker_ctx, @@ -1316,7 +1417,7 @@ async def test_uses_latest_cdna_row_within_the_run( ) target_gene = sample_score_set.target_genes[0] run_date = self._run_mapped_date(session, target_gene.id) - # A newer cdna row (higher id) for the same target and same run date. + # A newer cdna row (higher id) for the same target, from the same run. session.add( TargetGeneMapping( target_gene_id=target_gene.id, @@ -1325,6 +1426,7 @@ async def test_uses_latest_cdna_row_within_the_run( preferred=False, tool_version="pytest.0.0", mapped_date=run_date, + job_run_id=sample_independent_variant_mapping_run.id, ) ) session.commit() @@ -1359,10 +1461,10 @@ async def test_ignores_stale_cdna_row_from_a_different_run( sample_independent_reverse_translation_run, sample_score_set, ): - """A cdna row left behind by a *different* run (different run date) is not used: - the current run emitted no cdna row, so the variant is skipped (transcript - unresolved) rather than reverse-translated against the stale transcript -- the - mapped_date anchor narrows the accumulation edge case (mavedb-api#763).""" + """A leftover cdna row that does not belong to the current run is not used: the current run + emitted no cdna row, so the variant is skipped (transcript unresolved) rather than + reverse-translated against the stale transcript. The record resolves its transcript by its own + run's job_run_id, so a row carrying no such anchor never leaks in (mavedb-api#763).""" variant = Variant( score_set_id=sample_score_set.id, urn="variant:1", From 6ecb48d8ddbf64292816d2de0e353dbb06e9a3a4 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 2 Sep 2026 11:43:13 -0700 Subject: [PATCH 133/252] chore(dev): pin ruff to 0.6.8 to match the pre-commit hook The dev dependency allowed ruff ~0.15.0 while the pre-commit hook pins v0.6.8, and the two format some idioms differently (dict-argument call hugging, chained-call breaks). Since the hook is what gates commits, running a newer local ruff rewrote code the hook then reverted, thrashing diffs. Pin the dev dependency to 0.6.8 so poetry run ruff and the hook agree; bump both together in future. --- poetry.lock | 40 ++++++++++++++++++++-------------------- pyproject.toml | 2 +- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/poetry.lock b/poetry.lock index a7cee0a75..ffa4c5618 100644 --- a/poetry.lock +++ b/poetry.lock @@ -4167,30 +4167,30 @@ pyasn1 = ">=0.1.3" [[package]] name = "ruff" -version = "0.15.12" +version = "0.6.8" description = "An extremely fast Python linter and code formatter, written in Rust." optional = false python-versions = ">=3.7" groups = ["dev"] files = [ - {file = "ruff-0.15.12-py3-none-linux_armv6l.whl", hash = "sha256:f86f176e188e94d6bdbc09f09bfd9dc729059ad93d0e7390b5a73efe19f8861c"}, - {file = "ruff-0.15.12-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:e3bcd123364c3770b8e1b7baaf343cc99a35f197c5c6e8af79015c666c423a6c"}, - {file = "ruff-0.15.12-py3-none-macosx_11_0_arm64.whl", hash = "sha256:fe87510d000220aa1ed530d4448a7c696a0cae1213e5ec30e5874287b66557b5"}, - {file = "ruff-0.15.12-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:84a1630093121375a3e2a95b4a6dc7b59e2b4ee76216e32d81aae550a832d002"}, - {file = "ruff-0.15.12-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fb129f40f114f089ebe0ca56c0d251cf2061b17651d464bb6478dc01e69f11f5"}, - {file = "ruff-0.15.12-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b0c862b172d695db7598426b8af465e7e9ac00a3ea2a3630ee67eb82e366aaa6"}, - {file = "ruff-0.15.12-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2849ea9f3484c3aca43a82f484210370319e7170df4dfe4843395ddf6c57bc33"}, - {file = "ruff-0.15.12-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9e77c7e51c07fe396826d5969a5b846d9cd4c402535835fb6e21ce8b28fef847"}, - {file = "ruff-0.15.12-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:83b2f4f2f3b1026b5fb449b467d9264bf22067b600f7b6f41fc5958909f449d0"}, - {file = "ruff-0.15.12-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:9ba3b8f1afd7e2e43d8943e55f249e13f9682fde09711644a6e7290eb4f3e339"}, - {file = "ruff-0.15.12-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:e852ba9fdc890655e1d78f2df1499efbe0e54126bd405362154a75e2bde159c5"}, - {file = "ruff-0.15.12-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:dd8aed930da53780d22fc70bdf84452c843cf64f8cb4eb38984319c24c5cd5fd"}, - {file = "ruff-0.15.12-py3-none-musllinux_1_2_i686.whl", hash = "sha256:01da3988d225628b709493d7dc67c3b9b12c0210016b08690ef9bd27970b262b"}, - {file = "ruff-0.15.12-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:9cae0f92bd5700d1213188b31cd3bdd2b315361296d10b96b8e2337d3d11f53e"}, - {file = "ruff-0.15.12-py3-none-win32.whl", hash = "sha256:d0185894e038d7043ba8fd6aee7499ece6462dc0ea9f1e260c7451807c714c20"}, - {file = "ruff-0.15.12-py3-none-win_amd64.whl", hash = "sha256:c87a162d61ab3adca47c03f7f717c68672edec7d1b5499e652331780fe74950d"}, - {file = "ruff-0.15.12-py3-none-win_arm64.whl", hash = "sha256:a538f7a82d061cee7be55542aca1d86d1393d55d81d4fcc314370f4340930d4f"}, - {file = "ruff-0.15.12.tar.gz", hash = "sha256:ecea26adb26b4232c0c2ca19ccbc0083a68344180bba2a600605538ce51a40a6"}, + {file = "ruff-0.6.8-py3-none-linux_armv6l.whl", hash = "sha256:77944bca110ff0a43b768f05a529fecd0706aac7bcce36d7f1eeb4cbfca5f0f2"}, + {file = "ruff-0.6.8-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:27b87e1801e786cd6ede4ada3faa5e254ce774de835e6723fd94551464c56b8c"}, + {file = "ruff-0.6.8-py3-none-macosx_11_0_arm64.whl", hash = "sha256:cd48f945da2a6334f1793d7f701725a76ba93bf3d73c36f6b21fb04d5338dcf5"}, + {file = "ruff-0.6.8-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:677e03c00f37c66cea033274295a983c7c546edea5043d0c798833adf4cf4c6f"}, + {file = "ruff-0.6.8-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9f1476236b3eacfacfc0f66aa9e6cd39f2a624cb73ea99189556015f27c0bdeb"}, + {file = "ruff-0.6.8-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6f5a2f17c7d32991169195d52a04c95b256378bbf0de8cb98478351eb70d526f"}, + {file = "ruff-0.6.8-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:5fd0d4b7b1457c49e435ee1e437900ced9b35cb8dc5178921dfb7d98d65a08d0"}, + {file = "ruff-0.6.8-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f8034b19b993e9601f2ddf2c517451e17a6ab5cdb1c13fdff50c1442a7171d87"}, + {file = "ruff-0.6.8-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6cfb227b932ba8ef6e56c9f875d987973cd5e35bc5d05f5abf045af78ad8e098"}, + {file = "ruff-0.6.8-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6ef0411eccfc3909269fed47c61ffebdcb84a04504bafa6b6df9b85c27e813b0"}, + {file = "ruff-0.6.8-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:007dee844738c3d2e6c24ab5bc7d43c99ba3e1943bd2d95d598582e9c1b27750"}, + {file = "ruff-0.6.8-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:ce60058d3cdd8490e5e5471ef086b3f1e90ab872b548814e35930e21d848c9ce"}, + {file = "ruff-0.6.8-py3-none-musllinux_1_2_i686.whl", hash = "sha256:1085c455d1b3fdb8021ad534379c60353b81ba079712bce7a900e834859182fa"}, + {file = "ruff-0.6.8-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:70edf6a93b19481affd287d696d9e311388d808671bc209fb8907b46a8c3af44"}, + {file = "ruff-0.6.8-py3-none-win32.whl", hash = "sha256:792213f7be25316f9b46b854df80a77e0da87ec66691e8f012f887b4a671ab5a"}, + {file = "ruff-0.6.8-py3-none-win_amd64.whl", hash = "sha256:ec0517dc0f37cad14a5319ba7bba6e7e339d03fbf967a6d69b0907d61be7a263"}, + {file = "ruff-0.6.8-py3-none-win_arm64.whl", hash = "sha256:8d3bb2e3fbb9875172119021a13eed38849e762499e3cfde9588e4b4d70968dc"}, + {file = "ruff-0.6.8.tar.gz", hash = "sha256:a5bf44b1aa0adaf6d9d20f86162b34f7c593bfedabc51239953e446aefc8ce18"}, ] [[package]] @@ -5233,4 +5233,4 @@ server = ["aiocache", "alembic", "alembic-utils", "arq", "authlib", "biocommons" [metadata] lock-version = "2.1" python-versions = "^3.11" -content-hash = "615d9e01ee217622e30a81624e5f042b18b2ff5a6d519114e005dcc22fa5cff6" +content-hash = "186d92c69885b37db17b63bba4e2e56a87224bd52a6468bb68b4e40061bebb44" diff --git a/pyproject.toml b/pyproject.toml index 20fc4cbf7..4644b0ad2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -89,7 +89,7 @@ types-python-jose = "~3.3.4" types-PyYAML = "~6.0.12.20240808" redis = "~5.0.2" requests-mock = "~1.11.0" -ruff = "~0.15.0" +ruff = "0.6.8" SQLAlchemy = { extras = ["mypy"], version = "~2.0.0" } From 39791a6b2aa8566dd0ee31c8ed4a59951c7d78c9 Mon Sep 17 00:00:00 2001 From: David Reinhart Date: Wed, 2 Sep 2026 13:28:46 -0700 Subject: [PATCH 134/252] fix(logging): record the complete request path for variant endpoints PopulatedRawContextMiddleware took the request path from request.url.path. Starlette builds that URL by reassembling the scope into a string and re-parsing it, so urlsplit reads everything after the first '#' as a fragment. A variant URN is {score_set_urn}#{n}, which means the path in the canonical log line stopped at the score set: the variant number and the sub-resource were both dropped, and so was the query string, which sits after the '#' in the reassembled string. /variants/{urn}, /variants/{urn}/csv and /variants/{urn}/csv-namespaces logged one indistinguishable path, and no log line recorded which variant had been asked for. Read the path from the ASGI scope, which arrives percent-decoded and whole. ASGI guarantees 'path' on both http and websocket scopes, so this needs none of the defensiveness the method read below it carries, where only an http scope has the key. Nothing reads this context key. log_request serializes the context into the "Request completed." line and no behaviour depends on the value, so the change is confined to what the logs say. lib/slack.py interpolates request.url whole, and URL.__str__ returns the unparsed string, so the fragment survives there and it was never affected. --- src/mavedb/lib/logging/context.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/mavedb/lib/logging/context.py b/src/mavedb/lib/logging/context.py index 075efb586..ad3e84413 100644 --- a/src/mavedb/lib/logging/context.py +++ b/src/mavedb/lib/logging/context.py @@ -23,7 +23,10 @@ async def set_context(self, request: Union[Request, HTTPConnection]) -> dict: ctx: dict[str, Any] = {} ctx["request_ns"] = time.time_ns() - ctx["path"] = request.url.path + # The ASGI scope rather than request.url.path, which truncates at a '#'. Starlette builds + # request.url by re-parsing, so a variant URN's '#' opens a fragment there and the variant + # number and sub-resource are dropped: three /variants routes all logged the same path. + ctx["path"] = request.scope["path"] if isinstance(request, Request): ctx["method"] = request.method From c2153f3fa4b94ce5a2870e2a9de09880732f627d Mon Sep 17 00:00:00 2001 From: David Reinhart Date: Tue, 8 Sep 2026 08:41:34 -0700 Subject: [PATCH 135/252] test(urns): guard the forwarding lib tests behind the server extra The two "Pytest on Core Dependencies" jobs run poetry install --with dev without --extras server, so fastapi, starlette, arq, cdot and psycopg2 are all absent. tests/lib/test_urn_redirects.py imported mavedb.lib.urn_redirects at module scope, which reaches fastapi and starlette directly and arq, biocommons and cdot through mavedb.deps, so the module failed to import at collection rather than skipping, and one unimportable module ends the whole run: "Interrupted: 1 error during collection". Guard the import the way every sibling module already does. tests/routers/test_urn_redirects.py carried its guards from the start; this one was written without them. Verified under a meta path finder that makes the server extra unimportable: tests/ collects clean and runs 980 passed, 116 skipped, where before it stopped during collection. --- tests/lib/test_urn_redirects.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/lib/test_urn_redirects.py b/tests/lib/test_urn_redirects.py index 9f5efb116..aab0b2da5 100644 --- a/tests/lib/test_urn_redirects.py +++ b/tests/lib/test_urn_redirects.py @@ -1,7 +1,17 @@ """Tests for forwarding URNs that publication has retired.""" +# ruff: noqa: E402 + import pytest +# The module under test reaches fastapi and starlette directly, and arq, biocommons and cdot through +# mavedb.deps, all of which live in the `server` extra. Guarding the import keeps the core-dependency +# CI job skipping this module rather than failing to collect it. The DB fixtures need psycopg2. +pytest.importorskip("psycopg2") +pytest.importorskip("fastapi") +pytest.importorskip("arq") +pytest.importorskip("cdot") + from mavedb.lib.urn_redirects import forwarded_path, record_urn_redirect from mavedb.models.urn_redirect import UrnRedirect From 32879d23109338ee093fc2f364932d0c746c6f6d Mon Sep 17 00:00:00 2001 From: Estelle Da Date: Fri, 11 Sep 2026 16:47:02 +1000 Subject: [PATCH 136/252] Fix the outer source not found error in both Method sources and Threshold sources in creating a new score set and calibration. --- src/mavedb/lib/external_publications.py | 11 ++++++++++- src/mavedb/lib/identifiers.py | 10 ++++++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/mavedb/lib/external_publications.py b/src/mavedb/lib/external_publications.py index a48ff7185..96c4f8dd7 100644 --- a/src/mavedb/lib/external_publications.py +++ b/src/mavedb/lib/external_publications.py @@ -564,4 +564,13 @@ def _fetch(self, url: str, return_format: str = "json") -> Any: return [] response.raise_for_status() - return json.loads(response.text) if return_format == "json" else response.text + + if return_format == "json": + if not response.text or not response.text.strip(): + return [] + try: + return json.loads(response.text) + except json.JSONDecodeError: + return [] + + return response.text diff --git a/src/mavedb/lib/identifiers.py b/src/mavedb/lib/identifiers.py index d5bfdbd46..893442aff 100644 --- a/src/mavedb/lib/identifiers.py +++ b/src/mavedb/lib/identifiers.py @@ -216,7 +216,10 @@ async def fetch_biorxiv_article(identifier: str) -> Optional[RxivContentDetail]: Fetch an existing bioRxiv article from Rxiv """ fetch = Rxiv("https://api.biorxiv.org", "biorxiv") - articles = fetch.content_detail(identifier=identifier) + try: + articles = fetch.content_detail(identifier=identifier) + except Exception: + return None try: return articles[-1] except IndexError: @@ -228,7 +231,10 @@ async def fetch_medrxiv_article(identifier: str) -> Optional[RxivContentDetail]: Fetch an existing medRxiv article from Rxiv """ fetch = Rxiv("https://api.biorxiv.org", "medrxiv") - articles = fetch.content_detail(identifier=identifier) + try: + articles = fetch.content_detail(identifier=identifier) + except Exception: + return None try: return articles[-1] except IndexError: From 8a4b5e41d087dc2361bbd018f64fd4bcda424097 Mon Sep 17 00:00:00 2001 From: Estelle Da Date: Mon, 14 Sep 2026 15:57:49 +1000 Subject: [PATCH 137/252] Debug for test_create_experiment_rxiv_timeout. --- src/mavedb/lib/identifiers.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/mavedb/lib/identifiers.py b/src/mavedb/lib/identifiers.py index 893442aff..d5bfdbd46 100644 --- a/src/mavedb/lib/identifiers.py +++ b/src/mavedb/lib/identifiers.py @@ -216,10 +216,7 @@ async def fetch_biorxiv_article(identifier: str) -> Optional[RxivContentDetail]: Fetch an existing bioRxiv article from Rxiv """ fetch = Rxiv("https://api.biorxiv.org", "biorxiv") - try: - articles = fetch.content_detail(identifier=identifier) - except Exception: - return None + articles = fetch.content_detail(identifier=identifier) try: return articles[-1] except IndexError: @@ -231,10 +228,7 @@ async def fetch_medrxiv_article(identifier: str) -> Optional[RxivContentDetail]: Fetch an existing medRxiv article from Rxiv """ fetch = Rxiv("https://api.biorxiv.org", "medrxiv") - try: - articles = fetch.content_detail(identifier=identifier) - except Exception: - return None + articles = fetch.content_detail(identifier=identifier) try: return articles[-1] except IndexError: From 04fa106a5e5d998ec0d9079d0c6980a5c4095209 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Mon, 14 Sep 2026 09:58:21 -0700 Subject: [PATCH 138/252] feat(reverse-translation): skip cis-phased multivariant hgvs before rt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cis-phased multivariant assay HGVS (`NC_…:g.[a;b;…]`) cannot be forward-translated -- the engine's parser rejects the allele-list opening bracket. These were previously sent to the engine anyway, where they counted as failures that the drop gate reads as a regression. - add `is_cis_phased_hgvs` to key on the allele-list opener while excluding tandem-repeat brackets (`c.101_102[4]`) - screen cis-phased inputs out of the reverse translation job before they reach the engine - add `CIS_PHASED_UNSUPPORTED` event reason and record these as a benign NOT_APPLICABLE skip instead of a failure --- src/mavedb/lib/hgvs.py | 12 +++++ src/mavedb/models/enums/event_reason.py | 3 ++ .../variant_processing/reverse_translation.py | 38 ++++++++++++-- tests/lib/test_hgvs.py | 18 +++++++ .../test_reverse_translation.py | 51 +++++++++++++++++++ 5 files changed, 118 insertions(+), 4 deletions(-) diff --git a/src/mavedb/lib/hgvs.py b/src/mavedb/lib/hgvs.py index f974506b9..b7f2428f1 100644 --- a/src/mavedb/lib/hgvs.py +++ b/src/mavedb/lib/hgvs.py @@ -117,6 +117,18 @@ def parse_simple_substitution(hgvs: Optional[str]) -> Optional[SequenceBlock]: return parse_simple_nucleotide_substitution(hgvs) or parse_simple_protein_substitution(hgvs) +def is_cis_phased_hgvs(hgvs_string: str) -> bool: + """True for an accession-qualified allele-list expression (``NC_…:g.[123A>G;125T>C]``). + + Keys on the allele-list opener ``.[`` — the bracket immediately after the coordinate prefix — + so tandem-repeat brackets (``c.101_102[4]``, bracket after a position) are not misread as an + allele list. Reverse translation's forward parser rejects the ``[`` opening the list, so callers + skip these inputs rather than send a guaranteed failure to the engine. + """ + _, separator, remainder = hgvs_string.partition(":") + return bool(separator) and ".[" in remainder + + def split_cis_phased_hgvs(hgvs_string: str) -> list[str]: """Split a cis-phased multivariant HGVS expression into fully-qualified component strings. diff --git a/src/mavedb/models/enums/event_reason.py b/src/mavedb/models/enums/event_reason.py index ece15e554..9a9ffdbd3 100644 --- a/src/mavedb/models/enums/event_reason.py +++ b/src/mavedb/models/enums/event_reason.py @@ -44,6 +44,9 @@ class EventReason(str, Enum): ) NO_ASSAY_LEVEL_HGVS = "no_assay_level_hgvs" # no assay-level HGVS to translate (RT) NOT_TRANSLATABLE = "not_translatable" # protein consequence's edit type has no DNA equivalence class (RT) + CIS_PHASED_UNSUPPORTED = ( + "cis_phased_unsupported" # cis-phased multivariant assay HGVS; forward parser cannot read the allele list (RT) + ) # failed — errored API_ERROR = "api_error" # network/timeout/upstream error (ClinVar, VEP, LDH, CAR no-response) diff --git a/src/mavedb/worker/jobs/variant_processing/reverse_translation.py b/src/mavedb/worker/jobs/variant_processing/reverse_translation.py index dcaaef642..a33a70008 100644 --- a/src/mavedb/worker/jobs/variant_processing/reverse_translation.py +++ b/src/mavedb/worker/jobs/variant_processing/reverse_translation.py @@ -38,7 +38,7 @@ ) from mavedb.lib.annotation_status_manager import AnnotationStatusManager -from mavedb.lib.hgvs import extract_accession, strip_protein_prediction_parens +from mavedb.lib.hgvs import extract_accession, is_cis_phased_hgvs, strip_protein_prediction_parens from mavedb.lib.types.workflow import JobExecutionOutcome from mavedb.lib.variant_translations import get_or_create_allele from mavedb.lib.vrs_utils import translate_hgvs_to_variation @@ -319,7 +319,16 @@ async def reverse_translate_variants_for_score_set( variant_inputs: list[Any] = [] variant_input_map: dict[int, tuple[MappingRecord, Variant]] = {} skipped_variants: list[_TranscriptResolution] = [] + cis_phased_variants: list[_TranscriptResolution] = [] for p in transcript_resolutions: + # A cis-phased multivariant HGVS (NC_…:g.[a;b;…]) cannot be forward-translated: the engine's + # parser rejects the allele list at the opening "[". These are almost always true multi-mutant + # haplotypes, not single-codon components, so they have no meaningful cross-level sibling triple. + # Record a benign skip rather than send a guaranteed failure to the engine. + if p.rec.hgvs_assay_level and is_cis_phased_hgvs(p.rec.hgvs_assay_level): + cis_phased_variants.append(p) + continue + transcript = p.gene_transcript or ( transcript_by_protein.get(p.protein_accession) if p.protein_accession else None ) @@ -332,14 +341,24 @@ async def reverse_translate_variants_for_score_set( variant_input_map[id(inp)] = (p.rec, p.variant) total = len(variant_inputs) - job_manager.save_to_context({"total_variants": total, "skipped_variants": len(skipped_variants)}) + job_manager.save_to_context( + { + "total_variants": total, + "skipped_variants": len(skipped_variants), + "cis_phased_skipped": len(cis_phased_variants), + } + ) job_manager.update_progress( 10, 100, - f"Prepared {total} variants for reverse translation ({len(skipped_variants)} skipped, no coding transcript).", + f"Prepared {total} variants for reverse translation " + f"({len(skipped_variants)} no coding transcript, {len(cis_phased_variants)} cis-phased).", ) logger.info( - msg=f"Running reverse translation for {total} variants ({len(skipped_variants)} skipped).", + msg=( + f"Running reverse translation for {total} variants " + f"({len(skipped_variants)} skipped no-transcript, {len(cis_phased_variants)} skipped cis-phased)." + ), extra=job_manager.logging_context(), ) @@ -566,6 +585,17 @@ async def reverse_translate_variants_for_score_set( metadata={"hgvs_input": p.rec.hgvs_assay_level}, ) + # add cis-phased multivariant inputs as NOT_APPLICABLE skips + annotation_counts["skipped"] += len(cis_phased_variants) + for p in cis_phased_variants: + _annotate_translation( + annotation_manager, + variant_id=p.variant.id, + disposition=Disposition.NOT_APPLICABLE, + reason=EventReason.CIS_PHASED_UNSUPPORTED, + metadata={"hgvs_input": p.rec.hgvs_assay_level}, + ) + annotation_manager.flush() outcome_data = dict(annotation_counts) diff --git a/tests/lib/test_hgvs.py b/tests/lib/test_hgvs.py index 5ecb40e9d..47687c6a1 100644 --- a/tests/lib/test_hgvs.py +++ b/tests/lib/test_hgvs.py @@ -3,6 +3,7 @@ from mavedb.lib.hgvs import ( SequenceBlock, extract_accession, + is_cis_phased_hgvs, join_cis_phased_hgvs, parse_simple_nucleotide_substitution, parse_simple_protein_substitution, @@ -110,6 +111,23 @@ def test_extract_accession(hgvs, expected): assert extract_accession(hgvs) == expected +@pytest.mark.unit +@pytest.mark.parametrize( + ("hgvs", "expected"), + [ + ("NC_000016.10:g.[1314031A>T;1320511T>C]", True), + ("NM_000001.1:c.[1A>G;3T>C]", True), + ("NC_000016.10:g.[1314031A>T]", True), # single-member allele list still fails the forward parser + ("NC_000001.11:g.1000A>G", False), + ("NM_000546.6:c.1216G>A", False), + ("NM_000001.1:c.101_102[4]", False), # tandem-repeat brackets are not an allele list + ("g.[1000A>G;1002T>C]", False), # accession-less: not the shape RT feeds the engine + ], +) +def test_is_cis_phased_hgvs(hgvs, expected): + assert is_cis_phased_hgvs(hgvs) is expected + + def test_split_cis_phased_hgvs_passes_through_single_variant(): assert split_cis_phased_hgvs("NC_000001.11:g.1000A>G") == ["NC_000001.11:g.1000A>G"] diff --git a/tests/worker/jobs/variant_processing/test_reverse_translation.py b/tests/worker/jobs/variant_processing/test_reverse_translation.py index cea59c676..c4d16d055 100644 --- a/tests/worker/jobs/variant_processing/test_reverse_translation.py +++ b/tests/worker/jobs/variant_processing/test_reverse_translation.py @@ -353,6 +353,57 @@ async def test_same_day_remap_without_a_cdna_row_skips_rather_than_binding_a_sta assert _non_authoritative_links(session) == [] assert len(_cross_level_events(session, sample_score_set.id, reason="transcript_unresolved")) == 1 + async def test_cis_phased_assay_hgvs_is_skipped_before_the_engine( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """A cis-phased multivariant assay HGVS (``g.[a;b]``) cannot be forward-translated -- the + engine's parser rejects the allele list at the opening ``[``. The job screens it out up front + and records a benign NOT_APPLICABLE / cis_phased_unsupported skip, never handing it to the + engine (which would otherwise count it as a failure the drop gate reads as a regression).""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + # Map first so the variant gains a transcript-bearing authoritative record: the skip must fire + # on the cis-phased shape even though a transcript resolves, not as a transcript-unresolved skip. + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + cis_phased = "NC_000016.10:g.[1314031A>T;1320511T>C]" + record = _record_for(session, variant.id) + record.hgvs_assay_level = cis_phased + session.commit() + + def _fail_if_engine_sees_it(inputs, *, transcripts, coordinates, config): + assert list(inputs) == [], "a cis-phased input must be screened out before the engine" + return [], [] + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", _fail_if_engine_sees_it), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", fake_translate({})), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 0, "failed": 0, "skipped": 1, "alleles_created": 0} + assert _non_authoritative_links(session) == [] + + events = _cross_level_events(session, sample_score_set.id, reason="cis_phased_unsupported") + assert len(events) == 1 + assert events[0].disposition == "not_applicable" + assert events[0].event_metadata["hgvs_input"] == cis_phased + async def test_falls_back_to_mapped_date_when_the_record_has_no_job_run_id( self, session, From b5d45ca0fdd77687e96080e15cc5b812ed17535e Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Mon, 14 Sep 2026 13:07:22 -0700 Subject: [PATCH 139/252] fix(migration): handle allele cache misses due to VRS id drift --- .../migrate_mapped_variants_to_allele_substrate.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/alembic/manual_migrations/migrate_mapped_variants_to_allele_substrate.py b/alembic/manual_migrations/migrate_mapped_variants_to_allele_substrate.py index 68db9d1d6..0d9ddc0ec 100644 --- a/alembic/manual_migrations/migrate_mapped_variants_to_allele_substrate.py +++ b/alembic/manual_migrations/migrate_mapped_variants_to_allele_substrate.py @@ -420,6 +420,17 @@ def _authoritative_allele( return None existing = allele_cache.get(digest) + if existing is None: + # `allele_cache` is seeded by a per-chunk prefetch keyed on the STORED `post_mapped` id, but + # alleles deduplicate by their RECOMPUTED digest. Under VRS id drift (stored id != recomputed + # digest) an already-persisted allele carrying this true digest is absent from the cache, so a + # blind INSERT below would violate `uq_alleles_vrs_digest`, and the failed flush would roll the + # whole variant back — dropping its MappingRecord, not just the allele link. Fall back to a + # lookup by the recomputed digest (only on a cache miss, i.e. the drifted minority) so a match + # deduplicates onto the existing row instead of colliding. See scripts/audit_allele_identifiers. + existing = db.scalar(sa.select(Allele).where(Allele.vrs_digest == digest)) + if existing is not None: + allele_cache[digest] = existing if existing is not None: _fill_missing_allele_fields(existing, mv, level) return existing From c2692c39b4fd550c892bff05a41f654b57054277 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 15 Sep 2026 10:25:09 -0700 Subject: [PATCH 140/252] fix(mapped-variant): read the redirect target from the ASGI scope, not request.url A mapped-variant URN is {score_set_urn}#{n}, so request.url truncates at the '#' when Starlette rebuilds it by re-parsing the decoded path -- dropping the variant number, the sub-resource, and the query string that followed it. The redirect target itself was unaffected (it's built from the routed urn path param), but the query string appended to it and the surface logged to record_deprecated_usage were both silently wrong for any request combining a fragment-shaped URN with a real query string. Renamed the helper to _redirect_to_successor -- _redirect didn't say what it actually built (an RFC 8594 deprecation response), and the module already uses "successor" as the term for the replacement resource. --- src/mavedb/routers/mapped_variant.py | 24 ++++++++++++++++-------- tests/routers/test_mapped_variant.py | 9 +++++++++ 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/src/mavedb/routers/mapped_variant.py b/src/mavedb/routers/mapped_variant.py index 39417224e..244e490ff 100644 --- a/src/mavedb/routers/mapped_variant.py +++ b/src/mavedb/routers/mapped_variant.py @@ -21,15 +21,19 @@ ) -def _redirect(request: Request, target: str) -> RedirectResponse: +def _redirect_to_successor(request: Request, target: str) -> RedirectResponse: """Redirect to the successor, tagging the response with RFC 8594 deprecation headers. The redirect status already moves a well-behaved client, but the headers announce the deprecation to tooling that follows the 301 silently: ``Deprecation``, a ``Link`` naming this exact successor, and a ``Warning`` a human sees in logs. The successor here is a genuine 1:1 replacement, so it is safe to name. """ - url = f"{target}?{request.url.query}" if request.url.query else target - record_deprecated_usage(request.url.path, successor=target) + # The ASGI scope rather than request.url: a mapped-variant URN's '#' opens a fragment when Starlette + # rebuilds request.url by re-parsing the decoded path, which silently drops the query string too. + path = request.scope["path"] + query = request.scope.get("query_string", b"").decode("ascii") + url = f"{target}?{query}" if query else target + record_deprecated_usage(path, successor=target) headers = deprecation_headers( successor=target, warning=f"This resource has moved permanently to {target}; update your integration.", @@ -46,7 +50,7 @@ def _redirect(request: Request, target: str) -> RedirectResponse: ) def redirect_mapped_variant(*, urn: str, request: Request) -> RedirectResponse: """This resource has moved. Use ``GET /variants/{urn}`` instead.""" - return _redirect(request, f"{ROUTER_BASE_PREFIX}/variants/{quote(urn, safe='')}") + return _redirect_to_successor(request, f"{ROUTER_BASE_PREFIX}/variants/{quote(urn, safe='')}") @router.get( @@ -57,7 +61,7 @@ def redirect_mapped_variant(*, urn: str, request: Request) -> RedirectResponse: ) def redirect_mapped_variant_study_result(*, urn: str, request: Request) -> RedirectResponse: """This resource has moved. Use ``GET /variants/{urn}/va/study-result`` instead.""" - return _redirect(request, f"{ROUTER_BASE_PREFIX}/variants/{quote(urn, safe='')}/va/study-result") + return _redirect_to_successor(request, f"{ROUTER_BASE_PREFIX}/variants/{quote(urn, safe='')}/va/study-result") @router.get( @@ -68,7 +72,9 @@ def redirect_mapped_variant_study_result(*, urn: str, request: Request) -> Redir ) def redirect_mapped_variant_functional_impact_statement(*, urn: str, request: Request) -> RedirectResponse: """This resource has moved. Use ``GET /variants/{urn}/va/functional-statement`` instead.""" - return _redirect(request, f"{ROUTER_BASE_PREFIX}/variants/{quote(urn, safe='')}/va/functional-statement") + return _redirect_to_successor( + request, f"{ROUTER_BASE_PREFIX}/variants/{quote(urn, safe='')}/va/functional-statement" + ) @router.get( @@ -79,7 +85,9 @@ def redirect_mapped_variant_functional_impact_statement(*, urn: str, request: Re ) def redirect_mapped_variant_acmg_evidence_line(*, urn: str, request: Request) -> RedirectResponse: """This resource has moved. Use ``GET /variants/{urn}/va/pathogenicity-statement`` instead.""" - return _redirect(request, f"{ROUTER_BASE_PREFIX}/variants/{quote(urn, safe='')}/va/pathogenicity-statement") + return _redirect_to_successor( + request, f"{ROUTER_BASE_PREFIX}/variants/{quote(urn, safe='')}/va/pathogenicity-statement" + ) @router.get( @@ -106,4 +114,4 @@ def redirect_mapped_variants_by_identifier( ``as_of``; a caller relying on ``only_current=false`` should switch to passing an explicit ``as_of`` timestamp rather than expecting it to carry over through this redirect. """ - return _redirect(request, f"{ROUTER_BASE_PREFIX}/variants/vrs/{quote(identifier, safe='')}") + return _redirect_to_successor(request, f"{ROUTER_BASE_PREFIX}/variants/vrs/{quote(identifier, safe='')}") diff --git a/tests/routers/test_mapped_variant.py b/tests/routers/test_mapped_variant.py index 17438b5f4..5a2a33bfd 100644 --- a/tests/routers/test_mapped_variant.py +++ b/tests/routers/test_mapped_variant.py @@ -48,6 +48,15 @@ def test_redirect_mapped_variant_va_routes(client, suffix): assert response.headers["location"] == f"/api/v1/variants/{quote_plus(TEST_URN)}/{suffix}" +def test_redirect_carries_a_query_string_alongside_a_fragment_shaped_urn(client): + """TEST_URN's '#' would open a fragment if the target were built from request.url instead of the + ASGI scope, and a fragment swallows everything after it -- including a real query string.""" + response = client.get(f"/api/v1/mapped-variants/{quote_plus(TEST_URN)}?start=0&limit=1", follow_redirects=False) + + assert response.status_code == 301 + assert response.headers["location"] == f"/api/v1/variants/{quote_plus(TEST_URN)}?start=0&limit=1" + + def test_redirect_mapped_variants_by_identifier(client): identifier = "ga4gh:VA.0123456789abcdefghijklmnopqrstuv" response = client.get( From 69e035e6bc804ae8dca0394cacbc675dd22fae2e Mon Sep 17 00:00:00 2001 From: David Reinhart Date: Tue, 15 Sep 2026 13:14:07 -0700 Subject: [PATCH 141/252] fix(score-sets): reject publishing a score set that is already published publish_score_set assigned a fresh URN unconditionally: generate_score_set_urn ran on every call, and nothing checked whether the score set was already public. _handle_publish_action permits an owner to publish their own score set without regard to its state, so an owner could POST .../publish a second time and give a live public record a new URN, retiring the one already cited, indexed and shared. refresh_variant_urns then rewrote every variant URN to match. Reject the second call with a 409 before any URN is generated. This gap predates the URN forwarding in cc762ce6 and is reachable without it, but forwarding is what makes it worth closing now: the redirect table records what each retired URN became, and a repeat publish would fill it with rows retiring permanent URNs, which forwarded_path never matches because it only resolves tmp: shapes. With publication confined to private score sets, every recorded old URN is a temporary one. --- src/mavedb/routers/score_sets.py | 7 +++++++ tests/routers/test_score_set.py | 22 ++++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/src/mavedb/routers/score_sets.py b/src/mavedb/routers/score_sets.py index d38988695..cf11e3f86 100644 --- a/src/mavedb/routers/score_sets.py +++ b/src/mavedb/routers/score_sets.py @@ -2531,6 +2531,13 @@ async def publish_score_set( assert_permission(user_data, item, Action.PUBLISH) + if not item.private: + logger.info( + msg="Failed to publish score set; The requested score set has already been published.", + extra=logging_context(), + ) + raise HTTPException(status_code=409, detail="This score set has already been published.") + if not item.experiment: logger.info( msg="Failed to publish score set; The requested score set does not belong to an experiment.", diff --git a/tests/routers/test_score_set.py b/tests/routers/test_score_set.py index 7d9290a9a..5c5a395e4 100644 --- a/tests/routers/test_score_set.py +++ b/tests/routers/test_score_set.py @@ -1748,6 +1748,28 @@ def test_publish_score_set(session, data_provider, client, setup_router_db, data assert all([variant.urn.startswith("urn:mavedb:") for variant in score_set_variants]) +def test_cannot_publish_an_already_published_score_set(session, data_provider, client, setup_router_db, data_files): + """Publishing assigns a fresh URN unconditionally, so a second publish would rename a public record.""" + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") + + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published_score_set = publish_score_set(client, score_set["urn"]) + + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as worker_queue: + response = client.post(f"/api/v1/score-sets/{published_score_set['urn']}/publish") + worker_queue.assert_not_called() + + assert response.status_code == 409 + assert "already been published" in response.json()["detail"] + + # The URN the caller already shared still resolves to this record. + unchanged = client.get(f"/api/v1/score-sets/{published_score_set['urn']}") + assert unchanged.status_code == 200 + assert unchanged.json()["urn"] == published_score_set["urn"] + + def test_publish_score_set_discards_pipeline_when_entrypoint_enqueue_fails( session, data_provider, client, setup_router_db, data_files ): From df88be3471e009732f540ec1f19ce652b5b1d5f1 Mon Sep 17 00:00:00 2001 From: David Reinhart Date: Tue, 15 Sep 2026 14:43:38 -0700 Subject: [PATCH 142/252] refactor(urns): defer redirect target visibility to the permission rules _target_is_public decided whether a redirect's target could be named to an anonymous caller by reading the record's private column. That restated a rule the permission layer already owns, and would drift from it silently: a later change to what makes a record readable would have to be made in both places, or the Location header would disclose a URN a route would have withheld. Fetch the record and ask has_permission(None, target, Action.READ), which covers all three kinds a redirect can point at. Behaviour is unchanged for every case under test: a public target forwards, a private one is withheld, and a target that no longer exists is withheld. Dispatching on the URN pattern with the model named in each branch replaces the table of models the loop read from, which also keeps the fetched record concretely typed rather than as Base. Also cuts the design rationale from forward_retired_urns, leaving what a caller acts on. The removed paragraph on why writes are not forwarded argued from an owner being permitted to republish a published score set, which 69e035e6 no longer allows. --- src/mavedb/lib/urn_redirects.py | 45 +++++++++++---------------------- 1 file changed, 15 insertions(+), 30 deletions(-) diff --git a/src/mavedb/lib/urn_redirects.py b/src/mavedb/lib/urn_redirects.py index 584a0c232..d131d38ab 100644 --- a/src/mavedb/lib/urn_redirects.py +++ b/src/mavedb/lib/urn_redirects.py @@ -32,6 +32,8 @@ from mavedb.deps import get_db from mavedb.lib.logging.context import logging_context, save_to_logging_context +from mavedb.lib.permissions.actions import Action +from mavedb.lib.permissions.core import has_permission from mavedb.lib.validation.urn_re import ( MAVEDB_EXPERIMENT_SET_URN_RE, MAVEDB_EXPERIMENT_URN_RE, @@ -48,14 +50,6 @@ # Methods whose requests are forwarded. See forward_retired_urns for why a write is not. SAFE_METHODS = frozenset({"GET", "HEAD"}) -# The record kinds publication renames, recognized by the shape of the URN it gave them. Matched with -# fullmatch, under which the three patterns are mutually exclusive. -FORWARDING_TARGET_MODELS = ( - (MAVEDB_SCORE_SET_URN_RE, ScoreSet), - (MAVEDB_EXPERIMENT_URN_RE, Experiment), - (MAVEDB_EXPERIMENT_SET_URN_RE, ExperimentSet), -) - def record_urn_redirect(db: Session, old_urn: Optional[str], new_urn: str) -> None: """ @@ -84,22 +78,25 @@ def _target_is_public(db: Session, urn: str) -> bool: caller, since forwarding happens before a route checks anything. Publication only ever records a redirect onto a record it is making public, and nothing in the application returns a published record to private, so a private target should not arise; a row written out of band, or by some - later feature, would be enough for one to. Withholding on anything but a confirmed public record - keeps that from becoming a disclosure. + later feature, would be enough for one to. Deferring to ``Action.READ`` with no user keeps that + from becoming a disclosure, and keeps this answer in step with the rules a route would apply. A target that no longer exists is likewise not public: a deleted record leaves its redirect row behind, and forwarding to it would answer a permanent redirect with a 404. :param db: An active database session. :param urn: The URN a redirect points to. - :return: True only if a record under this URN exists and is public. + :return: True only if a record under this URN exists and an anonymous caller may read it. """ - for urn_re, model in FORWARDING_TARGET_MODELS: - if urn_re.fullmatch(urn): - private = db.execute(select(model.private).where(model.urn == urn)).scalar_one_or_none() - return private is False + target: Optional[ScoreSet | Experiment | ExperimentSet] = None + if MAVEDB_SCORE_SET_URN_RE.fullmatch(urn): + target = db.query(ScoreSet).filter(ScoreSet.urn == urn).one_or_none() + elif MAVEDB_EXPERIMENT_URN_RE.fullmatch(urn): + target = db.query(Experiment).filter(Experiment.urn == urn).one_or_none() + elif MAVEDB_EXPERIMENT_SET_URN_RE.fullmatch(urn): + target = db.query(ExperimentSet).filter(ExperimentSet.urn == urn).one_or_none() - return False + return target is not None and has_permission(None, target, Action.READ).permitted def forwarded_path(db: Session, path: str) -> Optional[str]: @@ -143,20 +140,8 @@ def forward_retired_urns(request: Request, db: Session = Depends(get_db)) -> Non implementation cover every route that takes a URN, sub-resources included: a stale link to a score set's scores CSV or mapped variants is forwarded on the same terms as a link to the score set. - A dependency rather than ASGI middleware, though it sits at the same single point in the request - path, because it needs the request's database session. Middleware runs outside dependency - resolution, so it would have to open a session of its own, which no ``dependency_overrides`` could - redirect and which would therefore reach past the test database. - - ``308`` rather than ``301``: the redirect is permanent, and 308 forbids a client from rewriting the - request to a GET on the way, which is what makes the header safe to emit for any method. - - Only reads are forwarded. What the issue asks for is that shared *links* keep working, and a write - is a different proposition: the caller addressed a private draft, and the record now under that URN - is published, with different rules and a wider audience. ``POST .../publish`` is the sharp case -- - an owner is permitted to publish a published score set, so forwarding a stale one would rename a - live public record. A write to a retired URN keeps getting the 404 it gets today, which tells the - client to look the record up again. + Only reads are forwarded. A write to a retired URN keeps getting the 404 it gets today, which + tells the client to look the record up again. """ if request.scope["method"] not in SAFE_METHODS: return From 38ca5e4b8de277a8ff150e76509077be9a7788bc Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 15 Sep 2026 15:21:57 -0700 Subject: [PATCH 143/252] fix(worker): keep mapping jobs alive during long dcd-mapping calls map_variants_for_score_set delegates the whole score set to dcd-mapping in a single opaque blocking call, so progress_updated_at cannot advance for its duration. cleanup_stalled_jobs (PROGRESS_STALL_MINUTES=30) then falsely reaps any set that maps for longer than 30 minutes, and the retry re-runs from scratch and is reaped again -- large sets can never complete. Bracket the map call with a liveness keepalive that refreshes the heartbeat on a cadence under the stall threshold, plus a size-aware asyncio.wait_for budget that bounds a wedged mapper instead of riding the 23.5h wall-clock backstop. The keepalive runs in the worker event loop, so it dies with a crashed worker and genuine crashes still go stale and get reaped. Interim workaround for delegating opaque work; the durable fix is the mapper reporting progress or chunking the call. Principle documented in best_practices.md under "Long external delegations". --- src/mavedb/worker/best_practices.md | 8 ++ .../worker/jobs/variant_processing/mapping.py | 66 +++++++++++++- .../jobs/variant_processing/test_mapping.py | 86 ++++++++++++++++++- 3 files changed, 156 insertions(+), 4 deletions(-) diff --git a/src/mavedb/worker/best_practices.md b/src/mavedb/worker/best_practices.md index e83eb42f1..5e3906f9d 100644 --- a/src/mavedb/worker/best_practices.md +++ b/src/mavedb/worker/best_practices.md @@ -120,6 +120,14 @@ job_manager.update_progress(50, 100, "Writing to database.") job_manager.update_progress(100, 100, "Variant creation complete.") ``` +### Long external delegations + +A job that hands work to an opaque external call — a single blocking request whose duration you don't control and whose interior you can't see — breaks the checkpoint model. `cleanup_stalled_jobs` reaps a RUNNING job whose `progress_updated_at` hasn't advanced in `PROGRESS_STALL_MINUTES` (30), but you cannot emit real progress while parked inside a black-box call. The heartbeat conflates two signals that opaque delegation forces apart: **liveness** (is the worker alive and still executing — always answerable) and **progress** (is the work advancing or wedged — not answerable from your side of the call). + +**Rule: any long external delegation must expose progress, or be given a bounded duration at its boundary.** Absent one of those, stall detection degrades to liveness and "the worker is up" becomes the only thing you can assert. Real progress comes only from the delegate reporting it (streaming / callback / a status you poll) or from chunking the call so each unit is a real checkpoint — both require a change on the delegate's side and are the preferred fix. + +Until then, the interim pattern (see `map_variants_for_score_set`) is a **liveness keepalive** — a concurrent `asyncio` task that refreshes the heartbeat on a cadence well under `PROGRESS_STALL_MINUTES` so a live worker is not falsely reaped — paired with a **size-aware `asyncio.wait_for` budget** so a genuinely wedged delegate fails in a sensible window rather than riding the 23.5h wall-clock backstop. The keepalive dies with a crashed worker, so real crashes still go stale and get reaped. This is an explicit workaround; it does not claim progress it cannot see. Never `time.sleep` in the keepalive — it blocks the shared event loop and freezes every other job on the worker. + ## Logging Context Always set up logging context early in the job function: diff --git a/src/mavedb/worker/jobs/variant_processing/mapping.py b/src/mavedb/worker/jobs/variant_processing/mapping.py index 063094ff5..f6385a994 100644 --- a/src/mavedb/worker/jobs/variant_processing/mapping.py +++ b/src/mavedb/worker/jobs/variant_processing/mapping.py @@ -6,13 +6,14 @@ """ import asyncio +import contextlib import functools import logging from collections import Counter -from datetime import date +from datetime import date, datetime, timezone from typing import Any -from sqlalchemy import cast, null, select +from sqlalchemy import cast, func, null, select from sqlalchemy.dialects.postgresql import JSONB from mavedb.data_providers.services import vrs_mapper @@ -47,6 +48,48 @@ logger = logging.getLogger(__name__) +# The score-set map is a single opaque blocking call to dcd-mapping: it cannot refresh the progress +# heartbeat mid-call, so `cleanup_stalled_jobs` (PROGRESS_STALL_MINUTES = 30) would reap a live worker +# on any set that maps for longer than that. Two guards bracket the call — a liveness keepalive that +# refreshes the heartbeat on a fixed cadence, and a size-aware wall-clock budget that bounds a wedged +# mapper to a sensible window instead of the 23.5h stall backstop. This is an explicit workaround for +# delegating opaque work; the real fix is for the mapper to report progress (or to chunk the call). +# See worker/best_practices.md, "Long external delegations". TODO: replace with real mapping progress. +MAP_PROGRESS_KEEPALIVE_SECONDS = 300 # heartbeat cadence; must stay well under PROGRESS_STALL_MINUTES (30m) +MAP_BUDGET_BASE_SECONDS = 30 * 60 # floor: overhead + small sets +MAP_BUDGET_PER_VARIANT_SECONDS = 1.0 # ~3x the observed ~0.3 s/variant map throughput, as slack +MAP_BUDGET_MAX_SECONDS = 22 * 60 * 60 # ceiling: stay under the 23.5h stall backstop / 24h ARQ ceiling + + +async def _keepalive_progress(job_manager: JobManager, interval_seconds: int) -> None: + """Refresh the job's progress heartbeat on a fixed cadence during an opaque, un-checkpointable call. + + Liveness, not progress: the task runs in the worker's event loop, so it dies with a crashed worker + and the heartbeat then correctly goes stale — only a *live* worker in a long delegation is kept from + being falsely reaped. Best-effort; a failed refresh is logged and the loop continues. + """ + while True: + await asyncio.sleep(interval_seconds) + try: + job_manager.update_progress( + 30, + 100, + f"Mapping variants using VRS mapping service (in progress, {datetime.now(timezone.utc):%H:%M:%SZ}).", + ) + except Exception: # noqa: BLE001 - the heartbeat is best-effort; a dropped tick is harmless + logger.warning("Mapping progress keepalive tick failed; continuing.", extra=job_manager.logging_context()) + + +def _map_budget_seconds(variant_count: int) -> int: + """Total wall-clock budget for the opaque map call, sized to the score set. + + Bounds a wedged external mapper to a size-appropriate window instead of the 23.5h stall backstop, + while leaving enough slack (the per-variant term is ~3x observed throughput) not to fail a slow but + healthy map. Clamped to a ceiling below the ARQ job timeout. + """ + budget = MAP_BUDGET_BASE_SECONDS + int(variant_count * MAP_BUDGET_PER_VARIANT_SECONDS) + return min(budget, MAP_BUDGET_MAX_SECONDS) + @with_pipeline_management async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobManager) -> JobExecutionOutcome: @@ -109,9 +152,26 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan mapping_results = None + variant_count = job_manager.db.scalar( + select(func.count(Variant.id)).where(Variant.score_set_id == score_set.id) + ) + map_budget = _map_budget_seconds(variant_count or 0) + logger.debug(msg="Mapping variants using VRS mapping service.", extra=job_manager.logging_context()) job_manager.update_progress(30, 100, "Mapping variants using VRS mapping service.") - mapping_results = await loop.run_in_executor(ctx["pool"], blocking) + # Keepalive heartbeat + size-aware budget around the opaque map call (see module docstring above). + keepalive = asyncio.create_task(_keepalive_progress(job_manager, MAP_PROGRESS_KEEPALIVE_SECONDS)) + try: + mapping_results = await asyncio.wait_for(loop.run_in_executor(ctx["pool"], blocking), timeout=map_budget) + except asyncio.TimeoutError as e: + raise TimeoutError( + f"Mapping for {score_set.urn} exceeded its {map_budget}s budget ({variant_count} variants); " + "the mapping service did not return in time." + ) from e + finally: + keepalive.cancel() + with contextlib.suppress(asyncio.CancelledError): + await keepalive logger.debug(msg="Done mapping variants.", extra=job_manager.logging_context()) job_manager.update_progress(80, 100, "Processing mapped variants.") diff --git a/tests/worker/jobs/variant_processing/test_mapping.py b/tests/worker/jobs/variant_processing/test_mapping.py index 3737d56bf..659cfed84 100644 --- a/tests/worker/jobs/variant_processing/test_mapping.py +++ b/tests/worker/jobs/variant_processing/test_mapping.py @@ -4,6 +4,8 @@ pytest.importorskip("arq") +import asyncio +import contextlib from asyncio.unix_events import _UnixSelectorEventLoop from datetime import date from unittest.mock import MagicMock, patch @@ -21,7 +23,13 @@ from mavedb.models.target_gene_mapping import TargetGeneMapping from mavedb.models.variant import Variant from mavedb.models.annotation_event import AnnotationEvent -from mavedb.worker.jobs.variant_processing.mapping import map_variants_for_score_set +from mavedb.worker.jobs.variant_processing.mapping import ( + MAP_BUDGET_BASE_SECONDS, + MAP_BUDGET_MAX_SECONDS, + _keepalive_progress, + _map_budget_seconds, + map_variants_for_score_set, +) from mavedb.worker.lib.managers.job_manager import JobManager from tests.helpers.constants import TEST_CODING_LAYER, TEST_GENOMIC_LAYER, TEST_PROTEIN_LAYER from tests.helpers.util.setup.worker import construct_mock_mapping_output, create_variants_in_score_set @@ -2050,3 +2058,79 @@ async def dummy_mapping_job(): for job_run in pipeline_run.job_runs: if job_run.id != sample_pipeline_variant_mapping_run.id: assert job_run.status == JobStatus.SKIPPED + + +MAP_MODULE = "mavedb.worker.jobs.variant_processing.mapping" + + +@pytest.mark.unit +def test_map_budget_seconds_floor_scale_and_ceiling(): + # Floor for an empty/tiny set (overhead only), linear scale in between, clamped at the ceiling. + assert _map_budget_seconds(0) == MAP_BUDGET_BASE_SECONDS + assert _map_budget_seconds(48_000) == MAP_BUDGET_BASE_SECONDS + 48_000 + assert _map_budget_seconds(100_000_000) == MAP_BUDGET_MAX_SECONDS + + +@pytest.mark.unit +@pytest.mark.asyncio +async def test_keepalive_progress_ticks_and_survives_a_failed_tick(): + """The liveness heartbeat refreshes progress on its cadence and keeps going if a tick fails.""" + ticks = 0 + + def _update(*args, **kwargs): + nonlocal ticks + ticks += 1 + if ticks == 2: # a transient commit failure must not kill the heartbeat + raise RuntimeError("transient progress-commit blip") + + jm = MagicMock() + jm.update_progress.side_effect = _update + jm.logging_context.return_value = {} + + task = asyncio.create_task(_keepalive_progress(jm, 0.001)) + await asyncio.sleep(0.05) + task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await task + + assert ticks >= 3 # kept ticking past the failing 2nd tick + assert all(call.args[0] == 30 for call in jm.update_progress.call_args_list) # each tick reasserts 30% liveness + + +@pytest.mark.unit +@pytest.mark.asyncio +async def test_map_variants_for_score_set_budget_timeout_fails_the_job( + session, + with_independent_processing_runs, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_score_set, +): + """A map that outlasts its size-aware budget fails the job, bounding a wedged mapper.""" + + async def slow_map(): + await asyncio.sleep(1) + return {} + + caught = None + result = None + with ( + patch(f"{MAP_MODULE}._map_budget_seconds", return_value=0.05), + patch.object(_UnixSelectorEventLoop, "run_in_executor", return_value=slow_map()), + ): + try: + result = await map_variants_for_score_set( + mock_worker_ctx, + sample_independent_variant_mapping_run.id, + JobManager(session, mock_worker_ctx["redis"], sample_independent_variant_mapping_run.id), + ) + except Exception as e: # noqa: BLE001 - asserted on below + caught = e + + assert sample_score_set.mapping_state == MappingState.failed + # Budget-exceeded surfaces as a timeout, whether the decorator re-raises or returns a failed outcome. + if caught is not None: + assert isinstance(caught, TimeoutError) + assert "budget" in str(caught) + else: + assert result.status == JobStatus.FAILED From a5c4044d3eea617eec98cc08e98eda5d953409a2 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 16 Sep 2026 09:19:46 -0700 Subject: [PATCH 144/252] feat(calibration-controls): add status enum and calibration_controls schema Groundwork for #754: CalibrationControlStatus enum (#748) and the CalibrationControl ORM model + calibration_controls table, with disease / controls_not_phi columns on score_calibrations and the ScoreCalibration.controls relationship (#749). Migration parents on the release head; no RT dependency. Validation (#751), PHI gate (#752), view models (#750), and endpoint wiring (#753) are deliberately not included. --- ...add_calibration_controls_table_and_phi_.py | 65 +++++++++++ src/mavedb/models/__init__.py | 1 + src/mavedb/models/calibration_control.py | 77 +++++++++++++ .../enums/calibration_control_status.py | 14 +++ src/mavedb/models/score_calibration.py | 20 ++++ ...e_calibration_functional_classification.py | 9 ++ tests/lib/test_calibration_control.py | 108 ++++++++++++++++++ 7 files changed, 294 insertions(+) create mode 100644 alembic/versions/024370c4bca7_add_calibration_controls_table_and_phi_.py create mode 100644 src/mavedb/models/calibration_control.py create mode 100644 src/mavedb/models/enums/calibration_control_status.py create mode 100644 tests/lib/test_calibration_control.py diff --git a/alembic/versions/024370c4bca7_add_calibration_controls_table_and_phi_.py b/alembic/versions/024370c4bca7_add_calibration_controls_table_and_phi_.py new file mode 100644 index 000000000..1743d8ff6 --- /dev/null +++ b/alembic/versions/024370c4bca7_add_calibration_controls_table_and_phi_.py @@ -0,0 +1,65 @@ +"""add calibration controls table and phi/disease columns + +Revision ID: 024370c4bca7 +Revises: a7f3c2e9b104 +Create Date: 2026-09-15 16:01:06.013964 + +""" + +import sqlalchemy as sa + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "024370c4bca7" +down_revision = "a7f3c2e9b104" +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + "calibration_controls", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("calibration_id", sa.Integer(), nullable=False), + sa.Column("variant_id", sa.Integer(), nullable=False), + sa.Column( + "clinical_status", + sa.Enum("pathogenic", "benign", name="calibrationcontrolstatus", native_enum=False, length=32), + nullable=False, + ), + sa.Column("created_by_id", sa.Integer(), nullable=False), + sa.Column("modified_by_id", sa.Integer(), nullable=False), + sa.Column("creation_date", sa.Date(), nullable=False), + sa.Column("modification_date", sa.Date(), nullable=False), + sa.ForeignKeyConstraint(["calibration_id"], ["score_calibrations.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["variant_id"], ["variants.id"]), + sa.ForeignKeyConstraint(["created_by_id"], ["users.id"]), + sa.ForeignKeyConstraint(["modified_by_id"], ["users.id"]), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("calibration_id", "variant_id", name="uq_calibration_controls_calibration_id_variant_id"), + ) + op.create_index( + op.f("ix_calibration_controls_calibration_id"), "calibration_controls", ["calibration_id"], unique=False + ) + op.create_index(op.f("ix_calibration_controls_variant_id"), "calibration_controls", ["variant_id"], unique=False) + op.create_index( + op.f("ix_calibration_controls_created_by_id"), "calibration_controls", ["created_by_id"], unique=False + ) + op.create_index( + op.f("ix_calibration_controls_modified_by_id"), "calibration_controls", ["modified_by_id"], unique=False + ) + + op.add_column("score_calibrations", sa.Column("disease", sa.String(), nullable=True)) + op.add_column("score_calibrations", sa.Column("controls_not_phi", sa.Boolean(), nullable=True)) + + +def downgrade(): + op.drop_column("score_calibrations", "controls_not_phi") + op.drop_column("score_calibrations", "disease") + + op.drop_index(op.f("ix_calibration_controls_modified_by_id"), table_name="calibration_controls") + op.drop_index(op.f("ix_calibration_controls_created_by_id"), table_name="calibration_controls") + op.drop_index(op.f("ix_calibration_controls_variant_id"), table_name="calibration_controls") + op.drop_index(op.f("ix_calibration_controls_calibration_id"), table_name="calibration_controls") + op.drop_table("calibration_controls") diff --git a/src/mavedb/models/__init__.py b/src/mavedb/models/__init__.py index 2f0d65b48..ba5334e99 100644 --- a/src/mavedb/models/__init__.py +++ b/src/mavedb/models/__init__.py @@ -1,6 +1,7 @@ __all__ = [ "access_key", "acmg_classification", + "calibration_control", "collection", "clinical_control", "controlled_keyword", diff --git a/src/mavedb/models/calibration_control.py b/src/mavedb/models/calibration_control.py new file mode 100644 index 000000000..363756616 --- /dev/null +++ b/src/mavedb/models/calibration_control.py @@ -0,0 +1,77 @@ +"""SQLAlchemy model for calibration controls. + +A *calibration control* is a variant whose clinical significance is independently +known (from a source outside MaveDB) and which was used as empirical ground truth +when deriving a calibration's score thresholds. Storing these controls lets a +clinician audit the evidence a calibration rests on before trusting its thresholds +for interpretation. +""" + +from __future__ import annotations + +from datetime import date +from typing import TYPE_CHECKING + +from sqlalchemy import Column, Date, Enum, ForeignKey, Integer, UniqueConstraint +from sqlalchemy.orm import Mapped, relationship + +from mavedb.db.base import Base +from mavedb.models.enums.calibration_control_status import CalibrationControlStatus + +if TYPE_CHECKING: + from mavedb.models.score_calibration import ScoreCalibration + from mavedb.models.user import User + from mavedb.models.variant import Variant + + +class CalibrationControl(Base): + """A variant with independently known clinical significance, used as ground truth for a calibration. + + Distinct from ``ScoreCalibrationFunctionalClassification`` and its ``variants`` + relationship, which record *bin membership* — the range a variant's functional score + happens to fall into. A ``CalibrationControl`` instead records *clinical ground truth*: + "this variant's pathogenicity is known from external evidence and was used to anchor + the calibration's thresholds." A control variant will usually also land in a bin, since + its score should agree with its known status, but the two relationships answer different + questions and are kept separate. + + Controls are constrained to variants in the calibration's own score set — a control + without a functional score is meaningless as calibration evidence. That constraint is + enforced at the API layer rather than in the schema (see issue #751). + """ + + __tablename__ = "calibration_controls" + __table_args__ = ( + # A variant may appear at most once per calibration and cannot be double counted. + UniqueConstraint("calibration_id", "variant_id", name="uq_calibration_controls_calibration_id_variant_id"), + ) + + id = Column(Integer, primary_key=True) + + # Deleting a calibration deletes its controls: a control has no meaning apart from the calibration it anchors. + # The DB-level cascade backstops the ORM ``delete-orphan`` cascade declared on ``ScoreCalibration.controls``. + calibration_id = Column( + Integer, ForeignKey("score_calibrations.id", ondelete="CASCADE"), nullable=False, index=True + ) + calibration: Mapped["ScoreCalibration"] = relationship("ScoreCalibration", back_populates="controls") + + variant_id = Column(Integer, ForeignKey("variants.id"), nullable=False, index=True) + variant: Mapped["Variant"] = relationship("Variant") + + clinical_status = Column( + Enum(CalibrationControlStatus, native_enum=False, validate_strings=True, length=32), + nullable=False, + ) + + created_by_id = Column(Integer, ForeignKey("users.id"), index=True, nullable=False) + created_by: Mapped["User"] = relationship("User", foreign_keys="CalibrationControl.created_by_id") + modified_by_id = Column(Integer, ForeignKey("users.id"), index=True, nullable=False) + modified_by: Mapped["User"] = relationship("User", foreign_keys="CalibrationControl.modified_by_id") + creation_date = Column(Date, nullable=False, default=date.today) + modification_date = Column(Date, nullable=False, default=date.today, onupdate=date.today) + + def __repr__(self) -> str: # pragma: no cover - repr utility + return ( + f"" + ) diff --git a/src/mavedb/models/enums/calibration_control_status.py b/src/mavedb/models/enums/calibration_control_status.py new file mode 100644 index 000000000..95c2fbb6e --- /dev/null +++ b/src/mavedb/models/enums/calibration_control_status.py @@ -0,0 +1,14 @@ +import enum + + +class CalibrationControlStatus(str, enum.Enum): + """Clinical significance of a calibration control variant. + + Deliberately restricted to the two-tier ACMG poles used to anchor a calibration's + thresholds. Intermediate tiers (VUS, likely pathogenic, likely benign) are excluded + by design: a control's value as empirical ground truth comes from a confident, binary + clinical call, not from a graded one. + """ + + pathogenic = "pathogenic" + benign = "benign" diff --git a/src/mavedb/models/score_calibration.py b/src/mavedb/models/score_calibration.py index 38ce1f286..08f4b3a0f 100644 --- a/src/mavedb/models/score_calibration.py +++ b/src/mavedb/models/score_calibration.py @@ -16,6 +16,7 @@ from mavedb.models.score_calibration_publication_identifier import ScoreCalibrationPublicationIdentifierAssociation if TYPE_CHECKING: + from mavedb.models.calibration_control import CalibrationControl from mavedb.models.publication_identifier import PublicationIdentifier from mavedb.models.score_set import ScoreSet from mavedb.models.user import User @@ -41,12 +42,31 @@ class ScoreCalibration(Base): baseline_score = Column(Float, nullable=True) baseline_score_description = Column(String, nullable=True) + # Optional free-text disease/disorder context for the controls, at the calibration level. Deliberately not a + # controlled vocabulary and never recorded per-control: a per-control disease label could let a phenotype be + # inferred for an individual variant, which risks PHI. + disease = Column(String, nullable=True) + + # Submitter's affirmation that the control data contains no PHI. Tristate on purpose: # None = not yet addressed, + # False = explicitly declined, True = affirmed. Publishing a calibration that has controls is gated on this being + # True (see #752). + controls_not_phi = Column(Boolean, nullable=True) + functional_classifications: Mapped[list["ScoreCalibrationFunctionalClassification"]] = relationship( "ScoreCalibrationFunctionalClassification", back_populates="calibration", cascade="all, delete-orphan", ) + # Ground-truth controls anchoring this calibration's thresholds. Separate from the per-bin ``variants`` on each + # functional classification, which record score-range membership rather than known clinical significance + # (see ``CalibrationControl``). + controls: Mapped[list["CalibrationControl"]] = relationship( + "CalibrationControl", + back_populates="calibration", + cascade="all, delete-orphan", + ) + publication_identifier_associations: Mapped[list[ScoreCalibrationPublicationIdentifierAssociation]] = relationship( "ScoreCalibrationPublicationIdentifierAssociation", back_populates="score_calibration", diff --git a/src/mavedb/models/score_calibration_functional_classification.py b/src/mavedb/models/score_calibration_functional_classification.py index b9cdabc49..9adc46a6e 100644 --- a/src/mavedb/models/score_calibration_functional_classification.py +++ b/src/mavedb/models/score_calibration_functional_classification.py @@ -22,6 +22,15 @@ class ScoreCalibrationFunctionalClassification(Base): + """A labeled functional-score range (or class) within a calibration, plus the variants that fall in it. + + The ``variants`` relationship records *bin membership* — which variants' functional scores land + in this range. That is distinct from ``CalibrationControl`` (see ``models/calibration_control.py``), + which records variants with independently known clinical significance used as ground truth for the + calibration. A variant can be both, but the two are stored separately because they answer different + questions: "where did this score land?" versus "what is this variant's known clinical status?". + """ + __tablename__ = "score_calibration_functional_classifications" id = Column(Integer, primary_key=True) diff --git a/tests/lib/test_calibration_control.py b/tests/lib/test_calibration_control.py new file mode 100644 index 000000000..2296ae0c2 --- /dev/null +++ b/tests/lib/test_calibration_control.py @@ -0,0 +1,108 @@ +# ruff: noqa: E402 + +import pytest + +pytest.importorskip("psycopg2") + +from sqlalchemy.exc import IntegrityError + +from mavedb.models.calibration_control import CalibrationControl +from mavedb.models.enums.calibration_control_status import CalibrationControlStatus +from mavedb.models.score_calibration import ScoreCalibration +from mavedb.models.user import User +from tests.helpers.constants import TEST_USER + + +def _make_calibration(session, score_set_id: int, user: User) -> ScoreCalibration: + """Persist a bare calibration on the given score set; controls need a parent calibration.""" + calibration = ScoreCalibration( + title="Calibration with controls", + score_set_id=score_set_id, + created_by=user, + modified_by=user, + ) + session.add(calibration) + session.commit() + session.refresh(calibration) + return calibration + + +def test_calibration_control_persists_and_links(session, setup_lib_db_with_variant): + variant = setup_lib_db_with_variant + user = session.query(User).filter(User.username == TEST_USER["username"]).first() + calibration = _make_calibration(session, variant.score_set_id, user) + + control = CalibrationControl( + calibration=calibration, + variant=variant, + clinical_status=CalibrationControlStatus.pathogenic, + created_by=user, + modified_by=user, + ) + session.add(control) + session.commit() + session.refresh(control) + + assert control.id is not None + # Enum value round-trips through the varchar column. + assert control.clinical_status is CalibrationControlStatus.pathogenic + assert control.creation_date is not None + # Relationship resolves both directions via back_populates. + assert control.calibration is calibration + assert control.variant is variant + assert control in calibration.controls + + +def test_calibration_control_unique_per_calibration_and_variant(session, setup_lib_db_with_variant): + variant = setup_lib_db_with_variant + user = session.query(User).filter(User.username == TEST_USER["username"]).first() + calibration = _make_calibration(session, variant.score_set_id, user) + + session.add( + CalibrationControl( + calibration=calibration, + variant=variant, + clinical_status=CalibrationControlStatus.pathogenic, + created_by=user, + modified_by=user, + ) + ) + session.commit() + + # A second control for the same (calibration, variant) pair violates the unique constraint, + # even with a different clinical status — a variant is one piece of evidence per calibration. + session.add( + CalibrationControl( + calibration=calibration, + variant=variant, + clinical_status=CalibrationControlStatus.benign, + created_by=user, + modified_by=user, + ) + ) + with pytest.raises(IntegrityError): + session.commit() + session.rollback() + + +def test_deleting_calibration_cascades_to_controls(session, setup_lib_db_with_variant): + variant = setup_lib_db_with_variant + user = session.query(User).filter(User.username == TEST_USER["username"]).first() + calibration = _make_calibration(session, variant.score_set_id, user) + + control = CalibrationControl( + calibration=calibration, + variant=variant, + clinical_status=CalibrationControlStatus.benign, + created_by=user, + modified_by=user, + ) + session.add(control) + session.commit() + control_id = control.id + + # A control has no meaning apart from its calibration, so deleting the calibration removes it. + session.delete(calibration) + session.commit() + + assert session.get(CalibrationControl, control_id) is None From 1b6b8770b8fa80358f74b1c035898bcb0d0f06a2 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 16 Sep 2026 09:50:23 -0700 Subject: [PATCH 145/252] feat(calibration-controls): add view models for controls (#750) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CalibrationControlBase/Modify/Create/SavedCalibrationControl in a new view_models/calibration_control.py. SavedCalibrationControl derives variant_urn from the variant relationship (mirrors ScoreCalibrationWithScoreSetUrn) so internal PKs never leak, and carries record_type + audit fields. Extend the calibration view models with disease, controls_not_phi, and controls: write models take CalibrationControlCreate; the saved response returns SavedCalibrationControl (empty list by default). Exclude the controls relationship from the model_dump splat in _create_score_calibration and modify_score_calibration — passing controls=None to the ORM relationship raised "None is not list-like" and 500'd every create/modify. disease and controls_not_phi are real columns and flow through. Controls persistence and the CSV path remain #753. --- src/mavedb/lib/score_calibrations.py | 2 + src/mavedb/view_models/calibration_control.py | 86 ++++++++++ src/mavedb/view_models/score_calibration.py | 10 ++ tests/view_models/test_calibration_control.py | 147 ++++++++++++++++++ 4 files changed, 245 insertions(+) create mode 100644 src/mavedb/view_models/calibration_control.py create mode 100644 tests/view_models/test_calibration_control.py diff --git a/src/mavedb/lib/score_calibrations.py b/src/mavedb/lib/score_calibrations.py index 11e1b2e88..ad778e430 100644 --- a/src/mavedb/lib/score_calibrations.py +++ b/src/mavedb/lib/score_calibrations.py @@ -181,6 +181,7 @@ async def _create_score_calibration( by_alias=False, exclude={ "functional_classifications", + "controls", "threshold_sources", "evidence_sources", "method_sources", @@ -459,6 +460,7 @@ async def modify_score_calibration( for attr, value in calibration_update.model_dump().items(): if attr not in { "functional_classifications", + "controls", "threshold_sources", "evidence_sources", "method_sources", diff --git a/src/mavedb/view_models/calibration_control.py b/src/mavedb/view_models/calibration_control.py new file mode 100644 index 000000000..b6f48e020 --- /dev/null +++ b/src/mavedb/view_models/calibration_control.py @@ -0,0 +1,86 @@ +"""Pydantic view models for calibration controls. + +A calibration control pairs a MaveDB variant with its independently known clinical +significance, serving as empirical ground truth when a calibration's score thresholds +are derived. These models are the API-layer representation used when controls are +created, updated, and served through the calibration endpoints. +""" + +from datetime import date +from typing import Any + +from pydantic import model_validator + +from mavedb.lib.validation.exceptions import ValidationError +from mavedb.models.enums.calibration_control_status import CalibrationControlStatus +from mavedb.view_models import record_type_validator, set_record_type +from mavedb.view_models.base.base import BaseModel +from mavedb.view_models.user import SavedUser + + +class CalibrationControlBase(BaseModel): + """Fields shared by every calibration control view model. + + ``variant_urn`` is the external identifier for the control's variant; the internal + primary key is never exposed. + """ + + variant_urn: str + clinical_status: CalibrationControlStatus + + +class CalibrationControlModify(CalibrationControlBase): + """Model used to modify an existing calibration control. + + Carries no additional fields — only the base fields can be updated. + """ + + pass + + +class CalibrationControlCreate(CalibrationControlModify): + """Model used to create a calibration control. + + Carries no additional fields — only the modifiable fields are required for creation. + """ + + pass + + +class SavedCalibrationControl(CalibrationControlBase): + """Persisted calibration control, including identifier and audit metadata.""" + + record_type: str = None # type: ignore + _record_type_factory = record_type_validator()(set_record_type) + + id: int + creation_date: date + modification_date: date + created_by: SavedUser + modified_by: SavedUser + + class Config: + """Pydantic configuration (ORM mode).""" + + from_attributes = True + + @model_validator(mode="before") + def generate_variant_urn(cls, data: Any): + """Expose the control's variant as ``variant_urn`` when building from an ORM object. + + The ORM row references a ``Variant`` relationship, not a bare URN. Mirrors the + synthetic-field pattern used by ``ScoreCalibrationWithScoreSetUrn``. + """ + if hasattr(data, "variant"): + try: + data.__setattr__("variant_urn", data.variant.urn) + except (AttributeError, KeyError) as exc: + raise ValidationError(f"Unable to coerce variant urn for {cls.__name__}: {exc}.") # type: ignore + + return data + + +class CalibrationControl(SavedCalibrationControl): + """Calibration control with its associated calibration and score set URNs.""" + + pass diff --git a/src/mavedb/view_models/score_calibration.py b/src/mavedb/view_models/score_calibration.py index 857d4e49f..87cde6f5a 100644 --- a/src/mavedb/view_models/score_calibration.py +++ b/src/mavedb/view_models/score_calibration.py @@ -26,6 +26,10 @@ SavedACMGClassification, ) from mavedb.view_models.base.base import BaseModel +from mavedb.view_models.calibration_control import ( + CalibrationControlCreate, + SavedCalibrationControl, +) from mavedb.view_models.publication_identifier import ( PublicationIdentifier, PublicationIdentifierBase, @@ -282,6 +286,9 @@ class ScoreCalibrationBase(BaseModel): baseline_score_description: Optional[str] = None notes: Optional[str] = None + disease: Optional[str] = None + controls_not_phi: Optional[bool] = None + functional_classifications: Optional[Sequence[FunctionalClassificationBase]] = None threshold_sources: Sequence[PublicationIdentifierBase] evidence_sources: Sequence[PublicationIdentifierBase] @@ -432,6 +439,8 @@ class ScoreCalibrationModify(ScoreCalibrationBase): score_set_urn: Optional[str] = None functional_classifications: Optional[Sequence[FunctionalClassificationModify]] = None + # None means "no change" on modify; an empty list clears all controls. + controls: Optional[Sequence[CalibrationControlCreate]] = None threshold_sources: Sequence[PublicationIdentifierCreate] evidence_sources: Sequence[PublicationIdentifierCreate] method_sources: Sequence[PublicationIdentifierCreate] @@ -498,6 +507,7 @@ class SavedScoreCalibration(ScoreCalibrationBase): private: bool = True functional_classifications: Optional[Sequence[SavedFunctionalClassification]] = None + controls: Sequence[SavedCalibrationControl] = [] threshold_sources: Sequence[SavedPublicationIdentifier] evidence_sources: Sequence[SavedPublicationIdentifier] method_sources: Sequence[SavedPublicationIdentifier] diff --git a/tests/view_models/test_calibration_control.py b/tests/view_models/test_calibration_control.py new file mode 100644 index 000000000..8b61d5120 --- /dev/null +++ b/tests/view_models/test_calibration_control.py @@ -0,0 +1,147 @@ +from copy import deepcopy +from datetime import date + +import pytest +from pydantic import ValidationError + +from mavedb.models.enums.calibration_control_status import CalibrationControlStatus +from mavedb.view_models.calibration_control import ( + CalibrationControlCreate, + SavedCalibrationControl, +) +from mavedb.view_models.score_calibration import ScoreCalibration, ScoreCalibrationCreate +from tests.helpers.constants import ( + TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED, + TEST_SAVED_BRNICH_SCORE_CALIBRATION_RANGE_BASED, + TEST_USER, +) +from tests.helpers.util.common import dummy_attributed_object_from_dict + +TEST_CONTROL_VARIANT_URN = "urn:mavedb:00000001-a-1#1" + + +def _saved_user_like(): + """An attributed object standing in for a persisted User (audit field).""" + return dummy_attributed_object_from_dict({"username": TEST_USER["username"]}) + + +def _control_attributed_object(status=CalibrationControlStatus.pathogenic): + """An attributed object standing in for a persisted CalibrationControl ORM row.""" + return dummy_attributed_object_from_dict( + { + "id": 1, + "variant": dummy_attributed_object_from_dict({"urn": TEST_CONTROL_VARIANT_URN}), + "clinical_status": status, + "creation_date": date(2026, 9, 16), + "modification_date": date(2026, 9, 16), + "created_by": _saved_user_like(), + "modified_by": _saved_user_like(), + } + ) + + +############################################################################## +# CalibrationControl view models from dicts (request bodies) +############################################################################## + + +def test_calibration_control_create_from_dict(): + control = CalibrationControlCreate(**{"variantUrn": TEST_CONTROL_VARIANT_URN, "clinicalStatus": "pathogenic"}) + + assert control.variant_urn == TEST_CONTROL_VARIANT_URN + assert control.clinical_status is CalibrationControlStatus.pathogenic + assert control.model_dump(by_alias=True)["variantUrn"] == TEST_CONTROL_VARIANT_URN + + +@pytest.mark.parametrize("status", ["pathogenic", "benign"]) +def test_calibration_control_create_accepts_both_statuses(status): + control = CalibrationControlCreate(variant_urn=TEST_CONTROL_VARIANT_URN, clinical_status=status) + assert control.clinical_status is CalibrationControlStatus(status) + + +def test_calibration_control_create_rejects_unknown_status(): + with pytest.raises(ValidationError): + CalibrationControlCreate(variant_urn=TEST_CONTROL_VARIANT_URN, clinical_status="likely_pathogenic") + + +def test_calibration_control_create_requires_variant_urn(): + with pytest.raises(ValidationError): + CalibrationControlCreate(clinical_status="benign") + + +############################################################################## +# SavedCalibrationControl from attributed objects (ORM models) +############################################################################## + + +def test_saved_calibration_control_synthesizes_variant_urn_from_orm(): + saved = SavedCalibrationControl.model_validate(_control_attributed_object()) + + # variant_urn is not a column on the ORM row; it is derived from the variant relationship. + assert saved.variant_urn == TEST_CONTROL_VARIANT_URN + assert saved.clinical_status is CalibrationControlStatus.pathogenic + assert saved.id == 1 + assert saved.created_by.username == TEST_USER["username"] + assert saved.modified_by.username == TEST_USER["username"] + assert saved.record_type == "SavedCalibrationControl" + + dumped = saved.model_dump(by_alias=True) + assert dumped["variantUrn"] == TEST_CONTROL_VARIANT_URN + assert dumped["recordType"] == "SavedCalibrationControl" + + +############################################################################## +# ScoreCalibration view models extended with controls / disease / controls_not_phi +############################################################################## + + +def test_score_calibration_create_accepts_controls_disease_and_phi(): + payload = deepcopy(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) + payload["disease"] = "Brugada syndrome" + payload["controls_not_phi"] = True + payload["controls"] = [ + {"variant_urn": TEST_CONTROL_VARIANT_URN, "clinical_status": "pathogenic"}, + {"variant_urn": "urn:mavedb:00000001-a-1#2", "clinical_status": "benign"}, + ] + + calibration = ScoreCalibrationCreate.model_validate(payload) + + assert calibration.disease == "Brugada syndrome" + assert calibration.controls_not_phi is True + assert len(calibration.controls) == 2 + assert calibration.controls[0].clinical_status is CalibrationControlStatus.pathogenic + assert calibration.controls[1].clinical_status is CalibrationControlStatus.benign + + +def test_saved_score_calibration_includes_controls(): + saved = deepcopy(TEST_SAVED_BRNICH_SCORE_CALIBRATION_RANGE_BASED) + saved["disease"] = "Brugada syndrome" + saved["controlsNotPhi"] = True + saved["controls"] = [ + { + "id": 1, + "variantUrn": TEST_CONTROL_VARIANT_URN, + "clinicalStatus": "pathogenic", + "creationDate": date(2026, 9, 16), + "modificationDate": date(2026, 9, 16), + "createdBy": {"orcidId": TEST_USER["username"]}, + "modifiedBy": {"orcidId": TEST_USER["username"]}, + } + ] + + calibration = ScoreCalibration.model_validate(dummy_attributed_object_from_dict(saved)) + + assert calibration.disease == "Brugada syndrome" + assert calibration.controls_not_phi is True + assert len(calibration.controls) == 1 + assert calibration.controls[0].variant_urn == TEST_CONTROL_VARIANT_URN + assert calibration.controls[0].clinical_status is CalibrationControlStatus.pathogenic + + +def test_saved_score_calibration_controls_default_to_empty_list(): + saved = deepcopy(TEST_SAVED_BRNICH_SCORE_CALIBRATION_RANGE_BASED) + saved.pop("controls", None) + + calibration = ScoreCalibration.model_validate(dummy_attributed_object_from_dict(saved)) + + assert calibration.controls == [] From 7aa7e744b6c0432eded54b001624f60042f5eb27 Mon Sep 17 00:00:00 2001 From: David Reinhart Date: Wed, 16 Sep 2026 14:53:11 -0700 Subject: [PATCH 146/252] test(urns): cover every entity type publication renames forwarded_path was only ever exercised against a score set target, so the experiment and experiment set branches df88be34 introduced in _target_is_public had no coverage at this level. The router tests reached them end to end, which is a slower signal and does not isolate the lookup. Drive the fixture and the parametrized tests from one ENTITY_TYPES table that names each type once, carrying the URN the fixture gives it, the route its URN sits under, the attribute path reaching it from the score set the fixtures build, and the sub-resource the API serves after its URN. Adding a type is one entry: the fixture iterates the table and every parametrize takes it as its argument source. Retired URNs come from generate_temp_urn rather than a hand-kept list, since a redirect row is unique on the URN it retires. That holds for as long as new types keep arriving through score set publication, which is the only thing that renames a URN today. An entity published by its own workflow would have no attribute path from a score set, so the fixture would need another way to build it. The table comment says so where someone adding an entry will read it. Every route-bearing test now runs for all three types. The sub-resource test covers the two types that have one, since the API serves nothing after an experiment set's URN, and the variant test stays on score sets, since a variant URN is built from a score set's. --- tests/lib/test_urn_redirects.py | 118 +++++++++++++++++++++++++------- 1 file changed, 94 insertions(+), 24 deletions(-) diff --git a/tests/lib/test_urn_redirects.py b/tests/lib/test_urn_redirects.py index aab0b2da5..4c7410aae 100644 --- a/tests/lib/test_urn_redirects.py +++ b/tests/lib/test_urn_redirects.py @@ -12,14 +12,48 @@ pytest.importorskip("arq") pytest.importorskip("cdot") +from mavedb.lib.temp_urns import generate_temp_urn from mavedb.lib.urn_redirects import forwarded_path, record_urn_redirect from mavedb.models.urn_redirect import UrnRedirect -from tests.helpers.constants import VALID_SCORE_SET_URN +from tests.helpers.constants import VALID_EXPERIMENT_SET_URN, VALID_EXPERIMENT_URN, VALID_SCORE_SET_URN RETIRED_URN = "tmp:00000000-0000-4000-8000-000000000001" +LIVE_TMP_URN = "tmp:00000000-0000-4000-8000-000000000002" PUBLISHED_URN = VALID_SCORE_SET_URN +# Every entity type publication renames. Adding an entry extends the fixture and each parametrized +# test below, which are all driven by these keys. +# +# urn the URN setup_lib_db_with_score_set gives this entity +# route the collection its URN sits under in a request path +# score_set_path the attribute path reaching it from that score set, empty for the score set +# itself. Every type here is renamed by score set publication; an entity with +# its own publishing workflow would need the fixture to build it another way. +# sub_resource a path segment the API serves after its URN, or None where it serves none +ENTITY_TYPES = { + "score_set": { + "urn": VALID_SCORE_SET_URN, + "route": "score-sets", + "score_set_path": "", + "sub_resource": "scores", + }, + "experiment": { + "urn": VALID_EXPERIMENT_URN, + "route": "experiments", + "score_set_path": "experiment", + "sub_resource": "score-sets", + }, + "experiment_set": { + "urn": VALID_EXPERIMENT_SET_URN, + "route": "experiment-sets", + "score_set_path": "experiment.experiment_set", + "sub_resource": None, + }, +} + +ENTITY_TYPES_WITH_SUB_RESOURCE = [name for name, entity in ENTITY_TYPES.items() if entity["sub_resource"]] + @pytest.mark.integration class TestRecordUrnRedirect: @@ -48,45 +82,81 @@ def test_ignores_a_rename_that_changes_nothing(self, session): class TestForwardedPath: @pytest.fixture def retired(self, session, setup_lib_db_with_score_set): - """Retire a temporary URN onto a real, public score set. + """Retire a temporary URN onto a public record of every entity type, keyed by type. A real record is needed, not just a row in the table: forwarding withholds a target it cannot confirm is public, so a redirect pointing at nothing forwards nowhere. """ - setup_lib_db_with_score_set.private = False - record_urn_redirect(session, RETIRED_URN, PUBLISHED_URN) - session.commit() + retired = {} + for entity_type, entity in ENTITY_TYPES.items(): + record = setup_lib_db_with_score_set + for attribute in filter(None, entity["score_set_path"].split(".")): + record = getattr(record, attribute) - def test_withholds_a_target_that_is_private(self, session, retired, setup_lib_db_with_score_set): - setup_lib_db_with_score_set.private = True - session.commit() + record.private = False - assert forwarded_path(session, f"/api/v1/score-sets/{RETIRED_URN}") is None + # A redirect row is unique on the URN it retires, so every type needs its own. + retired_urn = generate_temp_urn() + record_urn_redirect(session, retired_urn, record.urn) + retired[entity_type] = (retired_urn, record) - def test_withholds_a_target_that_does_not_exist(self, session): - """A deleted record leaves its redirect row behind.""" - record_urn_redirect(session, RETIRED_URN, PUBLISHED_URN) session.commit() + return retired - assert forwarded_path(session, f"/api/v1/score-sets/{RETIRED_URN}") is None + @pytest.mark.parametrize("entity_type", ENTITY_TYPES) + def test_forwards_a_retired_urn(self, session, retired, entity_type): + retired_urn, record = retired[entity_type] + route = ENTITY_TYPES[entity_type]["route"] - def test_forwards_a_retired_urn(self, session, retired): - assert forwarded_path(session, f"/api/v1/score-sets/{RETIRED_URN}") == f"/api/v1/score-sets/{PUBLISHED_URN}" + assert forwarded_path(session, f"/api/v1/{route}/{retired_urn}") == f"/api/v1/{route}/{record.urn}" + + @pytest.mark.parametrize("entity_type", ENTITY_TYPES_WITH_SUB_RESOURCE) + def test_forwards_a_sub_resource_of_a_retired_urn(self, session, retired, entity_type): + retired_urn, record = retired[entity_type] + route = ENTITY_TYPES[entity_type]["route"] + sub_resource = ENTITY_TYPES[entity_type]["sub_resource"] - def test_forwards_a_sub_resource_of_a_retired_urn(self, session, retired): assert ( - forwarded_path(session, f"/api/v1/score-sets/{RETIRED_URN}/scores") - == f"/api/v1/score-sets/{PUBLISHED_URN}/scores" + forwarded_path(session, f"/api/v1/{route}/{retired_urn}/{sub_resource}") + == f"/api/v1/{route}/{record.urn}/{sub_resource}" ) + @pytest.mark.parametrize("entity_type", ENTITY_TYPES) + def test_withholds_a_target_that_is_private(self, session, retired, entity_type): + retired_urn, record = retired[entity_type] + route = ENTITY_TYPES[entity_type]["route"] + + record.private = True + session.commit() + + assert forwarded_path(session, f"/api/v1/{route}/{retired_urn}") is None + + @pytest.mark.parametrize("entity_type", ENTITY_TYPES) + def test_withholds_a_target_that_does_not_exist(self, session, entity_type): + """A deleted record leaves its redirect row behind. Without the fixture, none of these exist.""" + entity = ENTITY_TYPES[entity_type] + + record_urn_redirect(session, RETIRED_URN, entity["urn"]) + session.commit() + + assert forwarded_path(session, f"/api/v1/{entity['route']}/{RETIRED_URN}") is None + + @pytest.mark.parametrize("entity_type", ENTITY_TYPES) + def test_leaves_a_path_naming_no_temporary_urn_alone(self, session, retired, entity_type): + _, record = retired[entity_type] + route = ENTITY_TYPES[entity_type]["route"] + + assert forwarded_path(session, f"/api/v1/{route}/{record.urn}") is None + def test_forwards_a_variant_of_a_retired_score_set(self, session, retired): """A variant URN is built from its score set's, so the score set's redirect carries it.""" - assert forwarded_path(session, f"/api/v1/variants/{RETIRED_URN}#4") == f"/api/v1/variants/{PUBLISHED_URN}#4" + retired_urn, score_set = retired["score_set"] - def test_leaves_a_path_naming_no_temporary_urn_alone(self, session, retired): - assert forwarded_path(session, f"/api/v1/score-sets/{PUBLISHED_URN}") is None + assert forwarded_path(session, f"/api/v1/variants/{retired_urn}#4") == f"/api/v1/variants/{score_set.urn}#4" - def test_leaves_a_live_temporary_urn_alone(self, session, retired): + @pytest.mark.parametrize("entity_type", ENTITY_TYPES) + def test_leaves_a_live_temporary_urn_alone(self, session, retired, entity_type): """An unpublished record still answers to its temporary URN, and has no row in the table.""" - live_urn = "tmp:00000000-0000-4000-8000-000000000002" - assert forwarded_path(session, f"/api/v1/score-sets/{live_urn}") is None + route = ENTITY_TYPES[entity_type]["route"] + + assert forwarded_path(session, f"/api/v1/{route}/{LIVE_TMP_URN}") is None From 8fde78321cdccd57e2e1502fe3e6bc09447ef905 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 16 Sep 2026 10:59:02 -0700 Subject: [PATCH 147/252] feat(calibration-controls): validate controls belong to the score set (#751) Add validate_calibration_controls_in_score_set to lib/score_calibrations.py: a batched, score-set-scoped lookup that rejects controls whose variant is unknown or belongs to a different score set (reported together, per the combined-error decision), plus a Counter-based duplicate check that fails with a readable message before the UNIQUE(calibration_id, variant_id) constraint. No-op for None/empty. Wiring the call into create/modify lands with the endpoint work in #753. --- src/mavedb/lib/score_calibrations.py | 53 +++++++++++++++++- tests/lib/test_calibration_control.py | 79 ++++++++++++++++++++++++++- 2 files changed, 130 insertions(+), 2 deletions(-) diff --git a/src/mavedb/lib/score_calibrations.py b/src/mavedb/lib/score_calibrations.py index ad778e430..58b5b259f 100644 --- a/src/mavedb/lib/score_calibrations.py +++ b/src/mavedb/lib/score_calibrations.py @@ -1,7 +1,8 @@ """Utilities for building and mutating score calibration ORM objects.""" import math -from typing import Optional, Union +from collections import Counter +from typing import Optional, Sequence, Union import pandas as pd from sqlalchemy import Float, and_, select @@ -16,6 +17,7 @@ hgvs_nt_column, hgvs_pro_column, ) +from mavedb.lib.validation.exceptions import ValidationError from mavedb.lib.validation.utilities import inf_or_float from mavedb.models.enums.score_calibration_relation import ScoreCalibrationRelation from mavedb.models.score_calibration import ScoreCalibration @@ -27,6 +29,55 @@ from mavedb.view_models import score_calibration +def validate_calibration_controls_in_score_set( + db: Session, + score_set: ScoreSet, + controls: Optional[Sequence[score_calibration.CalibrationControlCreate]], +) -> None: + """Ensure every calibration control references a variant in the calibration's score set. + + A control earns its role as calibration evidence from a variant the assay actually scored, + so each control must reference a variant belonging to the calibration's own score set. A + single scoped query resolves all submitted URNs at once; any URN it does not return either + does not exist in MaveDB or belongs to a different score set, and both are rejected together. + + Duplicate variant URNs within one submission are caught here with a readable message rather + than deferred to the ``UNIQUE(calibration_id, variant_id)`` database constraint. + + Args: + db: Database session used for the lookup. + score_set: The score set the calibration belongs to. + controls: Controls submitted on create or modify. ``None`` or empty is a no-op. + + Raises: + ValidationError: If any control URN is duplicated within the submission, absent from + MaveDB, or belongs to a different score set. + """ + if not controls: + return + + submitted_urns = [control.variant_urn for control in controls] + + counts = Counter(submitted_urns) + duplicate_urns = {urn for urn, count in counts.items() if count > 1} + if duplicate_urns: + raise ValidationError( + f"Duplicate control variant URNs detected within the submission: {', '.join(sorted(duplicate_urns))}." + ) + + existing_urns = set( + db.scalars( + select(Variant.urn).where(Variant.score_set_id == score_set.id, Variant.urn.in_(submitted_urns)) + ).all() + ) + missing_urns = set(counts) - existing_urns + if missing_urns: + raise ValidationError( + "The following control variants do not belong to the calibration's score set: " + f"{', '.join(sorted(missing_urns))}." + ) + + def create_functional_classification( db: Session, functional_range_create: Union[ diff --git a/tests/lib/test_calibration_control.py b/tests/lib/test_calibration_control.py index 2296ae0c2..2c9f254fd 100644 --- a/tests/lib/test_calibration_control.py +++ b/tests/lib/test_calibration_control.py @@ -6,11 +6,16 @@ from sqlalchemy.exc import IntegrityError +from mavedb.lib.score_calibrations import validate_calibration_controls_in_score_set +from mavedb.lib.validation.exceptions import ValidationError from mavedb.models.calibration_control import CalibrationControl from mavedb.models.enums.calibration_control_status import CalibrationControlStatus from mavedb.models.score_calibration import ScoreCalibration +from mavedb.models.score_set import ScoreSet from mavedb.models.user import User -from tests.helpers.constants import TEST_USER +from mavedb.models.variant import Variant +from mavedb.view_models.calibration_control import CalibrationControlCreate +from tests.helpers.constants import TEST_LICENSE, TEST_MINIMAL_VARIANT, TEST_SEQ_SCORESET, TEST_USER def _make_calibration(session, score_set_id: int, user: User) -> ScoreCalibration: @@ -106,3 +111,75 @@ def test_deleting_calibration_cascades_to_controls(session, setup_lib_db_with_va session.commit() assert session.get(CalibrationControl, control_id) is None + + +############################################################################## +# validate_calibration_controls_in_score_set (#751) +############################################################################## + + +def _variant_in_other_score_set(session, reference_score_set, user) -> Variant: + """Create a second score set (in the same experiment) with one variant, for cross-set tests.""" + scaffold = TEST_SEQ_SCORESET.copy() + scaffold.pop("target_genes") + other_score_set = ScoreSet( + **scaffold, + urn="urn:mavedb:00000002-a-1", + experiment_id=reference_score_set.experiment_id, + licence_id=TEST_LICENSE["id"], + ) + other_score_set.created_by = user + other_score_set.modified_by = user + session.add(other_score_set) + session.commit() + session.refresh(other_score_set) + + variant = Variant(**TEST_MINIMAL_VARIANT, urn=f"{other_score_set.urn}#1", score_set_id=other_score_set.id) + session.add(variant) + session.commit() + session.refresh(variant) + return variant + + +@pytest.mark.parametrize("controls", [None, []]) +def test_validate_controls_is_a_no_op_when_absent(session, setup_lib_db_with_variant, controls): + variant = setup_lib_db_with_variant + # Must not raise for None or an empty list. + validate_calibration_controls_in_score_set(session, variant.score_set, controls) + + +def test_validate_controls_accepts_variant_in_score_set(session, setup_lib_db_with_variant): + variant = setup_lib_db_with_variant + controls = [CalibrationControlCreate(variant_urn=variant.urn, clinical_status="pathogenic")] + + validate_calibration_controls_in_score_set(session, variant.score_set, controls) + + +def test_validate_controls_rejects_nonexistent_variant(session, setup_lib_db_with_variant): + variant = setup_lib_db_with_variant + controls = [CalibrationControlCreate(variant_urn="urn:mavedb:99999999-x-9#1", clinical_status="benign")] + + with pytest.raises(ValidationError, match="do not belong to the calibration's score set"): + validate_calibration_controls_in_score_set(session, variant.score_set, controls) + + +def test_validate_controls_rejects_variant_from_another_score_set(session, setup_lib_db_with_variant): + variant = setup_lib_db_with_variant + user = session.query(User).filter(User.username == TEST_USER["username"]).first() + other_variant = _variant_in_other_score_set(session, variant.score_set, user) + + controls = [CalibrationControlCreate(variant_urn=other_variant.urn, clinical_status="pathogenic")] + + with pytest.raises(ValidationError, match="do not belong to the calibration's score set"): + validate_calibration_controls_in_score_set(session, variant.score_set, controls) + + +def test_validate_controls_rejects_duplicate_variant_urns(session, setup_lib_db_with_variant): + variant = setup_lib_db_with_variant + controls = [ + CalibrationControlCreate(variant_urn=variant.urn, clinical_status="pathogenic"), + CalibrationControlCreate(variant_urn=variant.urn, clinical_status="benign"), + ] + + with pytest.raises(ValidationError, match="Duplicate control variant URNs detected"): + validate_calibration_controls_in_score_set(session, variant.score_set, controls) From cefdc2607160dec06a9a7c5cf94aabb60352a597 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 16 Sep 2026 11:21:23 -0700 Subject: [PATCH 148/252] feat(calibration-controls): gate publishing on PHI acknowledgment (#752) Block the private->public transition in publish_score_calibration_route when a calibration has controls but controls_not_phi is not True (None or False both block; only True clears). Zero-control calibrations are unaffected. The check lives in the router to keep HTTP semantics out of the lib layer. Tests attach controls at the ORM level since endpoint persistence is #753. --- src/mavedb/routers/score_calibrations.py | 25 ++++- tests/routers/test_calibration_controls.py | 111 +++++++++++++++++++++ 2 files changed, 134 insertions(+), 2 deletions(-) create mode 100644 tests/routers/test_calibration_controls.py diff --git a/src/mavedb/routers/score_calibrations.py b/src/mavedb/routers/score_calibrations.py index e8b31478c..6e876c29d 100644 --- a/src/mavedb/routers/score_calibrations.py +++ b/src/mavedb/routers/score_calibrations.py @@ -569,7 +569,11 @@ async def promote_score_calibration_to_primary_route( Promote a score calibration to be the primary calibration for its associated score set. """ save_to_logging_context( - {"requested_resource": urn, "resource_property": "primary", "demote_existing_primary": demote_existing_primary} + { + "requested_resource": urn, + "resource_property": "primary", + "demote_existing_primary": demote_existing_primary, + } ) item = ( @@ -691,6 +695,19 @@ def publish_score_calibration_route( logger.debug("The requested score calibration is already public", extra=logging_context()) return item + # Control data must be affirmed free of PHI before it can be made public. The affirmation is only + # meaningful when controls exist, so zero-control calibrations publish regardless. controls_not_phi + # is tristate: only True clears the gate — None (unaddressed) and False (declined) both block. + if item.controls and item.controls_not_phi is not True: + logger.debug("Calibration controls have not been affirmed free of PHI", extra=logging_context()) + raise HTTPException( + status_code=422, + detail=( + "This calibration has controls that have not been affirmed to be free of protected health " + "information (PHI). Set controls_not_phi to true before publishing." + ), + ) + # XXX: desired? # if item.score_set.private: # logger.debug( @@ -729,7 +746,11 @@ def get_functional_classification_variants( a `variant_count` summary for performance. """ save_to_logging_context( - {"requested_resource": urn, "requested_classification": classification_id, "resource_property": "variants"} + { + "requested_resource": urn, + "requested_classification": classification_id, + "resource_property": "variants", + } ) calibration = ( diff --git a/tests/routers/test_calibration_controls.py b/tests/routers/test_calibration_controls.py new file mode 100644 index 000000000..eeab1bba1 --- /dev/null +++ b/tests/routers/test_calibration_controls.py @@ -0,0 +1,111 @@ +# ruff: noqa: E402 + +import pytest + +arq = pytest.importorskip("arq") +cdot = pytest.importorskip("cdot") +fastapi = pytest.importorskip("fastapi") + +from sqlalchemy import select + +from mavedb.models.calibration_control import CalibrationControl +from mavedb.models.enums.calibration_control_status import CalibrationControlStatus +from mavedb.models.score_calibration import ScoreCalibration as CalibrationDbModel +from mavedb.models.variant import Variant +from tests.helpers.constants import ( + TEST_BIORXIV_IDENTIFIER, + TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED, + TEST_PUBMED_IDENTIFIER, +) +from tests.helpers.util.common import deepcamelize +from tests.helpers.util.experiment import create_experiment +from tests.helpers.util.score_calibration import create_test_score_calibration_in_score_set_via_client +from tests.helpers.util.score_set import create_seq_score_set_with_mapped_variants + +CALIBRATION_PUBLICATIONS = [ + {"dbName": "PubMed", "identifier": TEST_PUBMED_IDENTIFIER}, + {"dbName": "bioRxiv", "identifier": TEST_BIORXIV_IDENTIFIER}, +] + + +def _create_private_calibration(client, session, data_provider, data_files): + """Create a score set with variants and a private range-based calibration; return the calibration dict.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, + session, + data_provider, + experiment["urn"], + data_files / "scores.csv", + ) + return create_test_score_calibration_in_score_set_via_client( + client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) + ) + + +def _attach_control(session, calibration_urn, controls_not_phi): + """Attach one control to a calibration at the ORM level (endpoint wiring is #753) and set the PHI flag.""" + calibration = session.query(CalibrationDbModel).where(CalibrationDbModel.urn == calibration_urn).one() + variant = session.scalars(select(Variant).where(Variant.score_set_id == calibration.score_set_id)).first() + + session.add( + CalibrationControl( + calibration=calibration, + variant=variant, + clinical_status=CalibrationControlStatus.pathogenic, + created_by=calibration.created_by, + modified_by=calibration.created_by, + ) + ) + calibration.controls_not_phi = controls_not_phi + session.add(calibration) + session.commit() + + +@pytest.mark.parametrize("mock_publication_fetch", [CALIBRATION_PUBLICATIONS], indirect=["mock_publication_fetch"]) +@pytest.mark.parametrize("controls_not_phi", [None, False]) +def test_cannot_publish_calibration_with_controls_when_phi_not_affirmed( + client, setup_router_db, mock_publication_fetch, session, data_provider, data_files, controls_not_phi +): + calibration = _create_private_calibration(client, session, data_provider, data_files) + _attach_control(session, calibration["urn"], controls_not_phi=controls_not_phi) + + response = client.post(f"/api/v1/score-calibrations/{calibration['urn']}/publish") + + assert response.status_code == 422 + assert "protected health information" in response.json()["detail"].lower() + + # The gate must not have published the calibration. + refreshed = session.query(CalibrationDbModel).where(CalibrationDbModel.urn == calibration["urn"]).one() + assert refreshed.private is True + + +@pytest.mark.parametrize("mock_publication_fetch", [CALIBRATION_PUBLICATIONS], indirect=["mock_publication_fetch"]) +def test_can_publish_calibration_with_controls_when_phi_affirmed( + client, setup_router_db, mock_publication_fetch, session, data_provider, data_files +): + calibration = _create_private_calibration(client, session, data_provider, data_files) + _attach_control(session, calibration["urn"], controls_not_phi=True) + + response = client.post(f"/api/v1/score-calibrations/{calibration['urn']}/publish") + + assert response.status_code == 200 + assert response.json()["private"] is False + + +@pytest.mark.parametrize("mock_publication_fetch", [CALIBRATION_PUBLICATIONS], indirect=["mock_publication_fetch"]) +def test_can_publish_calibration_without_controls_regardless_of_phi_flag( + client, setup_router_db, mock_publication_fetch, session, data_provider, data_files +): + calibration = _create_private_calibration(client, session, data_provider, data_files) + + # No controls attached, but an explicit (non-True) PHI flag must not block publishing. + item = session.query(CalibrationDbModel).where(CalibrationDbModel.urn == calibration["urn"]).one() + item.controls_not_phi = False + session.add(item) + session.commit() + + response = client.post(f"/api/v1/score-calibrations/{calibration['urn']}/publish") + + assert response.status_code == 200 + assert response.json()["private"] is False From b3c1801b23fddc3a6d3f11d41b13d639c1e2969b Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 16 Sep 2026 12:55:58 -0700 Subject: [PATCH 149/252] feat(calibration-controls): persist controls on create and modify (#753) Add build_calibration_controls (validate via validate_calibration_controls_in_score_set, then resolve URNs to variants and construct CalibrationControl rows) and wire it into create_score_calibration_in_score_set and modify_score_calibration. Modify uses replace semantics: controls=None leaves existing controls untouched, a list (including []) deletes and rebuilds them. validate_calibration_controls_in_score_set now returns the validated variant URNs so build reuses them instead of rebuilding the list. Controls are read via getattr on the update/create models so the existing minimal test mocks (which predate the field) still work. CSV upload, the controls_not_phi reset-on-change, and controlsCount on list responses are the remaining slices. --- src/mavedb/lib/score_calibrations.py | 74 +++++++++++++++++++++++++-- tests/lib/test_calibration_control.py | 45 ++++++++++++++-- 2 files changed, 111 insertions(+), 8 deletions(-) diff --git a/src/mavedb/lib/score_calibrations.py b/src/mavedb/lib/score_calibrations.py index 58b5b259f..cd36b12e3 100644 --- a/src/mavedb/lib/score_calibrations.py +++ b/src/mavedb/lib/score_calibrations.py @@ -19,6 +19,7 @@ ) from mavedb.lib.validation.exceptions import ValidationError from mavedb.lib.validation.utilities import inf_or_float +from mavedb.models.calibration_control import CalibrationControl from mavedb.models.enums.score_calibration_relation import ScoreCalibrationRelation from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification @@ -33,8 +34,8 @@ def validate_calibration_controls_in_score_set( db: Session, score_set: ScoreSet, controls: Optional[Sequence[score_calibration.CalibrationControlCreate]], -) -> None: - """Ensure every calibration control references a variant in the calibration's score set. +) -> list[str]: + """Validate calibration controls against a score set and return their variant URNs. A control earns its role as calibration evidence from a variant the assay actually scored, so each control must reference a variant belonging to the calibration's own score set. A @@ -44,17 +45,23 @@ def validate_calibration_controls_in_score_set( Duplicate variant URNs within one submission are caught here with a readable message rather than deferred to the ``UNIQUE(calibration_id, variant_id)`` database constraint. + The validated URNs are returned so a caller that goes on to persist the controls can reuse + them (see :func:`build_calibration_controls`). + Args: db: Database session used for the lookup. score_set: The score set the calibration belongs to. - controls: Controls submitted on create or modify. ``None`` or empty is a no-op. + controls: Controls submitted on create or modify. ``None`` or empty returns ``[]``. + + Returns: + The submitted and validated variant URNs, in order; empty for ``None``/empty input. Raises: ValidationError: If any control URN is duplicated within the submission, absent from MaveDB, or belongs to a different score set. """ if not controls: - return + return [] submitted_urns = [control.variant_urn for control in controls] @@ -77,6 +84,50 @@ def validate_calibration_controls_in_score_set( f"{', '.join(sorted(missing_urns))}." ) + return submitted_urns + + +def build_calibration_controls( + db: Session, + score_set: ScoreSet, + controls: Optional[Sequence[score_calibration.CalibrationControlCreate]], + user: User, +) -> list[CalibrationControl]: + """Validate and construct transient ``CalibrationControl`` rows for a calibration. + + Validation (see :func:`validate_calibration_controls_in_score_set`) returns the validated + variant URNs, which this function resolves to their ``Variant`` rows in one query and turns + into unattached ``CalibrationControl`` rows with audit fields set. The caller assigns them to + the calibration's ``controls`` collection and commits. Returns an empty list for ``None`` or + empty input. + + Args: + db: Database session. + score_set: The score set the calibration belongs to; controls resolve within it. + controls: Submitted controls, or ``None``/empty for none. + user: The acting user, recorded on each control's audit fields. + """ + submitted_urns = validate_calibration_controls_in_score_set(db, score_set, controls) + if not controls: + return [] + + variants_by_urn = { + variant.urn: variant + for variant in db.scalars( + select(Variant).where(Variant.score_set_id == score_set.id, Variant.urn.in_(submitted_urns)) + ).all() + } + + return [ + CalibrationControl( + variant=variants_by_urn[control.variant_urn], + clinical_status=control.clinical_status, + created_by=user, + modified_by=user, + ) + for control in controls + ] + def create_functional_classification( db: Session, @@ -313,6 +364,10 @@ async def create_score_calibration_in_score_set( else: calibration.investigator_provided = False + calibration.controls = build_calibration_controls( + db, containing_score_set, getattr(calibration_create, "controls", None), user + ) + db.add(calibration) return calibration @@ -535,6 +590,17 @@ async def modify_score_calibration( db.add(persisted_functional_range) calibration.functional_classifications.append(persisted_functional_range) + # Replace semantics: a provided controls list (even empty) replaces all existing controls, while + # None leaves them untouched. TODO#752: reset controls_not_phi here once the re-acknowledgment + # behavior is settled against PUT semantics and the editor's same-request affirmation. + submitted_controls = getattr(calibration_update, "controls", None) + if submitted_controls is not None: + for control in list(calibration.controls): + db.delete(control) + calibration.controls.clear() + db.flush() + calibration.controls = build_calibration_controls(db, containing_score_set, submitted_controls, user) + db.add(calibration) return calibration diff --git a/tests/lib/test_calibration_control.py b/tests/lib/test_calibration_control.py index 2c9f254fd..050f780ca 100644 --- a/tests/lib/test_calibration_control.py +++ b/tests/lib/test_calibration_control.py @@ -6,7 +6,7 @@ from sqlalchemy.exc import IntegrityError -from mavedb.lib.score_calibrations import validate_calibration_controls_in_score_set +from mavedb.lib.score_calibrations import build_calibration_controls, validate_calibration_controls_in_score_set from mavedb.lib.validation.exceptions import ValidationError from mavedb.models.calibration_control import CalibrationControl from mavedb.models.enums.calibration_control_status import CalibrationControlStatus @@ -144,15 +144,16 @@ def _variant_in_other_score_set(session, reference_score_set, user) -> Variant: @pytest.mark.parametrize("controls", [None, []]) def test_validate_controls_is_a_no_op_when_absent(session, setup_lib_db_with_variant, controls): variant = setup_lib_db_with_variant - # Must not raise for None or an empty list. - validate_calibration_controls_in_score_set(session, variant.score_set, controls) + # Must not raise for None or an empty list, and returns no URNs. + assert validate_calibration_controls_in_score_set(session, variant.score_set, controls) == [] def test_validate_controls_accepts_variant_in_score_set(session, setup_lib_db_with_variant): variant = setup_lib_db_with_variant controls = [CalibrationControlCreate(variant_urn=variant.urn, clinical_status="pathogenic")] - validate_calibration_controls_in_score_set(session, variant.score_set, controls) + # Returns the validated URNs so persistence can reuse them without rebuilding the list. + assert validate_calibration_controls_in_score_set(session, variant.score_set, controls) == [variant.urn] def test_validate_controls_rejects_nonexistent_variant(session, setup_lib_db_with_variant): @@ -183,3 +184,39 @@ def test_validate_controls_rejects_duplicate_variant_urns(session, setup_lib_db_ with pytest.raises(ValidationError, match="Duplicate control variant URNs detected"): validate_calibration_controls_in_score_set(session, variant.score_set, controls) + + +############################################################################## +# build_calibration_controls (#753 lib persistence) +############################################################################## + + +@pytest.mark.parametrize("controls", [None, []]) +def test_build_calibration_controls_empty_for_absent_input(session, setup_lib_db_with_variant, controls): + variant = setup_lib_db_with_variant + user = session.query(User).filter(User.username == TEST_USER["username"]).first() + + assert build_calibration_controls(session, variant.score_set, controls, user) == [] + + +def test_build_calibration_controls_constructs_rows(session, setup_lib_db_with_variant): + variant = setup_lib_db_with_variant + user = session.query(User).filter(User.username == TEST_USER["username"]).first() + controls = [CalibrationControlCreate(variant_urn=variant.urn, clinical_status="benign")] + + built = build_calibration_controls(session, variant.score_set, controls, user) + + assert len(built) == 1 + assert built[0].variant is variant + assert built[0].clinical_status is CalibrationControlStatus.benign + assert built[0].created_by is user + assert built[0].modified_by is user + + +def test_build_calibration_controls_propagates_validation_error(session, setup_lib_db_with_variant): + variant = setup_lib_db_with_variant + user = session.query(User).filter(User.username == TEST_USER["username"]).first() + controls = [CalibrationControlCreate(variant_urn="urn:mavedb:99999999-x-9#1", clinical_status="benign")] + + with pytest.raises(ValidationError, match="do not belong to the calibration's score set"): + build_calibration_controls(session, variant.score_set, controls, user) From d5ab9bcfe3d142faf71d29046f1f26be4415661b Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 16 Sep 2026 15:16:15 -0700 Subject: [PATCH 150/252] feat(calibration-controls): reset controls_not_phi when controls change on modify (#753) On modify, controls_not_phi is no longer set by the generic attribute loop. An explicit value in the request always wins; otherwise replacing the controls invalidates any prior affirmation (reset to None, forcing re-acknowledgment before republishing), while leaving the controls untouched preserves it. This keeps an unrelated edit from silently wiping an affirmation and supports the editor's change-controls-and-affirm-in-one-save flow. --- src/mavedb/lib/score_calibrations.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/mavedb/lib/score_calibrations.py b/src/mavedb/lib/score_calibrations.py index cd36b12e3..73092fc69 100644 --- a/src/mavedb/lib/score_calibrations.py +++ b/src/mavedb/lib/score_calibrations.py @@ -567,6 +567,9 @@ async def modify_score_calibration( if attr not in { "functional_classifications", "controls", + # controls_not_phi carries re-acknowledgment semantics; set it explicitly so an + # unrelated edit cannot silently wipe a prior affirmation. + "controls_not_phi", "threshold_sources", "evidence_sources", "method_sources", @@ -591,8 +594,7 @@ async def modify_score_calibration( calibration.functional_classifications.append(persisted_functional_range) # Replace semantics: a provided controls list (even empty) replaces all existing controls, while - # None leaves them untouched. TODO#752: reset controls_not_phi here once the re-acknowledgment - # behavior is settled against PUT semantics and the editor's same-request affirmation. + # None leaves them untouched. submitted_controls = getattr(calibration_update, "controls", None) if submitted_controls is not None: for control in list(calibration.controls): @@ -601,6 +603,13 @@ async def modify_score_calibration( db.flush() calibration.controls = build_calibration_controls(db, containing_score_set, submitted_controls, user) + # Re-acknowledgment: an explicit controls_not_phi in the request always wins; otherwise, replacing the + # controls invalidates any prior affirmation, while leaving the controls untouched preserves it. + if "controls_not_phi" in calibration_update.model_fields_set: + calibration.controls_not_phi = calibration_update.controls_not_phi + elif submitted_controls is not None: + calibration.controls_not_phi = None + db.add(calibration) return calibration From 17a2b2d4776c365787b0bf92c8335899318119bf Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 16 Sep 2026 15:16:30 -0700 Subject: [PATCH 151/252] feat(calibration-controls): CSV validation with 1:1 HGVS resolution (#753) Add validate_and_standardize_calibration_controls_dataframe (mirroring the class-based validator) plus the clinical_status column constant. The controls CSV accepts the same variant_urn / hgvs_nt / hgvs_pro columns as the class-based CSV and a clinical_status column validated against CalibrationControlStatus (case-insensitive), returning CalibrationControlCreate rows so the router feeds inline and CSV controls through one path. Because a control is a single variant while an HGVS can match many (hgvs_pro collapses distinct nucleotide variants in a nucleotide-indexed score set), resolution rejects any identifier that maps to more than one variant in the score set rather than silently choosing one. --- .../lib/validation/constants/general.py | 1 + .../lib/validation/dataframe/calibration.py | 146 ++++++++++++++++++ tests/lib/test_calibration_control.py | 107 +++++++++++++ 3 files changed, 254 insertions(+) diff --git a/src/mavedb/lib/validation/constants/general.py b/src/mavedb/lib/validation/constants/general.py index 22ca4cbf5..14a087fc7 100644 --- a/src/mavedb/lib/validation/constants/general.py +++ b/src/mavedb/lib/validation/constants/general.py @@ -46,6 +46,7 @@ calibration_variant_column_name = "variant_urn" calibration_class_column_name = "class_name" +calibration_control_status_column_name = "clinical_status" valid_dataset_columns = [score_columns, count_columns] valid_variant_columns = [variant_score_data, variant_count_data] diff --git a/src/mavedb/lib/validation/dataframe/calibration.py b/src/mavedb/lib/validation/dataframe/calibration.py index 1c46be46b..7594c36c5 100644 --- a/src/mavedb/lib/validation/dataframe/calibration.py +++ b/src/mavedb/lib/validation/dataframe/calibration.py @@ -4,6 +4,7 @@ from mavedb.lib.validation.constants.general import ( calibration_class_column_name, + calibration_control_status_column_name, calibration_variant_column_name, hgvs_nt_column, hgvs_pro_column, @@ -11,6 +12,7 @@ from mavedb.lib.validation.dataframe.column import validate_data_column, validate_variant_column from mavedb.lib.validation.dataframe.dataframe import standardize_dataframe, validate_no_null_rows from mavedb.lib.validation.exceptions import ValidationError +from mavedb.models.enums.calibration_control_status import CalibrationControlStatus from mavedb.models.score_set import ScoreSet from mavedb.models.variant import Variant from mavedb.view_models import score_calibration @@ -22,6 +24,13 @@ hgvs_pro_column, ) +STANDARD_CALIBRATION_CONTROL_COLUMNS = ( + calibration_variant_column_name, + calibration_control_status_column_name, + hgvs_nt_column, + hgvs_pro_column, +) + def validate_and_standardize_calibration_classes_dataframe( db: Session, @@ -246,3 +255,140 @@ def validate_calibration_classes( unprovided_classes = defined_classes - provided_classes if unprovided_classes: raise ValidationError("Some defined classes in the calibration are missing from the classes file.") + + +def validate_and_standardize_calibration_controls_dataframe( + db: Session, + score_set: ScoreSet, + controls_df: pd.DataFrame, +) -> list[score_calibration.CalibrationControlCreate]: + """Validate a calibration controls CSV and convert it to ``CalibrationControlCreate`` rows. + + Mirrors :func:`validate_and_standardize_calibration_classes_dataframe`. The CSV must carry a + variant identifier column (``variant_urn``, ``hgvs_nt``, or ``hgvs_pro``) and a + ``clinical_status`` column. Every variant must belong to the score set, and every status must be + a ``CalibrationControlStatus`` value (case-insensitive). The chosen variant column is resolved to + MaveDB URNs so the returned controls carry ``variant_urn``, matching the inline-JSON contract. + + Args: + db: Database session for validation and URN resolution. + score_set: The score set the calibration belongs to. + controls_df: The parsed controls CSV. + + Returns: + One ``CalibrationControlCreate`` per row. + + Raises: + ValidationError: On unexpected/missing columns, invalid variants, or invalid status values. + """ + standardized_df = standardize_dataframe(controls_df, STANDARD_CALIBRATION_CONTROL_COLUMNS) + validate_calibration_control_df_column_names(standardized_df) + validate_no_null_rows(standardized_df) + + column_mapping = {c.lower(): c for c in standardized_df.columns} + index_column = choose_calibration_index_column(standardized_df) + + # Drop rows without a clinical status. + standardized_df = standardized_df.dropna( + subset=[column_mapping[calibration_control_status_column_name]] + ).reset_index(drop=True) + + for c in column_mapping: + if c in {calibration_variant_column_name, hgvs_nt_column, hgvs_pro_column}: + validate_variant_column(standardized_df[c], column_mapping[c] == index_column) + elif c == calibration_control_status_column_name: + validate_data_column(standardized_df[c], force_numeric=False) + validate_calibration_control_statuses(standardized_df[column_mapping[c]]) + + if column_mapping[c] == index_column: + validate_index_existence_in_score_set(db, score_set, standardized_df[column_mapping[c]], column_mapping[c]) + + return resolve_calibration_controls(db, score_set, standardized_df, index_column) + + +def validate_calibration_control_df_column_names(df: pd.DataFrame) -> None: + """Validate the column names of a calibration controls DataFrame. + + Requires a ``clinical_status`` column and at least one variant identifier column, and rejects + unexpected columns. Comparison is case-insensitive. Mirrors + :func:`validate_calibration_df_column_names`. + """ + if any(type(c) is not str for c in df.columns): + raise ValidationError("column names must be strings") + + if any(c.isspace() for c in df.columns) or any(len(c) == 0 for c in df.columns): + raise ValidationError("column names cannot be empty or whitespace") + + if len(df.columns) != len(set(c.lower() for c in df.columns)): + raise ValidationError("duplicate column names are not allowed (case-insensitive)") + + columns = [c.lower() for c in df.columns] + + if calibration_control_status_column_name not in columns: + raise ValidationError(f"missing required column: '{calibration_control_status_column_name}'") + + if set(columns).isdisjoint({hgvs_nt_column, hgvs_pro_column, calibration_variant_column_name}): + raise ValidationError( + f"at least one of {', '.join({hgvs_nt_column, hgvs_pro_column, calibration_variant_column_name})} must be present" + ) + + +def validate_calibration_control_statuses(statuses: pd.Series) -> None: + """Ensure every ``clinical_status`` value maps to a ``CalibrationControlStatus`` (case-insensitive).""" + valid_statuses = {status.value for status in CalibrationControlStatus} + invalid_statuses = { + str(status) for status in statuses.tolist() if str(status).strip().lower() not in valid_statuses + } + if invalid_statuses: + raise ValidationError( + f"Invalid clinical status value(s): {', '.join(sorted(invalid_statuses))}. " + f"Allowed values are: {', '.join(sorted(valid_statuses))}." + ) + + +def resolve_calibration_controls( + db: Session, + score_set: ScoreSet, + df: pd.DataFrame, + index_column: str, +) -> list[score_calibration.CalibrationControlCreate]: + """Convert a validated controls DataFrame to ``CalibrationControlCreate`` rows, resolving URNs. + + For a ``variant_urn`` index the URN is taken directly. For an HGVS index the values are mapped to + their variant URNs within the score set in a single query. A control is a single variant, so an + HGVS that matches more than one variant is ambiguous and rejected — whether a given HGVS column + collides depends on the score set's own index (``hgvs_pro`` is unique in a protein-indexed set but + not in a nucleotide-indexed one), so this is checked against the data rather than assumed. + """ + column_mapping = {c.lower(): c for c in df.columns} + status_column = column_mapping[calibration_control_status_column_name] + index_values = df[index_column].tolist() + + if index_column.lower() == calibration_variant_column_name: + urn_by_index_value = {value: value for value in index_values} + else: + hgvs_attribute = Variant.hgvs_nt if index_column.lower() == hgvs_nt_column else Variant.hgvs_pro + urns_by_hgvs: dict[str, list[str]] = {} + for urn, hgvs in db.execute( + select(Variant.urn, hgvs_attribute).where( + Variant.score_set_id == score_set.id, hgvs_attribute.in_(index_values) + ) + ).all(): + urns_by_hgvs.setdefault(hgvs, []).append(urn) + + ambiguous_identifiers = sorted(hgvs for hgvs, urns in urns_by_hgvs.items() if len(urns) > 1) + if ambiguous_identifiers: + raise ValidationError( + "These variant identifiers match more than one variant in the score set; " + f"use the variant URN instead: {', '.join(ambiguous_identifiers)}." + ) + + urn_by_index_value = {hgvs: urns[0] for hgvs, urns in urns_by_hgvs.items()} + + return [ + score_calibration.CalibrationControlCreate( + variant_urn=urn_by_index_value[row[index_column]], + clinical_status=CalibrationControlStatus(str(row[status_column]).strip().lower()), + ) + for _, row in df.iterrows() + ] diff --git a/tests/lib/test_calibration_control.py b/tests/lib/test_calibration_control.py index 050f780ca..5637f6286 100644 --- a/tests/lib/test_calibration_control.py +++ b/tests/lib/test_calibration_control.py @@ -4,9 +4,17 @@ pytest.importorskip("psycopg2") +import pandas as pd from sqlalchemy.exc import IntegrityError from mavedb.lib.score_calibrations import build_calibration_controls, validate_calibration_controls_in_score_set +from mavedb.lib.validation.constants.general import ( + calibration_control_status_column_name, + calibration_variant_column_name, + hgvs_nt_column, + hgvs_pro_column, +) +from mavedb.lib.validation.dataframe.calibration import validate_and_standardize_calibration_controls_dataframe from mavedb.lib.validation.exceptions import ValidationError from mavedb.models.calibration_control import CalibrationControl from mavedb.models.enums.calibration_control_status import CalibrationControlStatus @@ -220,3 +228,102 @@ def test_build_calibration_controls_propagates_validation_error(session, setup_l with pytest.raises(ValidationError, match="do not belong to the calibration's score set"): build_calibration_controls(session, variant.score_set, controls, user) + + +############################################################################## +# validate_and_standardize_calibration_controls_dataframe (#753 CSV path) +############################################################################## + + +def _controls_df(variant_values, statuses): + return pd.DataFrame( + {calibration_variant_column_name: variant_values, calibration_control_status_column_name: statuses} + ) + + +def test_controls_csv_validates_and_converts_to_create_rows(session, setup_lib_db_with_variant): + variant = setup_lib_db_with_variant + # Status is case-insensitive. + df = _controls_df([variant.urn], ["Pathogenic"]) + + controls = validate_and_standardize_calibration_controls_dataframe(session, variant.score_set, df) + + assert len(controls) == 1 + assert controls[0].variant_urn == variant.urn + assert controls[0].clinical_status is CalibrationControlStatus.pathogenic + + +def test_controls_csv_rejects_invalid_status(session, setup_lib_db_with_variant): + variant = setup_lib_db_with_variant + df = _controls_df([variant.urn], ["likely_pathogenic"]) + + with pytest.raises(ValidationError, match="Invalid clinical status"): + validate_and_standardize_calibration_controls_dataframe(session, variant.score_set, df) + + +def test_controls_csv_rejects_variant_not_in_score_set(session, setup_lib_db_with_variant): + variant = setup_lib_db_with_variant + df = _controls_df(["urn:mavedb:99999999-x-9#1"], ["benign"]) + + with pytest.raises(ValidationError, match="do not exist in the score set"): + validate_and_standardize_calibration_controls_dataframe(session, variant.score_set, df) + + +def test_controls_csv_rejects_missing_status_column(session, setup_lib_db_with_variant): + variant = setup_lib_db_with_variant + df = pd.DataFrame({calibration_variant_column_name: [variant.urn]}) + + with pytest.raises(ValidationError, match=calibration_control_status_column_name): + validate_and_standardize_calibration_controls_dataframe(session, variant.score_set, df) + + +def test_controls_csv_accepts_hgvs_nt_index(session, setup_lib_db_with_variant): + variant = setup_lib_db_with_variant + # hgvs_nt is unique within the score set, so it resolves 1:1 to the variant URN. + df = pd.DataFrame({hgvs_nt_column: [variant.hgvs_nt], calibration_control_status_column_name: ["benign"]}) + + controls = validate_and_standardize_calibration_controls_dataframe(session, variant.score_set, df) + + assert len(controls) == 1 + assert controls[0].variant_urn == variant.urn + assert controls[0].clinical_status is CalibrationControlStatus.benign + + +@pytest.mark.parametrize( + "index_column, shared_value, distinct_values", + [ + # hgvs_pro collision: two nucleotide variants collapsing to the same protein consequence + # (the expected case in a nucleotide-indexed score set). + (hgvs_pro_column, "p.Met1Val", ("c.1A>G", "c.3G>T")), + # hgvs_nt collision: not expected in practice, but the detection must still fire. + (hgvs_nt_column, "c.99A>G", ("p.Lys33Arg", "p.Lys33Gln")), + ], +) +def test_controls_csv_rejects_ambiguous_hgvs_index( + session, setup_lib_db_with_variant, index_column, shared_value, distinct_values +): + variant = setup_lib_db_with_variant + score_set = variant.score_set + + # Two variants that share the submitted index value but differ on the other HGVS field, so the + # index resolves to more than one variant in the score set. + for suffix, distinct_value in enumerate(distinct_values, start=2): + hgvs_nt = shared_value if index_column == hgvs_nt_column else distinct_value + hgvs_pro = shared_value if index_column == hgvs_pro_column else distinct_value + session.add( + Variant( + data=TEST_MINIMAL_VARIANT["data"], + hgvs_nt=hgvs_nt, + hgvs_pro=hgvs_pro, + creation_date=TEST_MINIMAL_VARIANT["creation_date"], + modification_date=TEST_MINIMAL_VARIANT["modification_date"], + urn=f"{score_set.urn}#{suffix}", + score_set_id=score_set.id, + ) + ) + session.commit() + + df = pd.DataFrame({index_column: [shared_value], calibration_control_status_column_name: ["pathogenic"]}) + + with pytest.raises(ValidationError, match="match more than one variant"): + validate_and_standardize_calibration_controls_dataframe(session, score_set, df) From bc47a8be42a57dbbd7ebdccb0f05bd07461109b2 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 16 Sep 2026 16:57:30 -0700 Subject: [PATCH 152/252] feat(calibration-controls): wire controls into create/update/read endpoints (#753) Add an optional controls_file CSV upload to the create and modify routes alongside the existing classes_file, documented in the multipart OpenAPI schema and route docstrings. Providing inline controls and a controls_file together returns 422; a controls_file is parsed and validated (URN or HGVS, 1:1) and fed through the same create/modify path as inline controls. GET returns controls; list responses are addressed separately (#753 slice 4). Integration tests cover inline create, CSV create, inline+file conflict, GET, modify replace/clear/omit, and the controls_not_phi re-acknowledgment behavior. --- src/mavedb/routers/score_calibrations.py | 95 ++++++++- tests/routers/test_calibration_controls.py | 225 +++++++++++++++++++++ 2 files changed, 317 insertions(+), 3 deletions(-) diff --git a/src/mavedb/routers/score_calibrations.py b/src/mavedb/routers/score_calibrations.py index 6e876c29d..d585dd45c 100644 --- a/src/mavedb/routers/score_calibrations.py +++ b/src/mavedb/routers/score_calibrations.py @@ -25,8 +25,17 @@ ) from mavedb.lib.score_sets import csv_data_to_df from mavedb.lib.types.authentication import UserData -from mavedb.lib.validation.constants.general import calibration_class_column_name, calibration_variant_column_name -from mavedb.lib.validation.dataframe.calibration import validate_and_standardize_calibration_classes_dataframe +from mavedb.lib.validation.constants.general import ( + calibration_class_column_name, + calibration_control_status_column_name, + calibration_variant_column_name, + hgvs_nt_column, + hgvs_pro_column, +) +from mavedb.lib.validation.dataframe.calibration import ( + validate_and_standardize_calibration_classes_dataframe, + validate_and_standardize_calibration_controls_dataframe, +) from mavedb.lib.validation.exceptions import ValidationError from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification @@ -224,6 +233,11 @@ async def get_primary_score_calibrations_for_score_set( "format": "binary", "description": "CSV file containing variant classifications", }, + "controls_file": { + "type": "string", + "format": "binary", + "description": "CSV file containing calibration controls", + }, }, } }, @@ -239,6 +253,10 @@ async def create_score_calibration_route( None, description=f"CSV file containing variant classifications. This file must contain two columns: '{calibration_variant_column_name}' and '{calibration_class_column_name}'.", ), + controls_file: Optional[UploadFile] = File( + None, + description=f"CSV file containing calibration controls. This file must contain a variant column (one of '{calibration_variant_column_name}', '{hgvs_nt_column}', '{hgvs_pro_column}') and a '{calibration_control_status_column_name}' column.", + ), db: Session = Depends(deps.get_db), user_data: UserData = Depends(require_current_user_with_email), ) -> ScoreCalibration: @@ -272,6 +290,7 @@ async def create_score_calibration_route( **Form Fields**: - `calibration_json` (string, required): JSON string containing the calibration data - `classes_file` (file, optional): CSV file containing variant classifications + - `controls_file` (file, optional): CSV file containing calibration controls **Example**: ```bash @@ -287,11 +306,16 @@ async def create_score_calibration_route( - If uploading a classes_file, it must be a valid CSV with variant classification data - User must have ADD_CALIBRATION permission on the score set (any authenticated user for published sets; contributors/owners/admins for private sets) - + ## File Upload Details The `classes_file` parameter accepts CSV files containing variant classification data. The file should have appropriate headers and contain columns for variant urns and class names. + The `controls_file` parameter accepts a CSV of calibration controls with a variant column + (one of `variant_urn`, `hgvs_nt`, `hgvs_pro`) and a `clinical_status` column (`pathogenic` or + `benign`, case-insensitive). Controls may be supplied either via this file or inline in + `calibration_json`, but not both. + ## Response Returns the created score calibration with its generated URN and associated score set information. """ @@ -338,6 +362,29 @@ async def create_score_calibration_route( detail=[{"loc": [e.custom_loc or "classesFile"], "msg": str(e), "type": "value_error"}], ) + # Controls may be supplied inline in the JSON payload or as a CSV, but not both. + if calibration.controls is not None and controls_file: + raise HTTPException( + status_code=422, + detail="Provide calibration controls either inline or via controls_file, not both.", + ) + + if controls_file: + try: + controls_df = csv_data_to_df(controls_file.file, induce_hgvs_cols=False) + except UnicodeDecodeError as e: + raise HTTPException( + status_code=400, detail=f"Error decoding file: {e}. Ensure the file has correct values." + ) + + try: + calibration.controls = validate_and_standardize_calibration_controls_dataframe(db, score_set, controls_df) + except ValidationError as e: + raise HTTPException( + status_code=422, + detail=[{"loc": [e.custom_loc or "controlsFile"], "msg": str(e), "type": "value_error"}], + ) + created_calibration = await create_score_calibration_in_score_set( db, calibration, user_data.user, variant_classes if classes_file else None ) @@ -372,6 +419,11 @@ async def create_score_calibration_route( "format": "binary", "description": "CSV file containing updated variant classifications", }, + "controls_file": { + "type": "string", + "format": "binary", + "description": "CSV file containing updated calibration controls (replaces existing)", + }, }, } }, @@ -388,6 +440,10 @@ async def modify_score_calibration_route( None, description=f"CSV file containing variant classifications. This file must contain two columns: '{calibration_variant_column_name}' and '{calibration_class_column_name}'.", ), + controls_file: Optional[UploadFile] = File( + None, + description=f"CSV file containing calibration controls. This file must contain a variant column (one of '{calibration_variant_column_name}', '{hgvs_nt_column}', '{hgvs_pro_column}') and a '{calibration_control_status_column_name}' column. Replaces existing controls.", + ), db: Session = Depends(deps.get_db), user_data: UserData = Depends(require_current_user_with_email), ) -> ScoreCalibration: @@ -421,6 +477,7 @@ async def modify_score_calibration_route( **Form Fields**: - `calibration_json` (string, required): JSON string containing the calibration update data - `classes_file` (file, optional): CSV file containing updated variant classifications + - `controls_file` (file, optional): CSV file containing calibration controls (replaces existing) **Example**: ```bash @@ -442,6 +499,12 @@ async def modify_score_calibration_route( The file should have appropriate headers and follow the expected format for variant classifications within the associated score set. + The `controls_file` parameter accepts a CSV of calibration controls with a variant column + (one of `variant_urn`, `hgvs_nt`, `hgvs_pro`) and a `clinical_status` column (`pathogenic` or + `benign`, case-insensitive). If provided, it replaces the calibration's existing controls. + Controls may be supplied either via this file or inline in `calibration_json`, but not both; + omitting both leaves existing controls unchanged. + ## Response Returns the updated score calibration with all modifications applied and any new classification data from the uploaded file. @@ -506,6 +569,32 @@ async def modify_score_calibration_route( detail=[{"loc": [e.custom_loc or "classesFile"], "msg": str(e), "type": "value_error"}], ) + # Controls may be supplied inline in the JSON payload or as a CSV, but not both. A controls_file + # replaces existing controls; omitting both leaves them unchanged (see modify replace semantics). + if calibration_update.controls is not None and controls_file: + raise HTTPException( + status_code=422, + detail="Provide calibration controls either inline or via controls_file, not both.", + ) + + if controls_file: + try: + controls_df = csv_data_to_df(controls_file.file, induce_hgvs_cols=False) + except UnicodeDecodeError as e: + raise HTTPException( + status_code=400, detail=f"Error decoding file: {e}. Ensure the file has correct values." + ) + + try: + calibration_update.controls = validate_and_standardize_calibration_controls_dataframe( + db, score_set, controls_df + ) + except ValidationError as e: + raise HTTPException( + status_code=422, + detail=[{"loc": [e.custom_loc or "controlsFile"], "msg": str(e), "type": "value_error"}], + ) + updated_calibration = await modify_score_calibration( db, item, calibration_update, user_data.user, variant_classes if classes_file else None ) diff --git a/tests/routers/test_calibration_controls.py b/tests/routers/test_calibration_controls.py index eeab1bba1..6d7fc55ba 100644 --- a/tests/routers/test_calibration_controls.py +++ b/tests/routers/test_calibration_controls.py @@ -6,11 +6,18 @@ cdot = pytest.importorskip("cdot") fastapi = pytest.importorskip("fastapi") +import json + from sqlalchemy import select +from mavedb.lib.validation.constants.general import ( + calibration_control_status_column_name, + calibration_variant_column_name, +) from mavedb.models.calibration_control import CalibrationControl from mavedb.models.enums.calibration_control_status import CalibrationControlStatus from mavedb.models.score_calibration import ScoreCalibration as CalibrationDbModel +from mavedb.models.score_set import ScoreSet as ScoreSetDbModel from mavedb.models.variant import Variant from tests.helpers.constants import ( TEST_BIORXIV_IDENTIFIER, @@ -28,6 +35,27 @@ ] +def _score_set_with_variant_urns(client, session, data_provider, data_files): + """Create a score set with mapped variants; return (score_set dict, list of its variant URNs).""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + score_set_orm = session.query(ScoreSetDbModel).where(ScoreSetDbModel.urn == score_set["urn"]).one() + variant_urns = [ + variant.urn + for variant in session.scalars(select(Variant).where(Variant.score_set_id == score_set_orm.id)).all() + ] + return score_set, variant_urns + + +def _calibration_payload(score_set_urn, **extra): + return {**deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED), "scoreSetUrn": score_set_urn, **extra} + + +CONTROLS_CSV_HEADER = f"{calibration_variant_column_name},{calibration_control_status_column_name}" + + def _create_private_calibration(client, session, data_provider, data_files): """Create a score set with variants and a private range-based calibration; return the calibration dict.""" experiment = create_experiment(client) @@ -109,3 +137,200 @@ def test_can_publish_calibration_without_controls_regardless_of_phi_flag( assert response.status_code == 200 assert response.json()["private"] is False + + +########################################################### +# Controls create / update / read wiring (#753) +########################################################### + + +def _create_with_controls(client, score_set_urn, controls, **extra): + response = client.post( + "/api/v1/score-calibrations/", json=_calibration_payload(score_set_urn, controls=controls, **extra) + ) + assert response.status_code == 200, response.text + return response.json() + + +@pytest.mark.parametrize("mock_publication_fetch", [CALIBRATION_PUBLICATIONS], indirect=["mock_publication_fetch"]) +def test_create_calibration_with_inline_controls( + client, setup_router_db, mock_publication_fetch, session, data_provider, data_files +): + score_set, variant_urns = _score_set_with_variant_urns(client, session, data_provider, data_files) + + calibration = _create_with_controls( + client, + score_set["urn"], + [ + {"variantUrn": variant_urns[0], "clinicalStatus": "pathogenic"}, + {"variantUrn": variant_urns[1], "clinicalStatus": "benign"}, + ], + ) + + assert {(c["variantUrn"], c["clinicalStatus"]) for c in calibration["controls"]} == { + (variant_urns[0], "pathogenic"), + (variant_urns[1], "benign"), + } + + +@pytest.mark.parametrize("mock_publication_fetch", [CALIBRATION_PUBLICATIONS], indirect=["mock_publication_fetch"]) +def test_create_calibration_with_controls_file( + client, setup_router_db, mock_publication_fetch, session, data_provider, data_files +): + score_set, variant_urns = _score_set_with_variant_urns(client, session, data_provider, data_files) + csv = f"{CONTROLS_CSV_HEADER}\n{variant_urns[0]},pathogenic\n{variant_urns[1]},Benign\n" + + response = client.post( + "/api/v1/score-calibrations/", + data={"calibration_json": json.dumps(_calibration_payload(score_set["urn"]))}, + files={"controls_file": ("controls.csv", csv, "text/csv")}, + ) + + assert response.status_code == 200, response.text + assert {(c["variantUrn"], c["clinicalStatus"]) for c in response.json()["controls"]} == { + (variant_urns[0], "pathogenic"), + (variant_urns[1], "benign"), + } + + +@pytest.mark.parametrize("mock_publication_fetch", [CALIBRATION_PUBLICATIONS], indirect=["mock_publication_fetch"]) +def test_create_calibration_rejects_inline_controls_and_controls_file_together( + client, setup_router_db, mock_publication_fetch, session, data_provider, data_files +): + score_set, variant_urns = _score_set_with_variant_urns(client, session, data_provider, data_files) + csv = f"{CONTROLS_CSV_HEADER}\n{variant_urns[0]},pathogenic\n" + + response = client.post( + "/api/v1/score-calibrations/", + data={ + "calibration_json": json.dumps( + _calibration_payload( + score_set["urn"], controls=[{"variantUrn": variant_urns[0], "clinicalStatus": "benign"}] + ) + ) + }, + files={"controls_file": ("controls.csv", csv, "text/csv")}, + ) + + assert response.status_code == 422 + assert "not both" in str(response.json()["detail"]) + + +@pytest.mark.parametrize("mock_publication_fetch", [CALIBRATION_PUBLICATIONS], indirect=["mock_publication_fetch"]) +def test_get_calibration_returns_controls( + client, setup_router_db, mock_publication_fetch, session, data_provider, data_files +): + score_set, variant_urns = _score_set_with_variant_urns(client, session, data_provider, data_files) + calibration = _create_with_controls( + client, score_set["urn"], [{"variantUrn": variant_urns[0], "clinicalStatus": "pathogenic"}] + ) + + response = client.get(f"/api/v1/score-calibrations/{calibration['urn']}") + + assert response.status_code == 200 + assert response.json()["controls"][0]["variantUrn"] == variant_urns[0] + + +@pytest.mark.parametrize("mock_publication_fetch", [CALIBRATION_PUBLICATIONS], indirect=["mock_publication_fetch"]) +def test_modify_replaces_controls(client, setup_router_db, mock_publication_fetch, session, data_provider, data_files): + score_set, variant_urns = _score_set_with_variant_urns(client, session, data_provider, data_files) + calibration = _create_with_controls( + client, score_set["urn"], [{"variantUrn": variant_urns[0], "clinicalStatus": "pathogenic"}] + ) + + response = client.put( + f"/api/v1/score-calibrations/{calibration['urn']}", + json=_calibration_payload( + score_set["urn"], controls=[{"variantUrn": variant_urns[1], "clinicalStatus": "benign"}] + ), + ) + + assert response.status_code == 200, response.text + controls = response.json()["controls"] + assert len(controls) == 1 + assert controls[0]["variantUrn"] == variant_urns[1] + assert controls[0]["clinicalStatus"] == "benign" + + +@pytest.mark.parametrize("mock_publication_fetch", [CALIBRATION_PUBLICATIONS], indirect=["mock_publication_fetch"]) +def test_modify_with_empty_controls_clears_them( + client, setup_router_db, mock_publication_fetch, session, data_provider, data_files +): + score_set, variant_urns = _score_set_with_variant_urns(client, session, data_provider, data_files) + calibration = _create_with_controls( + client, score_set["urn"], [{"variantUrn": variant_urns[0], "clinicalStatus": "pathogenic"}] + ) + + response = client.put( + f"/api/v1/score-calibrations/{calibration['urn']}", + json=_calibration_payload(score_set["urn"], controls=[]), + ) + + assert response.status_code == 200, response.text + assert response.json()["controls"] == [] + + +@pytest.mark.parametrize("mock_publication_fetch", [CALIBRATION_PUBLICATIONS], indirect=["mock_publication_fetch"]) +def test_modify_without_controls_leaves_them_unchanged( + client, setup_router_db, mock_publication_fetch, session, data_provider, data_files +): + score_set, variant_urns = _score_set_with_variant_urns(client, session, data_provider, data_files) + calibration = _create_with_controls( + client, score_set["urn"], [{"variantUrn": variant_urns[0], "clinicalStatus": "pathogenic"}] + ) + + # The payload omits controls entirely, so existing controls must be preserved. + response = client.put( + f"/api/v1/score-calibrations/{calibration['urn']}", json=_calibration_payload(score_set["urn"]) + ) + + assert response.status_code == 200, response.text + assert len(response.json()["controls"]) == 1 + + +@pytest.mark.parametrize("mock_publication_fetch", [CALIBRATION_PUBLICATIONS], indirect=["mock_publication_fetch"]) +def test_modify_controls_without_affirmation_resets_phi_flag( + client, setup_router_db, mock_publication_fetch, session, data_provider, data_files +): + score_set, variant_urns = _score_set_with_variant_urns(client, session, data_provider, data_files) + calibration = _create_with_controls( + client, + score_set["urn"], + [{"variantUrn": variant_urns[0], "clinicalStatus": "pathogenic"}], + controlsNotPhi=True, + ) + assert calibration["controlsNotPhi"] is True + + # Changing controls without re-affirming invalidates the prior affirmation. + response = client.put( + f"/api/v1/score-calibrations/{calibration['urn']}", + json=_calibration_payload( + score_set["urn"], controls=[{"variantUrn": variant_urns[1], "clinicalStatus": "benign"}] + ), + ) + + assert response.status_code == 200, response.text + assert response.json()["controlsNotPhi"] is None + + +@pytest.mark.parametrize("mock_publication_fetch", [CALIBRATION_PUBLICATIONS], indirect=["mock_publication_fetch"]) +def test_modify_controls_with_affirmation_keeps_phi_flag( + client, setup_router_db, mock_publication_fetch, session, data_provider, data_files +): + score_set, variant_urns = _score_set_with_variant_urns(client, session, data_provider, data_files) + calibration = _create_with_controls( + client, score_set["urn"], [{"variantUrn": variant_urns[0], "clinicalStatus": "pathogenic"}] + ) + + # Affirming in the same request that changes controls keeps the flag set. + response = client.put( + f"/api/v1/score-calibrations/{calibration['urn']}", + json=_calibration_payload( + score_set["urn"], + controls=[{"variantUrn": variant_urns[1], "clinicalStatus": "benign"}], + controlsNotPhi=True, + ), + ) + + assert response.status_code == 200, response.text + assert response.json()["controlsNotPhi"] is True From 11ca9a265e090c8741d23d58daeac85a04493406 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 16 Sep 2026 17:27:34 -0700 Subject: [PATCH 153/252] feat(calibration-controls): report controlsCount on list responses (#753) Add a controls_count correlated-COUNT column_property to ScoreCalibration (mirroring FunctionalClassification.variant_count). Move the full controls list off SavedScoreCalibration onto the detail models (the full ScoreCalibration and a new ScoreCalibrationDetailWithScoreSetUrn), and add controls_count to the base. The single-item calibration endpoints return the detail model (full controls + count); the two list endpoints return count-only. Net: calibration list responses and score-set-embedded SavedScoreCalibration carry controlsCount, while single-calibration detail, score-set full detail, and the public dump keep the full controls list. --- src/mavedb/models/score_calibration.py | 16 +++++++++++--- src/mavedb/routers/score_calibrations.py | 14 ++++++------- src/mavedb/view_models/score_calibration.py | 17 +++++++++++++-- tests/helpers/constants.py | 2 ++ tests/routers/test_calibration_controls.py | 23 +++++++++++++++++++++ 5 files changed, 60 insertions(+), 12 deletions(-) diff --git a/src/mavedb/models/score_calibration.py b/src/mavedb/models/score_calibration.py index 08f4b3a0f..d32b3ed9a 100644 --- a/src/mavedb/models/score_calibration.py +++ b/src/mavedb/models/score_calibration.py @@ -5,18 +5,18 @@ from datetime import date from typing import TYPE_CHECKING -from sqlalchemy import Boolean, Column, Date, Float, ForeignKey, Integer, String +from sqlalchemy import Boolean, Column, Date, Float, ForeignKey, Integer, String, func, select from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.ext.associationproxy import AssociationProxy, association_proxy -from sqlalchemy.orm import Mapped, relationship +from sqlalchemy.orm import Mapped, column_property, relationship from mavedb.db.base import Base from mavedb.lib.urns import generate_calibration_urn +from mavedb.models.calibration_control import CalibrationControl from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification from mavedb.models.score_calibration_publication_identifier import ScoreCalibrationPublicationIdentifierAssociation if TYPE_CHECKING: - from mavedb.models.calibration_control import CalibrationControl from mavedb.models.publication_identifier import PublicationIdentifier from mavedb.models.score_set import ScoreSet from mavedb.models.user import User @@ -67,6 +67,16 @@ class ScoreCalibration(Base): cascade="all, delete-orphan", ) + # Efficient count via correlated subquery — lets list/collection responses report how many + # controls a calibration has without loading the control rows. Mirrors + # ``ScoreCalibrationFunctionalClassification.variant_count``. + controls_count: Mapped[int] = column_property( + select(func.count(CalibrationControl.id)) + .where(CalibrationControl.calibration_id == id) + .correlate_except(CalibrationControl) + .scalar_subquery() + ) + publication_identifier_associations: Mapped[list[ScoreCalibrationPublicationIdentifierAssociation]] = relationship( "ScoreCalibrationPublicationIdentifierAssociation", back_populates="score_calibration", diff --git a/src/mavedb/routers/score_calibrations.py b/src/mavedb/routers/score_calibrations.py index d585dd45c..bc5c48e8d 100644 --- a/src/mavedb/routers/score_calibrations.py +++ b/src/mavedb/routers/score_calibrations.py @@ -87,7 +87,7 @@ def list_my_calibrations( @router.get( "/{urn}", - response_model=score_calibration.ScoreCalibrationWithScoreSetUrn, + response_model=score_calibration.ScoreCalibrationDetailWithScoreSetUrn, responses={404: {}}, ) def get_score_calibration( @@ -157,7 +157,7 @@ async def get_score_calibrations_for_score_set( @router.get( "/score-set/{score_set_urn}/primary", - response_model=score_calibration.ScoreCalibrationWithScoreSetUrn, + response_model=score_calibration.ScoreCalibrationDetailWithScoreSetUrn, responses={404: {}}, ) async def get_primary_score_calibrations_for_score_set( @@ -211,7 +211,7 @@ async def get_primary_score_calibrations_for_score_set( @router.post( "/", - response_model=score_calibration.ScoreCalibrationWithScoreSetUrn, + response_model=score_calibration.ScoreCalibrationDetailWithScoreSetUrn, responses={404: {}, 422: {"description": "Validation Error"}}, openapi_extra={ "requestBody": { @@ -397,7 +397,7 @@ async def create_score_calibration_route( @router.put( "/{urn}", - response_model=score_calibration.ScoreCalibrationWithScoreSetUrn, + response_model=score_calibration.ScoreCalibrationDetailWithScoreSetUrn, responses={404: {}, 422: {"description": "Validation Error"}}, openapi_extra={ "requestBody": { @@ -642,7 +642,7 @@ async def delete_score_calibration_route( @router.post( "/{urn}/promote-to-primary", - response_model=score_calibration.ScoreCalibrationWithScoreSetUrn, + response_model=score_calibration.ScoreCalibrationDetailWithScoreSetUrn, responses={404: {}}, ) async def promote_score_calibration_to_primary_route( @@ -715,7 +715,7 @@ async def promote_score_calibration_to_primary_route( @router.post( "/{urn}/demote-from-primary", - response_model=score_calibration.ScoreCalibrationWithScoreSetUrn, + response_model=score_calibration.ScoreCalibrationDetailWithScoreSetUrn, responses={404: {}}, ) def demote_score_calibration_from_primary_route( @@ -754,7 +754,7 @@ def demote_score_calibration_from_primary_route( @router.post( "/{urn}/publish", - response_model=score_calibration.ScoreCalibrationWithScoreSetUrn, + response_model=score_calibration.ScoreCalibrationDetailWithScoreSetUrn, responses={404: {}}, ) def publish_score_calibration_route( diff --git a/src/mavedb/view_models/score_calibration.py b/src/mavedb/view_models/score_calibration.py index 87cde6f5a..3406e2b6f 100644 --- a/src/mavedb/view_models/score_calibration.py +++ b/src/mavedb/view_models/score_calibration.py @@ -507,7 +507,9 @@ class SavedScoreCalibration(ScoreCalibrationBase): private: bool = True functional_classifications: Optional[Sequence[SavedFunctionalClassification]] = None - controls: Sequence[SavedCalibrationControl] = [] + # The full controls list lives on the detail models; the base (and any list/collection response) + # carries only the count to keep those payloads small. See ScoreCalibrationDetailWithScoreSetUrn. + controls_count: int = 0 threshold_sources: Sequence[SavedPublicationIdentifier] evidence_sources: Sequence[SavedPublicationIdentifier] method_sources: Sequence[SavedPublicationIdentifier] @@ -577,6 +579,7 @@ class ScoreCalibration(SavedScoreCalibration): """Complete score calibration model returned by the API.""" functional_classifications: Optional[Sequence[FunctionalClassification]] = None + controls: Sequence[SavedCalibrationControl] = [] threshold_sources: Sequence[PublicationIdentifier] evidence_sources: Sequence[PublicationIdentifier] method_sources: Sequence[PublicationIdentifier] @@ -585,7 +588,11 @@ class ScoreCalibration(SavedScoreCalibration): class ScoreCalibrationWithScoreSetUrn(SavedScoreCalibration): - """Complete score calibration model returned by the API, with score_set_urn.""" + """Score calibration model with score_set_urn, used for list/collection responses. + + Carries ``controls_count`` (from the base) but not the full controls list — see + ``ScoreCalibrationDetailWithScoreSetUrn`` for the single-item detail representation. + """ score_set_urn: str @@ -599,3 +606,9 @@ def generate_score_set_urn(cls, data: Any): f"Unable to coerce score set urn for {cls.__name__}: {exc}." # type: ignore ) return data + + +class ScoreCalibrationDetailWithScoreSetUrn(ScoreCalibrationWithScoreSetUrn): + """Single-calibration detail response: adds the full controls list to the list model.""" + + controls: Sequence[SavedCalibrationControl] = [] diff --git a/tests/helpers/constants.py b/tests/helpers/constants.py index faabe6d18..fb36eee2b 100644 --- a/tests/helpers/constants.py +++ b/tests/helpers/constants.py @@ -1692,6 +1692,8 @@ "primary": True, "private": False, "scoreSetId": 1, + "controls": [], + "controlsCount": 0, "createdBy": { "recordType": "User", "firstName": TEST_USER["first_name"], diff --git a/tests/routers/test_calibration_controls.py b/tests/routers/test_calibration_controls.py index 6d7fc55ba..660be6e97 100644 --- a/tests/routers/test_calibration_controls.py +++ b/tests/routers/test_calibration_controls.py @@ -231,6 +231,29 @@ def test_get_calibration_returns_controls( assert response.json()["controls"][0]["variantUrn"] == variant_urns[0] +@pytest.mark.parametrize("mock_publication_fetch", [CALIBRATION_PUBLICATIONS], indirect=["mock_publication_fetch"]) +def test_list_endpoint_reports_controls_count_without_full_controls( + client, setup_router_db, mock_publication_fetch, session, data_provider, data_files +): + score_set, variant_urns = _score_set_with_variant_urns(client, session, data_provider, data_files) + _create_with_controls( + client, + score_set["urn"], + [ + {"variantUrn": variant_urns[0], "clinicalStatus": "pathogenic"}, + {"variantUrn": variant_urns[1], "clinicalStatus": "benign"}, + ], + ) + + response = client.get(f"/api/v1/score-calibrations/score-set/{score_set['urn']}") + + assert response.status_code == 200, response.text + item = response.json()[0] + # List responses carry the count, not the full controls list. + assert item["controlsCount"] == 2 + assert "controls" not in item + + @pytest.mark.parametrize("mock_publication_fetch", [CALIBRATION_PUBLICATIONS], indirect=["mock_publication_fetch"]) def test_modify_replaces_controls(client, setup_router_db, mock_publication_fetch, session, data_provider, data_files): score_set, variant_urns = _score_set_with_variant_urns(client, session, data_provider, data_files) From 8eee8e9b99191feeeef72722108ef420b4833fad Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Thu, 17 Sep 2026 09:54:16 -0700 Subject: [PATCH 154/252] feat(calibration): model disease as a MONDO-coded concept (#754) Replace the free-text calibration disease column with a MONDO-coded disease_term FK into a new mondo_terms table, served on the wire as a GA4GH MappableConcept. The API input is the bare MONDO code (e.g. "MONDO:0015263"); the server resolves the canonical label and system against EBI OLS4 and get-or-creates the term on save. A null code resolves to the seeded generic "disease or disorder" root (MONDO:0000001), so the FK is never null. Validation against OLS is mandatory: an unknown code is rejected and an OLS outage raises MondoServiceError (503) rather than persisting an unvalidated label. Add a /diseases/search proxy over OLS4's /select endpoint to back the disease typeahead. The migration seeds the generic term and backfills existing calibrations before enforcing NOT NULL. Covered by unit tests that mock OLS, plus a network-marked canary suite exercising the real OLS4 service so upstream shape changes surface. --- ...add_calibration_controls_table_and_phi_.py | 54 ++++- src/mavedb/lib/exceptions.py | 6 + src/mavedb/lib/mondo.py | 203 ++++++++++++++++++ src/mavedb/lib/score_calibrations.py | 7 + src/mavedb/models/__init__.py | 1 + src/mavedb/models/mondo_term.py | 39 ++++ src/mavedb/models/score_calibration.py | 9 +- src/mavedb/routers/diseases.py | 42 ++++ src/mavedb/server_main.py | 12 ++ src/mavedb/view_models/score_calibration.py | 28 ++- tests/helpers/constants.py | 13 ++ tests/helpers/mocks/factories.py | 12 +- tests/lib/annotation/test_contribution.py | 2 + tests/lib/mondo/__init__.py | 0 tests/lib/mondo/network/__init__.py | 0 tests/lib/mondo/network/test_mondo.py | 54 +++++ tests/lib/mondo/test_mondo.py | 199 +++++++++++++++++ tests/lib/test_calibration_control.py | 2 + tests/routers/test_diseases.py | 51 +++++ tests/routers/test_score_calibrations.py | 67 ++++++ tests/routers/test_variant.py | 2 + tests/scripts/conftest_optional.py | 2 + tests/view_models/test_calibration_control.py | 15 +- 23 files changed, 806 insertions(+), 14 deletions(-) create mode 100644 src/mavedb/lib/mondo.py create mode 100644 src/mavedb/models/mondo_term.py create mode 100644 src/mavedb/routers/diseases.py create mode 100644 tests/lib/mondo/__init__.py create mode 100644 tests/lib/mondo/network/__init__.py create mode 100644 tests/lib/mondo/network/test_mondo.py create mode 100644 tests/lib/mondo/test_mondo.py create mode 100644 tests/routers/test_diseases.py diff --git a/alembic/versions/024370c4bca7_add_calibration_controls_table_and_phi_.py b/alembic/versions/024370c4bca7_add_calibration_controls_table_and_phi_.py index 1743d8ff6..b0999f4ef 100644 --- a/alembic/versions/024370c4bca7_add_calibration_controls_table_and_phi_.py +++ b/alembic/versions/024370c4bca7_add_calibration_controls_table_and_phi_.py @@ -1,4 +1,4 @@ -"""add calibration controls table and phi/disease columns +"""add calibration controls, mondo terms, and phi/disease columns Revision ID: 024370c4bca7 Revises: a7f3c2e9b104 @@ -16,6 +16,11 @@ branch_labels = None depends_on = None +# The generic "disease or disorder" MONDO term seeded so the non-nullable calibration FK always resolves. +MONDO_SYSTEM = "https://purl.obolibrary.org/obo/mondo.owl" +MONDO_GENERIC_CODE = "MONDO:0000001" +MONDO_GENERIC_LABEL = "disease or disorder" + def upgrade(): op.create_table( @@ -50,13 +55,56 @@ def upgrade(): op.f("ix_calibration_controls_modified_by_id"), "calibration_controls", ["modified_by_id"], unique=False ) - op.add_column("score_calibrations", sa.Column("disease", sa.String(), nullable=True)) + # MONDO disease terms, with the generic root seeded as the default for calibrations naming no disease. + op.create_table( + "mondo_terms", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("code", sa.String(), nullable=False), + sa.Column("system", sa.String(), nullable=False), + sa.Column("system_version", sa.String(), nullable=True), + sa.Column("label", sa.String(), nullable=False), + sa.Column("creation_date", sa.Date(), nullable=False), + sa.Column("modification_date", sa.Date(), nullable=False), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("system", "code", name="uq_mondo_terms_system_code"), + ) + op.execute( + sa.text( + "INSERT INTO mondo_terms (code, system, label, creation_date, modification_date) " + "VALUES (:code, :system, :label, CURRENT_DATE, CURRENT_DATE)" + ).bindparams(code=MONDO_GENERIC_CODE, system=MONDO_SYSTEM, label=MONDO_GENERIC_LABEL) + ) + + # Disease FK: add nullable, backfill existing calibrations to the generic term, then enforce NOT NULL. + op.add_column("score_calibrations", sa.Column("disease_term_id", sa.Integer(), nullable=True)) + op.execute( + sa.text( + "UPDATE score_calibrations SET disease_term_id = " + "(SELECT id FROM mondo_terms WHERE system = :system AND code = :code)" + ).bindparams(system=MONDO_SYSTEM, code=MONDO_GENERIC_CODE) + ) + op.alter_column("score_calibrations", "disease_term_id", nullable=False) + op.create_index( + op.f("ix_score_calibrations_disease_term_id"), "score_calibrations", ["disease_term_id"], unique=False + ) + op.create_foreign_key( + "fk_score_calibrations_disease_term_id_mondo_terms", + "score_calibrations", + "mondo_terms", + ["disease_term_id"], + ["id"], + ) + op.add_column("score_calibrations", sa.Column("controls_not_phi", sa.Boolean(), nullable=True)) def downgrade(): op.drop_column("score_calibrations", "controls_not_phi") - op.drop_column("score_calibrations", "disease") + + op.drop_constraint("fk_score_calibrations_disease_term_id_mondo_terms", "score_calibrations", type_="foreignkey") + op.drop_index(op.f("ix_score_calibrations_disease_term_id"), table_name="score_calibrations") + op.drop_column("score_calibrations", "disease_term_id") + op.drop_table("mondo_terms") op.drop_index(op.f("ix_calibration_controls_modified_by_id"), table_name="calibration_controls") op.drop_index(op.f("ix_calibration_controls_created_by_id"), table_name="calibration_controls") diff --git a/src/mavedb/lib/exceptions.py b/src/mavedb/lib/exceptions.py index 8b8633f1f..e0c079a67 100644 --- a/src/mavedb/lib/exceptions.py +++ b/src/mavedb/lib/exceptions.py @@ -222,6 +222,12 @@ class HGNCServiceError(Exception): pass +class MondoServiceError(Exception): + """Raised when the MONDO/OLS disease ontology service cannot provide search results.""" + + pass + + class LDHSubmissionFailureError(Exception): """Raised when submission to ClinGen Linked Data Hub (LDH) fails for all submissions.""" diff --git a/src/mavedb/lib/mondo.py b/src/mavedb/lib/mondo.py new file mode 100644 index 000000000..dbabbf01d --- /dev/null +++ b/src/mavedb/lib/mondo.py @@ -0,0 +1,203 @@ +"""MONDO disease-term resolution and search. + +Calibrations carry a disease/disorder context drawn from the Monarch Disease Ontology (MONDO). This +module owns the MONDO vocabulary integration: get-or-create of :class:`MondoTerm` rows (mirroring +:func:`mavedb.lib.taxonomies.find_or_create_taxonomy`), a typeahead search + validation against the +EBI OLS4 service, and conversion of a stored term to the GA4GH ``MappableConcept`` served on the wire. + +The generic term (``MONDO:0000001``, "disease or disorder") is the write-time default for a calibration +with no specific disease, so ``disease_term_id`` is never null. The annotation layer serializes whatever +term a calibration carries — generic or specific — as its VA-Spec disease condition (see +:func:`mavedb.lib.annotation.condition.calibration_disease_condition`). +""" + +import logging +from typing import Any, Optional, TypedDict + +import httpx +from ga4gh.core.models import Coding, MappableConcept, iriReference +from sqlalchemy.orm import Session + +from mavedb.lib.exceptions import MondoServiceError +from mavedb.lib.logging.context import format_raised_exception_info_as_dict, logging_context, save_to_logging_context +from mavedb.lib.validation.exceptions import ValidationError +from mavedb.models.mondo_term import MondoTerm + +logger = logging.getLogger(__name__) + +# The Monarch Disease Ontology (MONDO), the controlled vocabulary for disease terms used across MaveDB. +MONDO_SYSTEM = "https://purl.obolibrary.org/obo/mondo.owl" +# EBI Ontology Lookup Service (OLS4) — the source for MONDO term search and validation. +OLS_BASE_URL = "https://www.ebi.ac.uk/ols4/api" +MONDO_ONTOLOGY = "mondo" +# OLS can be slow; the default httpx 5s timeout is too tight for its autocomplete on short queries. +OLS_TIMEOUT = 10.0 + +# The generic "disease or disorder" root, used when a calibration names no specific disease. +MONDO_GENERIC_CODE = "MONDO:0000001" +MONDO_GENERIC_LABEL = "disease or disorder" + +DEFAULT_SEARCH_LIMIT = 20 + + +class MondoSuggestion(TypedDict): + """A single MONDO term resolved from OLS: the CURIE code, its label, and its resolvable IRI.""" + + code: str + label: str + iri: str + + +def mondo_iri(code: str) -> str: + """The resolvable OBO IRI for a MONDO CURIE (e.g. ``MONDO:0015263`` → ``.../obo/MONDO_0015263``).""" + return f"https://purl.obolibrary.org/obo/{code.replace(':', '_')}" + + +def mondo_term_to_mappable_concept(term: MondoTerm) -> MappableConcept: + """Serialize a stored :class:`MondoTerm` as a GA4GH disease ``MappableConcept``.""" + return MappableConcept( + conceptType="Disease", + name=term.label, + primaryCoding=Coding( + code=term.code, + system=term.system, + systemVersion=term.system_version, + iris=[iriReference(root=mondo_iri(term.code))], + ), + ) + + +def mondo_suggestion_to_mappable_concept(suggestion: MondoSuggestion) -> MappableConcept: + """Build a disease ``MappableConcept`` from an OLS search suggestion (for typeahead results).""" + return MappableConcept( + conceptType="Disease", + name=suggestion["label"], + primaryCoding=Coding( + code=suggestion["code"], + system=MONDO_SYSTEM, + iris=[iriReference(root=suggestion["iri"])], + ), + ) + + +def generic_disease_mappable_concept() -> MappableConcept: + """The generic "disease or disorder" concept, MaveDB's single unspecified-disease sentinel.""" + return MappableConcept( + conceptType="Disease", + name=MONDO_GENERIC_LABEL, + primaryCoding=Coding( + code=MONDO_GENERIC_CODE, + system=MONDO_SYSTEM, + iris=[iriReference(root=mondo_iri(MONDO_GENERIC_CODE))], + ), + ) + + +async def search_mondo(query: str, limit: int = DEFAULT_SEARCH_LIMIT) -> list[MondoSuggestion]: + """Search MONDO disease classes by free text via OLS, for the disease typeahead.""" + if not query or not query.strip(): + return [] + + # OLS4's /select endpoint is the autocomplete-optimized path; /search times out on broad prefixes + # (e.g. a two-letter query), which is often what a typeahead sends. + params: dict[str, Any] = { + "q": query.strip(), + "ontology": MONDO_ONTOLOGY, + "rows": limit, + "fieldList": "iri,label,obo_id", + } + save_to_logging_context({"mondo_search_query": query, "mondo_search_limit": limit}) + try: + async with httpx.AsyncClient(timeout=OLS_TIMEOUT) as client: + response = await client.get(f"{OLS_BASE_URL}/select", params=params) + response.raise_for_status() + docs = response.json().get("response", {}).get("docs", []) + except httpx.HTTPError as exc: + save_to_logging_context(format_raised_exception_info_as_dict(exc)) + logger.error(msg="MONDO/OLS search request failed.", exc_info=exc, extra=logging_context()) + raise MondoServiceError("Disease ontology service temporarily unavailable") from exc + + return [ + MondoSuggestion(code=doc["obo_id"], label=doc["label"], iri=doc.get("iri", mondo_iri(doc["obo_id"]))) + for doc in docs + if doc.get("obo_id") and doc.get("label") + ] + + +async def fetch_mondo_term(code: str) -> Optional[MondoSuggestion]: + """Resolve one MONDO CURIE to its canonical label + IRI via OLS, or None if it does not exist.""" + params: dict[str, Any] = { + "q": code, + "ontology": MONDO_ONTOLOGY, + "queryFields": "obo_id", + "exact": "true", + "rows": 1, + "fieldList": "iri,label,obo_id", + } + try: + async with httpx.AsyncClient(timeout=OLS_TIMEOUT) as client: + response = await client.get(f"{OLS_BASE_URL}/search", params=params) + response.raise_for_status() + docs = response.json().get("response", {}).get("docs", []) + except httpx.HTTPError as exc: + save_to_logging_context(format_raised_exception_info_as_dict(exc)) + logger.error(msg="MONDO/OLS term lookup failed.", exc_info=exc, extra=logging_context()) + raise MondoServiceError("Disease ontology service temporarily unavailable") from exc + + if not docs: + return None + + doc = docs[0] + return MondoSuggestion(code=doc["obo_id"], label=doc["label"], iri=doc.get("iri", mondo_iri(code))) + + +def get_generic_disease_term(db: Session) -> MondoTerm: + """Get-or-create the generic disease term. Known-canonical, so it needs no OLS round trip.""" + term = ( + db.query(MondoTerm).filter(MondoTerm.system == MONDO_SYSTEM, MondoTerm.code == MONDO_GENERIC_CODE).one_or_none() + ) + if term is None: + term = MondoTerm(code=MONDO_GENERIC_CODE, system=MONDO_SYSTEM, label=MONDO_GENERIC_LABEL) # type: ignore[call-arg] + db.add(term) + db.flush() + + return term + + +async def find_or_create_mondo_term(db: Session, code: str) -> MondoTerm: + """Find an existing MONDO term by code, or create one after validating it against OLS. + + A new code is validated against OLS and its canonical label stored. An unknown code raises + ``ValidationError``; an OLS outage raises ``MondoServiceError`` (via :func:`fetch_mondo_term`) so a + term is never persisted without a validated label — a controlled vocabulary must stay validated. + """ + # The generic term is canonical and seeded — resolve it directly, never via OLS. + if code == MONDO_GENERIC_CODE: + return get_generic_disease_term(db) + + term = db.query(MondoTerm).filter(MondoTerm.system == MONDO_SYSTEM, MondoTerm.code == code).one_or_none() + if term is not None: + return term + + resolved = await fetch_mondo_term(code) + if resolved is None: + raise ValidationError(f"'{code}' is not a valid MONDO disease term.", custom_loc=["body", "disease"]) + + term = MondoTerm(code=resolved["code"], system=MONDO_SYSTEM, label=resolved["label"]) # type: ignore[call-arg] + db.add(term) + db.flush() + + return term + + +async def resolve_disease_term(db: Session, code: Optional[str]) -> MondoTerm: + """Resolve a calibration's submitted disease code to a stored MONDO term. + + ``None`` (no disease specified) resolves to the generic "disease or disorder" term; any other code + is validated/created via :func:`find_or_create_mondo_term`. Shared by the score calibration create + and update paths so the write-time default lives in one place. + """ + if code is None: + return get_generic_disease_term(db) + + return await find_or_create_mondo_term(db, code) diff --git a/src/mavedb/lib/score_calibrations.py b/src/mavedb/lib/score_calibrations.py index 73092fc69..5d270e043 100644 --- a/src/mavedb/lib/score_calibrations.py +++ b/src/mavedb/lib/score_calibrations.py @@ -10,6 +10,7 @@ from mavedb.lib.acmg import find_or_create_acmg_classification from mavedb.lib.identifiers import find_or_create_publication_identifier +from mavedb.lib.mondo import resolve_disease_term from mavedb.lib.types.score_calibrations import ClassificationDict from mavedb.lib.validation.constants.general import ( calibration_class_column_name, @@ -284,6 +285,7 @@ async def _create_score_calibration( exclude={ "functional_classifications", "controls", + "disease", "threshold_sources", "evidence_sources", "method_sources", @@ -296,6 +298,8 @@ async def _create_score_calibration( modified_by=user, ) # type: ignore[call-arg] + calibration.disease_term = await resolve_disease_term(db, getattr(calibration_create, "disease", None)) + if containing_score_set: calibration.score_set = containing_score_set calibration.score_set_id = containing_score_set.id @@ -567,6 +571,7 @@ async def modify_score_calibration( if attr not in { "functional_classifications", "controls", + "disease", # controls_not_phi carries re-acknowledgment semantics; set it explicitly so an # unrelated edit cannot silently wipe a prior affirmation. "controls_not_phi", @@ -581,6 +586,8 @@ async def modify_score_calibration( }: setattr(calibration, attr, value) + calibration.disease_term = await resolve_disease_term(db, getattr(calibration_update, "disease", None)) + calibration.score_set = containing_score_set calibration.score_set_id = containing_score_set.id calibration.publication_identifier_associations = updated_assocs diff --git a/src/mavedb/models/__init__.py b/src/mavedb/models/__init__.py index ba5334e99..2da3e40b7 100644 --- a/src/mavedb/models/__init__.py +++ b/src/mavedb/models/__init__.py @@ -17,6 +17,7 @@ "legacy_keyword", "license", "mapped_variant", + "mondo_term", "pipeline", "publication_identifier", "published_variant", diff --git a/src/mavedb/models/mondo_term.py b/src/mavedb/models/mondo_term.py new file mode 100644 index 000000000..7059e1b0a --- /dev/null +++ b/src/mavedb/models/mondo_term.py @@ -0,0 +1,39 @@ +"""SQLAlchemy model for MONDO disease terms. + +A ``MondoTerm`` is a single controlled-vocabulary entry (a MONDO code + its label) referenced by +calibrations as their disease/disorder context. The columns are deliberately the generic coding +columns (``system``/``code``/``system_version``/``label``) rather than MONDO-specific ones: this table +is the narrow, first consumer of a shared concept model, and matching the column shape of +``controlled_keywords`` (and the eventual ``mappable_concepts`` table) keeps a later promotion additive +rather than a reshape. On the wire a term is served as a GA4GH ``MappableConcept``. +""" + +from datetime import date +from typing import Optional + +from sqlalchemy import Date, Integer, String, UniqueConstraint +from sqlalchemy.orm import Mapped, mapped_column + +from mavedb.db.base import Base + + +class MondoTerm(Base): + __tablename__ = "mondo_terms" + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + + # The coding: ``code`` is the MONDO CURIE (e.g. "MONDO:0015263"), ``system`` its terminology + # identifier. ``system`` is constant for MONDO today but stored explicitly so a promotion to a + # multi-system concept table needs no backfill. ``UNIQUE(system, code)`` dedupes shared terms. + code: Mapped[str] = mapped_column(String, nullable=False) + system: Mapped[str] = mapped_column(String, nullable=False) + system_version: Mapped[Optional[str]] = mapped_column(String, nullable=True) + label: Mapped[str] = mapped_column(String, nullable=False) + + creation_date: Mapped[date] = mapped_column(Date, nullable=False, default=date.today) + modification_date: Mapped[date] = mapped_column(Date, nullable=False, default=date.today, onupdate=date.today) + + __table_args__ = (UniqueConstraint("system", "code", name="uq_mondo_terms_system_code"),) + + def __repr__(self) -> str: # pragma: no cover - repr utility + return f"" diff --git a/src/mavedb/models/score_calibration.py b/src/mavedb/models/score_calibration.py index d32b3ed9a..e2c1fec7e 100644 --- a/src/mavedb/models/score_calibration.py +++ b/src/mavedb/models/score_calibration.py @@ -17,6 +17,7 @@ from mavedb.models.score_calibration_publication_identifier import ScoreCalibrationPublicationIdentifierAssociation if TYPE_CHECKING: + from mavedb.models.mondo_term import MondoTerm from mavedb.models.publication_identifier import PublicationIdentifier from mavedb.models.score_set import ScoreSet from mavedb.models.user import User @@ -42,10 +43,10 @@ class ScoreCalibration(Base): baseline_score = Column(Float, nullable=True) baseline_score_description = Column(String, nullable=True) - # Optional free-text disease/disorder context for the controls, at the calibration level. Deliberately not a - # controlled vocabulary and never recorded per-control: a per-control disease label could let a phenotype be - # inferred for an individual variant, which risks PHI. - disease = Column(String, nullable=True) + # MONDO-coded disease/disorder context. Non-nullable and defaulted to the generic "disease or disorder" term, + # so every calibration carries a concept and consumers never branch on null. + disease_term_id = Column(Integer, ForeignKey("mondo_terms.id"), nullable=False, index=True) + disease_term: Mapped["MondoTerm"] = relationship("MondoTerm") # Submitter's affirmation that the control data contains no PHI. Tristate on purpose: # None = not yet addressed, # False = explicitly declined, True = affirmed. Publishing a calibration that has controls is gated on this being diff --git a/src/mavedb/routers/diseases.py b/src/mavedb/routers/diseases.py new file mode 100644 index 000000000..6f2d5e70c --- /dev/null +++ b/src/mavedb/routers/diseases.py @@ -0,0 +1,42 @@ +from typing import Any + +from fastapi import APIRouter, Query +from ga4gh.core.models import MappableConcept + +from mavedb.lib.logging.context import save_to_logging_context +from mavedb.lib.logging.logged_route import LoggedRoute +from mavedb.lib.mondo import DEFAULT_SEARCH_LIMIT, mondo_suggestion_to_mappable_concept, search_mondo +from mavedb.routers.shared import GATEWAY_ERROR_RESPONSES, PUBLIC_ERROR_RESPONSES, ROUTER_BASE_PREFIX + +TAG_NAME = "Diseases" + +router = APIRouter( + prefix=f"{ROUTER_BASE_PREFIX}/diseases", + tags=[TAG_NAME], + responses={**PUBLIC_ERROR_RESPONSES}, + route_class=LoggedRoute, +) + +metadata = { + "name": TAG_NAME, + "description": "Search disease terms from the Monarch Disease Ontology (MONDO).", +} + + +@router.get( + "/search", + status_code=200, + response_model=list[MappableConcept], + summary="Search MONDO disease terms", + responses={**GATEWAY_ERROR_RESPONSES}, +) +async def search_diseases( + *, + q: str = Query(..., description="Free-text query for a MONDO disease term."), + limit: int = Query(DEFAULT_SEARCH_LIMIT, ge=1, le=100, description="Maximum number of results."), +) -> Any: + """Typeahead search for MONDO disease terms, returned as GA4GH ``MappableConcept`` suggestions.""" + save_to_logging_context({"query": q, "limit": limit}) + + suggestions = await search_mondo(q, limit) + return [mondo_suggestion_to_mappable_concept(suggestion) for suggestion in suggestions] diff --git a/src/mavedb/server_main.py b/src/mavedb/server_main.py index 880bfcfe1..be83c0d6f 100644 --- a/src/mavedb/server_main.py +++ b/src/mavedb/server_main.py @@ -25,6 +25,7 @@ HGNCGeneNotFoundError, HGNCServiceError, MixedTargetError, + MondoServiceError, NonexistentIdentifierError, ) from mavedb.lib.logging.canonical import log_request @@ -44,6 +45,7 @@ api_information, collections, controlled_keywords, + diseases, doi_identifiers, experiment_sets, experiments, @@ -100,6 +102,7 @@ app.include_router(api_information.router) app.include_router(collections.router) app.include_router(controlled_keywords.router) +app.include_router(diseases.router) app.include_router(doi_identifiers.router) app.include_router(experiment_sets.router) app.include_router(experiments.router) @@ -178,6 +181,14 @@ async def hgnc_service_error_exception_handler(request: Request, exc: HGNCServic return response +@app.exception_handler(MondoServiceError) +async def mondo_service_error_exception_handler(request: Request, exc: MondoServiceError): + response = JSONResponse(status_code=503, content={"message": "Disease ontology service temporarily unavailable"}) + save_to_logging_context(format_raised_exception_info_as_dict(exc)) + log_request(request, response, time.time_ns()) + return response + + @app.exception_handler(EutilsRequestError) async def nonexistent_pmid_error_exception_handler(request: Request, exc: EutilsRequestError): response = JSONResponse(status_code=404, content={"message": str(exc)}) @@ -248,6 +259,7 @@ def customize_openapi_schema(): api_information.metadata, collections.metadata, controlled_keywords.metadata, + diseases.metadata, doi_identifiers.metadata, experiment_sets.metadata, experiments.metadata, diff --git a/src/mavedb/view_models/score_calibration.py b/src/mavedb/view_models/score_calibration.py index 3406e2b6f..5d5e6337e 100644 --- a/src/mavedb/view_models/score_calibration.py +++ b/src/mavedb/view_models/score_calibration.py @@ -7,8 +7,10 @@ from datetime import date from typing import TYPE_CHECKING, Any, Collection, Optional, Sequence, Union +from ga4gh.core.models import MappableConcept from pydantic import Field, field_validator, model_validator +from mavedb.lib.mondo import generic_disease_mappable_concept, mondo_term_to_mappable_concept from mavedb.lib.oddspaths import oddspaths_evidence_strength_equivalent from mavedb.lib.validation.exceptions import ValidationError from mavedb.lib.validation.transform import ( @@ -286,7 +288,6 @@ class ScoreCalibrationBase(BaseModel): baseline_score_description: Optional[str] = None notes: Optional[str] = None - disease: Optional[str] = None controls_not_phi: Optional[bool] = None functional_classifications: Optional[Sequence[FunctionalClassificationBase]] = None @@ -441,6 +442,13 @@ class ScoreCalibrationModify(ScoreCalibrationBase): functional_classifications: Optional[Sequence[FunctionalClassificationModify]] = None # None means "no change" on modify; an empty list clears all controls. controls: Optional[Sequence[CalibrationControlCreate]] = None + disease: Optional[str] = Field( + None, + description=( + 'The MONDO code (e.g. "MONDO:0015263") for this calibration\'s disease context; validated ' + 'against OLS. Omitted or null resolves to the generic "disease or disorder" term.' + ), + ) threshold_sources: Sequence[PublicationIdentifierCreate] evidence_sources: Sequence[PublicationIdentifierCreate] method_sources: Sequence[PublicationIdentifierCreate] @@ -510,6 +518,9 @@ class SavedScoreCalibration(ScoreCalibrationBase): # The full controls list lives on the detail models; the base (and any list/collection response) # carries only the count to keep those payloads small. See ScoreCalibrationDetailWithScoreSetUrn. controls_count: int = 0 + # The FK is non-nullable, so this is always populated by ``generate_disease_concept`` when building + # from an ORM object; an unspecified disease resolves to the generic "disease or disorder" concept. + disease: MappableConcept threshold_sources: Sequence[SavedPublicationIdentifier] evidence_sources: Sequence[SavedPublicationIdentifier] method_sources: Sequence[SavedPublicationIdentifier] @@ -574,6 +585,21 @@ def generate_threshold_classification_and_method_sources(cls, data: Any): # typ ) return data + @model_validator(mode="before") + def generate_disease_concept(cls, data: Any): + """Serialize the stored MONDO term as a ``disease`` MappableConcept when building from an ORM object. + + Falls back to the generic concept if the relationship is unexpectedly empty, so the non-nullable + field is always satisfied. + """ + if hasattr(data, "disease_term"): + term = data.disease_term + data.__setattr__( + "disease", + mondo_term_to_mappable_concept(term) if term is not None else generic_disease_mappable_concept(), + ) + return data + class ScoreCalibration(SavedScoreCalibration): """Complete score calibration model returned by the API.""" diff --git a/tests/helpers/constants.py b/tests/helpers/constants.py index fb36eee2b..53e0a4e5e 100644 --- a/tests/helpers/constants.py +++ b/tests/helpers/constants.py @@ -1652,6 +1652,17 @@ } +SAVED_GENERIC_DISEASE_CONCEPT = { + "conceptType": "Disease", + "name": "disease or disorder", + "primaryCoding": { + "system": "https://purl.obolibrary.org/obo/mondo.owl", + "code": "MONDO:0000001", + "iris": ["https://purl.obolibrary.org/obo/MONDO_0000001"], + }, +} + + TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED = { "title": "Test BRNICH Score Calibration", "research_use_only": False, @@ -1694,6 +1705,7 @@ "scoreSetId": 1, "controls": [], "controlsCount": 0, + "disease": SAVED_GENERIC_DISEASE_CONCEPT, "createdBy": { "recordType": "User", "firstName": TEST_USER["first_name"], @@ -1764,6 +1776,7 @@ "private": False, "urn": VALID_CALIBRATION_URN, "scoreSetId": 1, + "disease": SAVED_GENERIC_DISEASE_CONCEPT, "createdBy": { "recordType": "User", "firstName": TEST_USER["first_name"], diff --git a/tests/helpers/mocks/factories.py b/tests/helpers/mocks/factories.py index 6e83d089c..c78532695 100644 --- a/tests/helpers/mocks/factories.py +++ b/tests/helpers/mocks/factories.py @@ -8,6 +8,7 @@ from datetime import date, datetime from unittest.mock import MagicMock +from mavedb.lib.mondo import MONDO_GENERIC_CODE, MONDO_GENERIC_LABEL, MONDO_SYSTEM from mavedb.models.enums.acmg_criterion import ACMGCriterion from mavedb.models.enums.functional_classification import FunctionalClassification as FunctionalClassificationOptions from mavedb.models.enums.strength_of_evidence import StrengthOfEvidenceProvided @@ -169,11 +170,19 @@ def create_mock_score_set( ) +def create_mock_mondo_term( + code=MONDO_GENERIC_CODE, label=MONDO_GENERIC_LABEL, system=MONDO_SYSTEM, system_version=None +): + """Create a mock MondoTerm, defaulting to the generic "disease or disorder" term.""" + return create_sealed_mock(code=code, label=label, system=system, system_version=system_version) + + def create_mock_score_calibration(functional_classifications=None, primary=True, **kwargs): """Create a mock ScoreCalibration for both Pydantic validation and direct attribute access. When ``functional_classifications`` is not provided a minimal default list is - created (single not_specified classification with no ACMG). + created (single not_specified classification with no ACMG). ``disease_term`` defaults to + the generic MONDO term (calibrations always carry a non-null disease FK). """ user = kwargs.get("created_by") or create_mock_user() @@ -207,6 +216,7 @@ def create_mock_score_calibration(functional_classifications=None, primary=True, "primary": primary, "private": kwargs.get("private", not primary), "researchUseOnly": kwargs.get("research_use_only", False), + "diseaseTerm": kwargs.get("disease_term", create_mock_mondo_term()), "thresholdSources": kwargs.get("threshold_sources", []), "classificationSources": kwargs.get("classification_sources", []), "methodSources": kwargs.get("method_sources", []), diff --git a/tests/lib/annotation/test_contribution.py b/tests/lib/annotation/test_contribution.py index 51597db37..986d86f15 100644 --- a/tests/lib/annotation/test_contribution.py +++ b/tests/lib/annotation/test_contribution.py @@ -23,6 +23,7 @@ mavedb_score_calibration_contribution, mavedb_vrs_contribution, ) +from mavedb.lib.mondo import get_generic_disease_term from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.user import User from tests.helpers.mocks.factories import ( @@ -326,6 +327,7 @@ def test_contributions_with_real_db_objects(self, session, setup_lib_db_with_map primary=True, investigator_provided=True, private=False, + disease_term=get_generic_disease_term(session), created_by_id=creator.id, modified_by_id=creator.id, ) diff --git a/tests/lib/mondo/__init__.py b/tests/lib/mondo/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/lib/mondo/network/__init__.py b/tests/lib/mondo/network/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/lib/mondo/network/test_mondo.py b/tests/lib/mondo/network/test_mondo.py new file mode 100644 index 000000000..0daaf2866 --- /dev/null +++ b/tests/lib/mondo/network/test_mondo.py @@ -0,0 +1,54 @@ +# ruff: noqa: E402 +"""Network tests for the MONDO/OLS integration. Require a live connection to www.ebi.ac.uk (OLS4). + +The unit tests in ``tests/lib/mondo/test_mondo.py`` mock OLS, so these are the only exposure to real upstream +behavior — a canary for changes to OLS4's endpoints, response shape, or term identity. Sockets are +blocked for unmarked tests, so these run only under the ``network`` marker and are not part of the +default suite. +""" + +import pytest + +# starlette is required for logging context functionality pulled in by mavedb.lib.mondo. +pytest.importorskip("starlette") + +from mavedb.lib.mondo import MONDO_GENERIC_CODE, fetch_mondo_term, search_mondo + +# Brugada syndrome: a stable, long-established MONDO disease term used as a known-good fixture. +BRUGADA_CODE = "MONDO:0015263" + + +@pytest.mark.network +@pytest.mark.asyncio +class TestMondoOlsNetwork: + """Canary tests exercising the real OLS4 service.""" + + async def test_search_returns_known_term_with_expected_shape(self): + # /select endpoint: a known query surfaces the expected code, in the {code, label, iri} shape. + results = await search_mondo("brugada syndrome") + assert results, "OLS returned no results for a known disease query." + assert BRUGADA_CODE in {result["code"] for result in results} + first = results[0] + assert first["code"].startswith("MONDO:") + assert first["label"] + assert first["iri"].startswith("http") + + async def test_search_broad_prefix_returns_results(self): + # A two-letter prefix must still return via /select — /search timed out on these (the #754 switch). + assert await search_mondo("ca") + + async def test_fetch_known_code_returns_canonical_term(self): + # /search exact-obo_id lookup: a known code resolves to its canonical label. + resolved = await fetch_mondo_term(BRUGADA_CODE) + assert resolved is not None + assert resolved["code"] == BRUGADA_CODE + assert "brugada" in resolved["label"].lower() + + async def test_fetch_generic_root_resolves(self): + resolved = await fetch_mondo_term(MONDO_GENERIC_CODE) + assert resolved is not None + assert resolved["code"] == MONDO_GENERIC_CODE + + async def test_fetch_unknown_code_returns_none(self): + # The not-found path must stay a clean None so validation can reject unknown codes. + assert await fetch_mondo_term("MONDO:9999999") is None diff --git a/tests/lib/mondo/test_mondo.py b/tests/lib/mondo/test_mondo.py new file mode 100644 index 000000000..b50550970 --- /dev/null +++ b/tests/lib/mondo/test_mondo.py @@ -0,0 +1,199 @@ +"""Tests for mavedb.lib.mondo: MONDO term resolution, search, and MappableConcept serialization.""" + +# ruff: noqa: E402 + +import pytest + +pytest.importorskip("psycopg2") + +import httpx + +from mavedb.lib.exceptions import MondoServiceError +from mavedb.lib.mondo import ( + MONDO_GENERIC_CODE, + MONDO_GENERIC_LABEL, + MONDO_SYSTEM, + fetch_mondo_term, + find_or_create_mondo_term, + generic_disease_mappable_concept, + get_generic_disease_term, + mondo_iri, + mondo_suggestion_to_mappable_concept, + mondo_term_to_mappable_concept, + search_mondo, +) +from mavedb.lib.validation.exceptions import ValidationError +from mavedb.models.mondo_term import MondoTerm + + +class _FakeResponse: + def __init__(self, payload): + self._payload = payload + + def raise_for_status(self): + pass + + def json(self): + return self._payload + + +class _FakeAsyncClient: + """Minimal stand-in for httpx.AsyncClient returning a canned OLS payload.""" + + def __init__(self, payload): + self._payload = payload + + async def __aenter__(self): + return self + + async def __aexit__(self, *exc): + return False + + async def get(self, url, params=None): + return _FakeResponse(self._payload) + + +class _FailingAsyncClient: + """Stand-in for httpx.AsyncClient whose request fails, as when OLS is unreachable.""" + + async def __aenter__(self): + return self + + async def __aexit__(self, *exc): + return False + + async def get(self, url, params=None): + raise httpx.ConnectError("ols down") + + +def _ols_payload(docs): + return {"response": {"docs": docs}} + + +@pytest.mark.unit +class TestMondoSerialization: + def test_mondo_iri_replaces_curie_colon(self): + assert mondo_iri("MONDO:0015263") == "https://purl.obolibrary.org/obo/MONDO_0015263" + + def test_term_to_mappable_concept(self): + term = MondoTerm( + code="MONDO:0015263", system=MONDO_SYSTEM, system_version="2026-01-01", label="Brugada syndrome" + ) + concept = mondo_term_to_mappable_concept(term) + assert concept.conceptType == "Disease" + assert concept.name == "Brugada syndrome" + assert concept.primaryCoding.code.root == "MONDO:0015263" + assert concept.primaryCoding.system == MONDO_SYSTEM + assert concept.primaryCoding.iris[0].root == "https://purl.obolibrary.org/obo/MONDO_0015263" + + def test_generic_concept_is_the_mondo_root(self): + concept = generic_disease_mappable_concept() + assert concept.primaryCoding.code.root == MONDO_GENERIC_CODE + assert concept.name == MONDO_GENERIC_LABEL + + def test_suggestion_to_mappable_concept(self): + concept = mondo_suggestion_to_mappable_concept( + {"code": "MONDO:0015263", "label": "Brugada syndrome", "iri": "https://example.org/MONDO_0015263"} + ) + assert concept.primaryCoding.code.root == "MONDO:0015263" + assert concept.primaryCoding.iris[0].root == "https://example.org/MONDO_0015263" + + +@pytest.mark.asyncio +class TestMondoOlsSearch: + async def test_search_mondo_parses_ols_docs(self, monkeypatch): + payload = _ols_payload( + [ + { + "obo_id": "MONDO:0015263", + "label": "Brugada syndrome", + "iri": "https://purl.obolibrary.org/obo/MONDO_0015263", + }, + {"obo_id": "MONDO:0000001", "label": "disease or disorder"}, # missing iri → derived + ] + ) + monkeypatch.setattr(httpx, "AsyncClient", lambda *a, **k: _FakeAsyncClient(payload)) + + results = await search_mondo("brugada") + assert results[0] == { + "code": "MONDO:0015263", + "label": "Brugada syndrome", + "iri": "https://purl.obolibrary.org/obo/MONDO_0015263", + } + assert results[1]["iri"] == mondo_iri("MONDO:0000001") + + async def test_search_mondo_empty_query_skips_request(self, monkeypatch): + # No network expected; if a request were made the fake would still be needed, so assert on the shortcut. + assert await search_mondo(" ") == [] + + async def test_fetch_mondo_term_returns_none_when_absent(self, monkeypatch): + monkeypatch.setattr(httpx, "AsyncClient", lambda *a, **k: _FakeAsyncClient(_ols_payload([]))) + assert await fetch_mondo_term("MONDO:9999999") is None + + async def test_search_mondo_wraps_http_errors_in_service_error(self, monkeypatch): + monkeypatch.setattr(httpx, "AsyncClient", lambda *a, **k: _FailingAsyncClient()) + + with pytest.raises(MondoServiceError): + await search_mondo("brugada") + + async def test_fetch_mondo_term_wraps_http_errors_in_service_error(self, monkeypatch): + monkeypatch.setattr(httpx, "AsyncClient", lambda *a, **k: _FailingAsyncClient()) + + with pytest.raises(MondoServiceError): + await fetch_mondo_term("MONDO:0015263") + + +@pytest.mark.asyncio +class TestFindOrCreateMondoTerm: + async def test_creates_from_ols_and_is_idempotent(self, session, monkeypatch): + async def fake_fetch(code): + return {"code": code, "label": "Brugada syndrome", "iri": mondo_iri(code)} + + monkeypatch.setattr("mavedb.lib.mondo.fetch_mondo_term", fake_fetch) + + term = await find_or_create_mondo_term(session, "MONDO:0015263") + assert term.code == "MONDO:0015263" + assert term.label == "Brugada syndrome" + assert term.system == MONDO_SYSTEM + + # A second call reuses the row rather than creating a duplicate. + again = await find_or_create_mondo_term(session, "MONDO:0015263") + assert again.id == term.id + assert session.query(MondoTerm).filter(MondoTerm.code == "MONDO:0015263").count() == 1 + + async def test_generic_code_resolves_without_ols(self, session, monkeypatch): + async def fail_fetch(code): + raise AssertionError("OLS should not be consulted for the generic term") + + monkeypatch.setattr("mavedb.lib.mondo.fetch_mondo_term", fail_fetch) + term = await find_or_create_mondo_term(session, MONDO_GENERIC_CODE) + assert term.code == MONDO_GENERIC_CODE + assert term.label == MONDO_GENERIC_LABEL + + async def test_rejects_unknown_code(self, session, monkeypatch): + async def fake_fetch(code): + return None + + monkeypatch.setattr("mavedb.lib.mondo.fetch_mondo_term", fake_fetch) + with pytest.raises(ValidationError): + await find_or_create_mondo_term(session, "MONDO:9999999") + + async def test_ols_outage_propagates_service_error(self, session, monkeypatch): + # A controlled term is never persisted unvalidated: an OLS outage surfaces as MondoServiceError. + async def fake_fetch(code): + raise MondoServiceError("ols down") + + monkeypatch.setattr("mavedb.lib.mondo.fetch_mondo_term", fake_fetch) + with pytest.raises(MondoServiceError): + await find_or_create_mondo_term(session, "MONDO:0015263") + + +@pytest.mark.unit +class TestGenericDiseaseTerm: + def test_get_generic_disease_term_is_idempotent(self, session): + first = get_generic_disease_term(session) + assert first.code == MONDO_GENERIC_CODE + assert first.system == MONDO_SYSTEM + second = get_generic_disease_term(session) + assert second.id == first.id + assert session.query(MondoTerm).filter(MondoTerm.code == MONDO_GENERIC_CODE).count() == 1 diff --git a/tests/lib/test_calibration_control.py b/tests/lib/test_calibration_control.py index 5637f6286..7a04a1f28 100644 --- a/tests/lib/test_calibration_control.py +++ b/tests/lib/test_calibration_control.py @@ -18,6 +18,7 @@ from mavedb.lib.validation.exceptions import ValidationError from mavedb.models.calibration_control import CalibrationControl from mavedb.models.enums.calibration_control_status import CalibrationControlStatus +from mavedb.lib.mondo import get_generic_disease_term from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.score_set import ScoreSet from mavedb.models.user import User @@ -31,6 +32,7 @@ def _make_calibration(session, score_set_id: int, user: User) -> ScoreCalibratio calibration = ScoreCalibration( title="Calibration with controls", score_set_id=score_set_id, + disease_term=get_generic_disease_term(session), created_by=user, modified_by=user, ) diff --git a/tests/routers/test_diseases.py b/tests/routers/test_diseases.py new file mode 100644 index 000000000..19c3c2bf9 --- /dev/null +++ b/tests/routers/test_diseases.py @@ -0,0 +1,51 @@ +"""Tests for the /diseases/search MONDO typeahead endpoint.""" + +# ruff: noqa: E402 + +import pytest + +pytest.importorskip("arq") +pytest.importorskip("cdot") +pytest.importorskip("fastapi") + +from mavedb.lib.exceptions import MondoServiceError + + +@pytest.fixture +def mock_mondo_search(monkeypatch): + async def fake_search(query, limit=20): + return [ + { + "code": "MONDO:0015263", + "label": "Brugada syndrome", + "iri": "https://purl.obolibrary.org/obo/MONDO_0015263", + } + ] + + monkeypatch.setattr("mavedb.routers.diseases.search_mondo", fake_search) + + +def test_search_diseases_returns_mappable_concepts(client, mock_mondo_search): + response = client.get("/api/v1/diseases/search", params={"q": "brugada"}) + + assert response.status_code == 200 + data = response.json() + assert len(data) == 1 + assert data[0]["conceptType"] == "Disease" + assert data[0]["name"] == "Brugada syndrome" + assert data[0]["primaryCoding"]["code"] == "MONDO:0015263" + + +def test_search_diseases_requires_a_query(client): + assert client.get("/api/v1/diseases/search").status_code == 422 + + +def test_search_diseases_surfaces_ontology_outage(client, monkeypatch): + async def failing_search(query, limit=20): + raise MondoServiceError("Disease ontology service temporarily unavailable") + + monkeypatch.setattr("mavedb.routers.diseases.search_mondo", failing_search) + + response = client.get("/api/v1/diseases/search", params={"q": "brugada"}) + assert response.status_code == 503 + assert "Disease ontology service temporarily unavailable" in response.json()["message"] diff --git a/tests/routers/test_score_calibrations.py b/tests/routers/test_score_calibrations.py index fe1aeba7c..e5bb5c486 100644 --- a/tests/routers/test_score_calibrations.py +++ b/tests/routers/test_score_calibrations.py @@ -14,6 +14,7 @@ from arq import ArqRedis from sqlalchemy import select +from mavedb.models.mondo_term import MondoTerm as MondoTermDbModel from mavedb.models.score_calibration import ScoreCalibration as CalibrationDbModel from mavedb.models.score_set import ScoreSet as ScoreSetDbModel from tests.helpers.constants import ( @@ -148,6 +149,72 @@ def test_creating_user_can_get_score_calibration_when_private( assert calibration_response["private"] is True +@pytest.mark.parametrize( + "mock_publication_fetch", + [ + [ + {"dbName": "PubMed", "identifier": TEST_PUBMED_IDENTIFIER}, + {"dbName": "bioRxiv", "identifier": TEST_BIORXIV_IDENTIFIER}, + ] + ], + indirect=["mock_publication_fetch"], +) +def test_create_score_calibration_without_disease_defaults_to_generic( + client, setup_router_db, mock_publication_fetch, session, data_provider, data_files +): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + calibration = create_test_score_calibration_in_score_set_via_client( + client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) + ) + + disease = client.get(f"/api/v1/score-calibrations/{calibration['urn']}").json()["disease"] + assert disease["conceptType"] == "Disease" + assert disease["primaryCoding"]["code"] == "MONDO:0000001" + + +@pytest.mark.parametrize( + "mock_publication_fetch", + [ + [ + {"dbName": "PubMed", "identifier": TEST_PUBMED_IDENTIFIER}, + {"dbName": "bioRxiv", "identifier": TEST_BIORXIV_IDENTIFIER}, + ] + ], + indirect=["mock_publication_fetch"], +) +def test_create_score_calibration_with_disease_resolves_mondo_concept( + client, setup_router_db, mock_publication_fetch, session, data_provider, data_files, monkeypatch +): + async def fake_fetch(code): + return { + "code": code, + "label": "Brugada syndrome", + "iri": f"https://purl.obolibrary.org/obo/{code.replace(':', '_')}", + } + + monkeypatch.setattr("mavedb.lib.mondo.fetch_mondo_term", fake_fetch) + + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + payload = { + **deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED), + "disease": "MONDO:0015263", + } + calibration = create_test_score_calibration_in_score_set_via_client(client, score_set["urn"], payload) + + disease = client.get(f"/api/v1/score-calibrations/{calibration['urn']}").json()["disease"] + assert disease["primaryCoding"]["code"] == "MONDO:0015263" + assert disease["name"] == "Brugada syndrome" + + stored = session.query(MondoTermDbModel).filter(MondoTermDbModel.code == "MONDO:0015263").one() + assert stored.label == "Brugada syndrome" + + @pytest.mark.parametrize( "mock_publication_fetch", [ diff --git a/tests/routers/test_variant.py b/tests/routers/test_variant.py index 48f3fbd99..d865f7c9c 100644 --- a/tests/routers/test_variant.py +++ b/tests/routers/test_variant.py @@ -176,6 +176,7 @@ class TestPrivateCalibrationsAreNotServedOverHttp: """ def _private_calibration(self, session, score_set_urn): + from mavedb.lib.mondo import get_generic_disease_term from mavedb.models.score_calibration import ScoreCalibration score_set = session.scalars(select(ScoreSetDbModel).where(ScoreSetDbModel.urn == score_set_urn)).one() @@ -188,6 +189,7 @@ def _private_calibration(self, session, score_set_urn): primary=False, private=True, calibration_metadata={}, + disease_term=get_generic_disease_term(session), created_by_id=score_set.created_by_id, modified_by_id=score_set.modified_by_id, ) diff --git a/tests/scripts/conftest_optional.py b/tests/scripts/conftest_optional.py index c2216f6d5..65b16e8a7 100644 --- a/tests/scripts/conftest_optional.py +++ b/tests/scripts/conftest_optional.py @@ -12,6 +12,7 @@ from mavedb.models.acmg_classification import ACMGClassification from mavedb.models.enums.acmg_criterion import ACMGCriterion from mavedb.models.enums.functional_classification import FunctionalClassification +from mavedb.lib.mondo import get_generic_disease_term from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification @@ -65,6 +66,7 @@ def _make(score_set, *, private=False, research_use_only=False, title=None): primary=not (private or research_use_only), private=private, calibration_metadata={}, + disease_term=get_generic_disease_term(session), created_by_id=sample_user.id, modified_by_id=sample_user.id, ) diff --git a/tests/view_models/test_calibration_control.py b/tests/view_models/test_calibration_control.py index 8b61d5120..3f1bf5b5b 100644 --- a/tests/view_models/test_calibration_control.py +++ b/tests/view_models/test_calibration_control.py @@ -4,6 +4,7 @@ import pytest from pydantic import ValidationError +from mavedb.lib.mondo import MONDO_SYSTEM from mavedb.models.enums.calibration_control_status import CalibrationControlStatus from mavedb.view_models.calibration_control import ( CalibrationControlCreate, @@ -97,7 +98,7 @@ def test_saved_calibration_control_synthesizes_variant_urn_from_orm(): def test_score_calibration_create_accepts_controls_disease_and_phi(): payload = deepcopy(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) - payload["disease"] = "Brugada syndrome" + payload["disease"] = "MONDO:0015263" payload["controls_not_phi"] = True payload["controls"] = [ {"variant_urn": TEST_CONTROL_VARIANT_URN, "clinical_status": "pathogenic"}, @@ -106,16 +107,18 @@ def test_score_calibration_create_accepts_controls_disease_and_phi(): calibration = ScoreCalibrationCreate.model_validate(payload) - assert calibration.disease == "Brugada syndrome" + assert calibration.disease == "MONDO:0015263" assert calibration.controls_not_phi is True assert len(calibration.controls) == 2 assert calibration.controls[0].clinical_status is CalibrationControlStatus.pathogenic assert calibration.controls[1].clinical_status is CalibrationControlStatus.benign -def test_saved_score_calibration_includes_controls(): +def test_saved_score_calibration_includes_controls_and_disease_concept(): saved = deepcopy(TEST_SAVED_BRNICH_SCORE_CALIBRATION_RANGE_BASED) - saved["disease"] = "Brugada syndrome" + saved["disease_term"] = dummy_attributed_object_from_dict( + {"code": "MONDO:0015263", "system": MONDO_SYSTEM, "system_version": None, "label": "Brugada syndrome"} + ) saved["controlsNotPhi"] = True saved["controls"] = [ { @@ -131,7 +134,9 @@ def test_saved_score_calibration_includes_controls(): calibration = ScoreCalibration.model_validate(dummy_attributed_object_from_dict(saved)) - assert calibration.disease == "Brugada syndrome" + assert calibration.disease.conceptType == "Disease" + assert calibration.disease.primaryCoding.code.root == "MONDO:0015263" + assert calibration.disease.name == "Brugada syndrome" assert calibration.controls_not_phi is True assert len(calibration.controls) == 1 assert calibration.controls[0].variant_urn == TEST_CONTROL_VARIANT_URN From 8c132aa4b0c36c1d4027bb9d79777f3682d85a4b Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Thu, 17 Sep 2026 09:55:46 -0700 Subject: [PATCH 155/252] refactor(annotation): derive the disease condition from the calibration term (#754) With every calibration now carrying a non-null MONDO disease term, build the VA-Spec Condition directly from it instead of the hardcoded generic mappable-concept helper. The clinical-impact proposition takes the authoritative (strongest) calibration and uses its disease as the single objectCondition every evidence line on the statement targets. Removes the now-unused annotation disease constants module. --- src/mavedb/lib/annotation/annotate.py | 12 +- src/mavedb/lib/annotation/condition.py | 24 +-- src/mavedb/lib/annotation/constants.py | 2 - src/mavedb/lib/annotation/proposition.py | 6 +- tests/lib/annotation/test_condition.py | 163 +++++---------------- tests/lib/annotation/test_constants.py | 40 ----- tests/lib/annotation/test_evidence_line.py | 17 ++- tests/lib/annotation/test_proposition.py | 20 ++- 8 files changed, 85 insertions(+), 199 deletions(-) delete mode 100644 src/mavedb/lib/annotation/constants.py delete mode 100644 tests/lib/annotation/test_constants.py diff --git a/src/mavedb/lib/annotation/annotate.py b/src/mavedb/lib/annotation/annotate.py index 83b13b714..70f926425 100644 --- a/src/mavedb/lib/annotation/annotate.py +++ b/src/mavedb/lib/annotation/annotate.py @@ -120,7 +120,6 @@ def variant_pathogenicity_statement( study_result = mapped_variant_to_experimental_variant_impact_study_result(mapped_variant) functional_proposition = mapped_variant_to_experimental_variant_functional_impact_proposition(mapped_variant) - clinical_proposition = mapped_variant_to_experimental_variant_clinical_impact_proposition(mapped_variant) eligible_calibrations = calibrations_available_for_annotation( mapped_variant, @@ -137,6 +136,17 @@ def variant_pathogenicity_statement( if not strongest_calibration: return None + # The statement carries one proposition (targeted by every evidence line), so its disease context + # is the strongest calibration's — the same calibration that anchors the ACMG classification. + # + # TODO#XXX - Pooling calibrations with different disease contexts is not supported by the current VA-Spec model. If a + # variant has multiple calibrations with different disease contexts, the strongest calibration is used for the + # statement's disease context and we should consider whether to filter out calibrations with different disease contexts + # from the evidence lines. + clinical_proposition = mapped_variant_to_experimental_variant_clinical_impact_proposition( + mapped_variant, strongest_calibration + ) + # Get the classification from the strongest range (used for the functional statement within clinical evidence) # If strongest_range is None, the variant is not in any range, so classification will be INDETERMINATE _, classification = functional_classification_of_variant(mapped_variant, strongest_calibration) diff --git a/src/mavedb/lib/annotation/condition.py b/src/mavedb/lib/annotation/condition.py index cbd5fabe0..ae1f1b7af 100644 --- a/src/mavedb/lib/annotation/condition.py +++ b/src/mavedb/lib/annotation/condition.py @@ -1,21 +1,13 @@ -from ga4gh.core.models import Coding, iriReference as IRI, MappableConcept from ga4gh.va_spec.base.domain_entities import Condition -from mavedb.lib.annotation.constants import GENERIC_DISEASE_MEDGEN_CODE, MEDGEN_SYSTEM +from mavedb.lib.mondo import mondo_term_to_mappable_concept +from mavedb.models.score_calibration import ScoreCalibration -def generic_disease_condition_iri() -> IRI: - return IRI(root=f"http://identifiers.org/medgen/{GENERIC_DISEASE_MEDGEN_CODE}") +def calibration_disease_condition(score_calibration: ScoreCalibration) -> Condition: + """The disease/disorder a calibration applies to, as a VA-Spec ``Condition``. - -def generic_disease_condition() -> Condition: - return Condition( - root=MappableConcept( - conceptType="Disease", - primaryCoding=Coding( - code=GENERIC_DISEASE_MEDGEN_CODE, - system=MEDGEN_SYSTEM, - iris=[generic_disease_condition_iri()], - ), - ) - ) + Every calibration carries a non-null MONDO disease term (often the generic "disease or + disorder" (``MONDO:0000001``)) which is serialized directly to a condition. + """ + return Condition(root=mondo_term_to_mappable_concept(score_calibration.disease_term)) diff --git a/src/mavedb/lib/annotation/constants.py b/src/mavedb/lib/annotation/constants.py deleted file mode 100644 index 90b7dfecf..000000000 --- a/src/mavedb/lib/annotation/constants.py +++ /dev/null @@ -1,2 +0,0 @@ -GENERIC_DISEASE_MEDGEN_CODE = "C0012634" -MEDGEN_SYSTEM = "https://www.ncbi.nlm.nih.gov/medgen/" diff --git a/src/mavedb/lib/annotation/proposition.py b/src/mavedb/lib/annotation/proposition.py index 963327c73..59213d66e 100644 --- a/src/mavedb/lib/annotation/proposition.py +++ b/src/mavedb/lib/annotation/proposition.py @@ -1,14 +1,16 @@ from ga4gh.core.models import Coding, MappableConcept from ga4gh.va_spec.base.core import ExperimentalVariantFunctionalImpactProposition, VariantPathogenicityProposition -from mavedb.lib.annotation.condition import generic_disease_condition +from mavedb.lib.annotation.condition import calibration_disease_condition from mavedb.lib.annotation.document import experiment_to_document from mavedb.lib.annotation.util import sequence_feature_for_mapped_variant, variation_from_mapped_variant from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.score_calibration import ScoreCalibration def mapped_variant_to_experimental_variant_clinical_impact_proposition( mapped_variant: MappedVariant, + score_calibration: ScoreCalibration, ) -> VariantPathogenicityProposition: coding, system = sequence_feature_for_mapped_variant(mapped_variant) sequence_feature = MappableConcept( @@ -19,7 +21,7 @@ def mapped_variant_to_experimental_variant_clinical_impact_proposition( description=f"Variant pathogenicity proposition for {mapped_variant.variant.urn}.", subjectVariant=variation_from_mapped_variant(mapped_variant), predicate="isCausalFor", - objectCondition=generic_disease_condition(), + objectCondition=calibration_disease_condition(score_calibration), geneContextQualifier=sequence_feature if system == "https://www.genenames.org/" else None, # only include gene context if we have a gene identifier diff --git a/tests/lib/annotation/test_condition.py b/tests/lib/annotation/test_condition.py index 817628794..2619f1abd 100644 --- a/tests/lib/annotation/test_condition.py +++ b/tests/lib/annotation/test_condition.py @@ -1,12 +1,14 @@ """ -Tests for mavedb.lib.annotation.condition module. +Tests for mavedb.lib.annotation.condition. -This module tests functions for creating GA4GH Condition objects and IRIs, -specifically for generic disease conditions used in variant annotations. +A calibration's disease condition is serialized directly from its stored MONDO term (see +mavedb.lib.mondo); an unspecified disease resolves to the generic term at write time. """ # ruff: noqa: E402 +from types import SimpleNamespace + import pytest pytest.importorskip("psycopg2") @@ -15,142 +17,43 @@ from ga4gh.core.models import iriReference as IRI from ga4gh.va_spec.base.domain_entities import Condition -from mavedb.lib.annotation.condition import generic_disease_condition, generic_disease_condition_iri -from mavedb.lib.annotation.constants import GENERIC_DISEASE_MEDGEN_CODE, MEDGEN_SYSTEM - - -@pytest.mark.unit -class TestGenericDiseaseConditionIriUnit: - """Unit tests for generic_disease_condition_iri function.""" - - def test_returns_correct_iri_structure(self): - """Test that function returns proper IRI object with expected root.""" - iri = generic_disease_condition_iri() - - assert isinstance(iri, IRI) - expected_root = f"http://identifiers.org/medgen/{GENERIC_DISEASE_MEDGEN_CODE}" - assert iri.root == expected_root - - def test_uses_correct_medgen_code(self): - """Test that function uses the correct MedGen code from constants.""" - iri = generic_disease_condition_iri() - - assert GENERIC_DISEASE_MEDGEN_CODE in iri.root - assert iri.root.endswith(GENERIC_DISEASE_MEDGEN_CODE) - - def test_uses_correct_iri_format(self): - """Test that IRI follows the expected identifiers.org format.""" - iri = generic_disease_condition_iri() +from mavedb.lib.annotation.condition import calibration_disease_condition +from mavedb.lib.mondo import MONDO_GENERIC_CODE, MONDO_GENERIC_LABEL, MONDO_SYSTEM, mondo_iri +from mavedb.models.mondo_term import MondoTerm - assert iri.root.startswith("http://identifiers.org/medgen/") - assert "identifiers.org" in iri.root - def test_iri_consistency(self): - """Test that multiple calls return consistent IRI values.""" - iri1 = generic_disease_condition_iri() - iri2 = generic_disease_condition_iri() - - assert iri1.root == iri2.root +def _calibration_with_disease(term: MondoTerm) -> SimpleNamespace: + """A stand-in calibration exposing only the disease_term the condition builder reads.""" + return SimpleNamespace(disease_term=term) @pytest.mark.unit -class TestGenericDiseaseConditionUnit: - """Unit tests for generic_disease_condition function.""" - - def test_returns_correct_condition_structure(self): - """Test that function returns proper Condition object.""" - condition = generic_disease_condition() - +class TestCalibrationDiseaseConditionUnit: + """Unit tests for the calibration disease condition.""" + + def test_builds_condition_from_the_calibration_disease_term(self): + term = MondoTerm( + code="MONDO:0015263", system=MONDO_SYSTEM, system_version="2026-01-01", label="Brugada syndrome" + ) + condition = calibration_disease_condition(_calibration_with_disease(term)) assert isinstance(condition, Condition) - assert hasattr(condition, "root") assert isinstance(condition.root, MappableConcept) - - def test_concept_type_is_disease(self): - """Test that condition has correct conceptType.""" - condition = generic_disease_condition() - assert condition.root.conceptType == "Disease" + assert condition.root.name == "Brugada syndrome" - def test_primary_coding_structure(self): - """Test that primary coding has correct structure and values.""" - condition = generic_disease_condition() - coding = condition.root.primaryCoding - + def test_primary_coding_reflects_the_stored_term(self): + term = MondoTerm(code="MONDO:0015263", system=MONDO_SYSTEM, label="Brugada syndrome") + coding = calibration_disease_condition(_calibration_with_disease(term)).root.primaryCoding assert isinstance(coding, Coding) - assert coding.code.root == GENERIC_DISEASE_MEDGEN_CODE - assert coding.system == MEDGEN_SYSTEM - - def test_coding_iris_structure(self): - """Test that coding includes correct IRI list.""" - condition = generic_disease_condition() - coding = condition.root.primaryCoding - + assert coding.code.root == "MONDO:0015263" + assert coding.system == MONDO_SYSTEM assert isinstance(coding.iris, list) assert len(coding.iris) == 1 - - iri = coding.iris[0] - assert isinstance(iri, IRI) - - def test_coding_iri_matches_helper_function(self): - """Test that coding IRI matches the output of helper function.""" - condition = generic_disease_condition() - coding = condition.root.primaryCoding - expected_iri = generic_disease_condition_iri() - - assert len(coding.iris) == 1 - assert coding.iris[0].root == expected_iri.root - - def test_uses_correct_constants(self): - """Test that condition uses the correct constant values.""" - condition = generic_disease_condition() - coding = condition.root.primaryCoding - - assert coding.code.root == GENERIC_DISEASE_MEDGEN_CODE - assert coding.system == MEDGEN_SYSTEM - - def test_iri_contains_medgen_code(self): - """Test that IRI contains the correct MedGen code.""" - condition = generic_disease_condition() - coding = condition.root.primaryCoding - iri_root = coding.iris[0].root - - expected_iri_content = f"http://identifiers.org/medgen/{GENERIC_DISEASE_MEDGEN_CODE}" - assert iri_root == expected_iri_content - - def test_condition_consistency(self): - """Test that multiple calls return consistent condition structures.""" - condition1 = generic_disease_condition() - condition2 = generic_disease_condition() - - assert condition1.root.conceptType == condition2.root.conceptType - assert condition1.root.primaryCoding.code == condition2.root.primaryCoding.code - assert condition1.root.primaryCoding.system == condition2.root.primaryCoding.system - assert condition1.root.primaryCoding.iris[0].root == condition2.root.primaryCoding.iris[0].root - - -@pytest.mark.unit -class TestConditionConsistencyUnit: - """Unit tests for cross-function consistency in condition helpers.""" - - def test_iri_function_integration_with_condition(self): - """Test that standalone IRI function produces same result as condition's IRI.""" - standalone_iri = generic_disease_condition_iri() - condition = generic_disease_condition() - condition_iri = condition.root.primaryCoding.iris[0] - - assert standalone_iri.root == condition_iri.root - - def test_complete_condition_structure_integration(self): - """Test the complete condition structure matches expected GA4GH format.""" - condition = generic_disease_condition() - - # Verify complete structure - assert condition.root.conceptType == "Disease" - assert condition.root.primaryCoding.code.root == GENERIC_DISEASE_MEDGEN_CODE - assert condition.root.primaryCoding.system == MEDGEN_SYSTEM - assert len(condition.root.primaryCoding.iris) == 1 - - # Verify IRI structure - iri = condition.root.primaryCoding.iris[0] - expected_iri_root = f"http://identifiers.org/medgen/{GENERIC_DISEASE_MEDGEN_CODE}" - assert iri.root == expected_iri_root + assert isinstance(coding.iris[0], IRI) + assert coding.iris[0].root == mondo_iri("MONDO:0015263") + + def test_generic_disease_term_yields_the_mondo_root(self): + term = MondoTerm(code=MONDO_GENERIC_CODE, system=MONDO_SYSTEM, label=MONDO_GENERIC_LABEL) + condition = calibration_disease_condition(_calibration_with_disease(term)) + assert condition.root.primaryCoding.code.root == MONDO_GENERIC_CODE + assert condition.root.name == MONDO_GENERIC_LABEL diff --git a/tests/lib/annotation/test_constants.py b/tests/lib/annotation/test_constants.py deleted file mode 100644 index 558cc42e2..000000000 --- a/tests/lib/annotation/test_constants.py +++ /dev/null @@ -1,40 +0,0 @@ -""" -Tests for mavedb.lib.annotation.constants module. - -This module tests constant values used throughout the annotation system. -""" - -# ruff: noqa: E402 - -import pytest - -pytest.importorskip("psycopg2") - -from mavedb.lib.annotation.constants import ( - GENERIC_DISEASE_MEDGEN_CODE, - MEDGEN_SYSTEM, -) - - -@pytest.mark.unit -class TestAnnotationConstants: - """Unit tests for annotation constants.""" - - def test_generic_disease_medgen_code(self): - """Test generic disease MedGen code constant.""" - assert GENERIC_DISEASE_MEDGEN_CODE == "C0012634" - assert isinstance(GENERIC_DISEASE_MEDGEN_CODE, str) - - def test_medgen_system(self): - """Test MedGen system URL constant.""" - expected_url = "https://www.ncbi.nlm.nih.gov/medgen/" - assert MEDGEN_SYSTEM == expected_url - assert isinstance(MEDGEN_SYSTEM, str) - assert MEDGEN_SYSTEM.startswith("https://") - assert MEDGEN_SYSTEM.endswith("/") - - def test_constants_immutability(self): - """Test that constants are properly defined as strings.""" - # These should be string constants, not mutable objects - assert isinstance(GENERIC_DISEASE_MEDGEN_CODE, str) - assert isinstance(MEDGEN_SYSTEM, str) diff --git a/tests/lib/annotation/test_evidence_line.py b/tests/lib/annotation/test_evidence_line.py index fb6f33b74..d983793d0 100644 --- a/tests/lib/annotation/test_evidence_line.py +++ b/tests/lib/annotation/test_evidence_line.py @@ -25,6 +25,7 @@ mapped_variant_to_experimental_variant_functional_impact_proposition, ) from mavedb.lib.annotation.statement import mapped_variant_to_functional_statement +from tests.helpers.mocks.factories import create_mock_score_calibration @pytest.mark.unit @@ -62,7 +63,9 @@ def test_acmg_evidence_line_with_met_valid_clinical_classification( "mavedb.lib.annotation.evidence_line.pathogenicity_classification_of_variant", return_value=(MagicMock(label="Test Range"), expected_outcome, expected_strength), ): - proposition = mapped_variant_to_experimental_variant_clinical_impact_proposition(mapped_variant) + proposition = mapped_variant_to_experimental_variant_clinical_impact_proposition( + mapped_variant, score_calibration + ) study_result = variant_study_result(mapped_variant) evidence = functional_evidence_line(mapped_variant, score_calibration, [study_result]) result = acmg_evidence_line(mapped_variant, score_calibration, proposition, [evidence]) @@ -100,7 +103,9 @@ def test_acmg_evidence_line_with_not_met_clinical_classification( "mavedb.lib.annotation.evidence_line.pathogenicity_classification_of_variant", return_value=(MagicMock(label="Test Range"), expected_outcome, expected_strength), ): - proposition = mapped_variant_to_experimental_variant_clinical_impact_proposition(mapped_variant) + proposition = mapped_variant_to_experimental_variant_clinical_impact_proposition( + mapped_variant, score_calibration + ) study_result = variant_study_result(mapped_variant) evidence = functional_evidence_line(mapped_variant, score_calibration, [study_result]) result = acmg_evidence_line(mapped_variant, score_calibration, proposition, [evidence]) @@ -123,7 +128,9 @@ def test_acmg_evidence_line_with_no_calibrations_raises_error(self, mock_mapped_ score_calibration = MagicMock() with pytest.raises(ValueError, match="does not have a score set with score calibrations"): - proposition = mapped_variant_to_experimental_variant_clinical_impact_proposition(mock_mapped_variant) + proposition = mapped_variant_to_experimental_variant_clinical_impact_proposition( + mock_mapped_variant, create_mock_score_calibration() + ) study_result = variant_study_result(mock_mapped_variant) acmg_evidence_line(mock_mapped_variant, score_calibration, proposition, [study_result]) @@ -151,7 +158,9 @@ def test_acmg_evidence_line_accepts_statement_evidence_without_serialization_err score_calibration, ExperimentalVariantFunctionalImpactClassification.NORMAL, ) - clinical_proposition = mapped_variant_to_experimental_variant_clinical_impact_proposition(mapped_variant) + clinical_proposition = mapped_variant_to_experimental_variant_clinical_impact_proposition( + mapped_variant, score_calibration + ) result = acmg_evidence_line(mapped_variant, score_calibration, clinical_proposition, [functional_statement]) diff --git a/tests/lib/annotation/test_proposition.py b/tests/lib/annotation/test_proposition.py index cb883e0bc..14563f343 100644 --- a/tests/lib/annotation/test_proposition.py +++ b/tests/lib/annotation/test_proposition.py @@ -21,6 +21,7 @@ mapped_variant_to_experimental_variant_clinical_impact_proposition, mapped_variant_to_experimental_variant_functional_impact_proposition, ) +from tests.helpers.mocks.factories import create_mock_mondo_term, create_mock_score_calibration @pytest.mark.unit @@ -28,16 +29,27 @@ class TestExperimentalVariantClinicalImpactProposition: """Unit tests for experimental variant clinical impact proposition creation.""" def test_mapped_variant_to_experimental_variant_clinical_impact_proposition(self, mock_mapped_variant): - """Test creation of clinical impact proposition from mapped variant.""" - result = mapped_variant_to_experimental_variant_clinical_impact_proposition(mock_mapped_variant) + """The proposition's condition defaults to the calibration's generic disease term.""" + calibration = create_mock_score_calibration() + result = mapped_variant_to_experimental_variant_clinical_impact_proposition(mock_mapped_variant, calibration) assert isinstance(result, VariantPathogenicityProposition) assert result.description == f"Variant pathogenicity proposition for {mock_mapped_variant.variant.urn}." assert isinstance(result.subjectVariant, MolecularVariation) assert result.predicate == "isCausalFor" assert result.objectCondition.root.conceptType == "Disease" - assert result.objectCondition.root.primaryCoding.code.root == "C0012634" - assert result.objectCondition.root.primaryCoding.system == "https://www.ncbi.nlm.nih.gov/medgen/" + assert result.objectCondition.root.primaryCoding.code.root == "MONDO:0000001" + assert result.objectCondition.root.primaryCoding.system == "https://purl.obolibrary.org/obo/mondo.owl" + + def test_clinical_impact_proposition_reflects_the_calibration_disease(self, mock_mapped_variant): + """A calibration with a specific disease term drives the proposition's condition.""" + calibration = create_mock_score_calibration( + disease_term=create_mock_mondo_term(code="MONDO:0015263", label="Brugada syndrome") + ) + result = mapped_variant_to_experimental_variant_clinical_impact_proposition(mock_mapped_variant, calibration) + + assert result.objectCondition.root.primaryCoding.code.root == "MONDO:0015263" + assert result.objectCondition.root.name == "Brugada syndrome" @pytest.mark.unit From 18b1c8a8d25a6ccdeab870a642d5a83769bf885c Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Thu, 17 Sep 2026 09:56:31 -0700 Subject: [PATCH 156/252] feat(calibration-controls): reject controls on score set creation (#754) Controls resolve each variant URN against the score set's own variants, which are minted asynchronously from the scores file after the score set row exists, so none are resolvable at creation time. Reject a non-empty controls list with a 409, mirroring the existing class-based-range guard, rather than silently dropping the controls (the nested-create path calls standalone create_score_calibration, which never builds them). --- src/mavedb/routers/score_sets.py | 11 +++++++++++ tests/routers/test_score_set.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/src/mavedb/routers/score_sets.py b/src/mavedb/routers/score_sets.py index aff713429..66fda2d15 100644 --- a/src/mavedb/routers/score_sets.py +++ b/src/mavedb/routers/score_sets.py @@ -1919,6 +1919,17 @@ async def create_score_set( detail="Class-based calibrations are not supported on score set creation. Please create class-based calibrations after creating the score set.", ) + # TODO#592: Support for controls on score set creation + if calibration_create.controls: + logger.info( + msg="Failed to create score set; Calibration controls are not supported on score set creation.", + extra=logging_context(), + ) + raise HTTPException( + status_code=409, + detail="Calibration controls are not supported on score set creation. Please add controls after creating the score set.", + ) + created_calibration_item = await create_score_calibration( db, calibration_create, user_data.user, variant_classes=None ) diff --git a/tests/routers/test_score_set.py b/tests/routers/test_score_set.py index 7d9290a9a..d91bdea13 100644 --- a/tests/routers/test_score_set.py +++ b/tests/routers/test_score_set.py @@ -65,6 +65,7 @@ TEST_SAVED_TAXONOMY, TEST_USER, VALID_CLINGEN_CA_ID, + VALID_VARIANT_URN, ) from tests.helpers.dependency_overrider import DependencyOverrider from tests.helpers.mocks.factories import create_mock_mapped_variant @@ -286,6 +287,33 @@ def test_cannot_create_score_set_with_class_based_calibration(client, mock_publi assert "Class-based calibrations are not supported on score set creation" in response_data["detail"] +@pytest.mark.parametrize( + "mock_publication_fetch", + [ + ( + [ + {"dbName": "PubMed", "identifier": f"{TEST_PUBMED_IDENTIFIER}"}, + {"dbName": "bioRxiv", "identifier": f"{TEST_BIORXIV_IDENTIFIER}"}, + ] + ) + ], + indirect=["mock_publication_fetch"], +) +def test_cannot_create_score_set_with_calibration_controls(client, mock_publication_fetch, setup_router_db): + experiment = create_experiment(client) + score_set = deepcopy(TEST_MINIMAL_SEQ_SCORESET) + score_set["experimentUrn"] = experiment["urn"] + calibration = deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) + # Controls reference variants that don't exist until the scores file is processed after creation. + calibration["controls"] = [{"variantUrn": VALID_VARIANT_URN, "clinicalStatus": "pathogenic"}] + score_set.update({"scoreCalibrations": [calibration]}) + + response = client.post("/api/v1/score-sets/", json=score_set) + assert response.status_code == 409 + response_data = response.json() + assert "Calibration controls are not supported on score set creation" in response_data["detail"] + + @pytest.mark.parametrize( "mock_publication_fetch", [ From 6b387fc87b633d9590eadcd709ba3a8075c6b993 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Thu, 17 Sep 2026 11:49:07 -0700 Subject: [PATCH 157/252] fix(calibration-controls): re-enforce PHI gate on modify, not just publish (#753) Publishing already blocked a calibration from going public with unaffirmed controls, but the modify route never re-checked the invariant afterward. An admin (the only user who can reach this route on an already-public calibration) could add or replace controls without re-affirming, since replacing controls resets the affirmation flag. Apply the same PHI acknowledgment gate on modify that publish already enforces. --- src/mavedb/routers/score_calibrations.py | 16 ++ tests/routers/test_calibration_controls.py | 165 ++++++++++++++++++++- 2 files changed, 180 insertions(+), 1 deletion(-) diff --git a/src/mavedb/routers/score_calibrations.py b/src/mavedb/routers/score_calibrations.py index bc5c48e8d..62a5d88d4 100644 --- a/src/mavedb/routers/score_calibrations.py +++ b/src/mavedb/routers/score_calibrations.py @@ -599,6 +599,22 @@ async def modify_score_calibration_route( db, item, calibration_update, user_data.user, variant_classes if classes_file else None ) + # A public calibration may not carry unacknowledged controls, on this route the same as at + # publish time. controls_not_phi is tristate: only True clears the gate — None (unaddressed) and + # False (declined) both block. Only meaningful when controls exist. + if ( + not updated_calibration.private + and updated_calibration.controls + and updated_calibration.controls_not_phi is not True + ): + raise HTTPException( + status_code=422, + detail=( + "This calibration has controls that have not been affirmed to be free of protected health " + "information (PHI). Set controls_not_phi to true." + ), + ) + db.commit() db.refresh(updated_calibration) diff --git a/tests/routers/test_calibration_controls.py b/tests/routers/test_calibration_controls.py index 660be6e97..cf47e45c1 100644 --- a/tests/routers/test_calibration_controls.py +++ b/tests/routers/test_calibration_controls.py @@ -24,9 +24,13 @@ TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED, TEST_PUBMED_IDENTIFIER, ) +from tests.helpers.dependency_overrider import DependencyOverrider from tests.helpers.util.common import deepcamelize from tests.helpers.util.experiment import create_experiment -from tests.helpers.util.score_calibration import create_test_score_calibration_in_score_set_via_client +from tests.helpers.util.score_calibration import ( + create_test_score_calibration_in_score_set_via_client, + publish_test_score_calibration_via_client, +) from tests.helpers.util.score_set import create_seq_score_set_with_mapped_variants CALIBRATION_PUBLICATIONS = [ @@ -357,3 +361,162 @@ def test_modify_controls_with_affirmation_keeps_phi_flag( assert response.status_code == 200, response.text assert response.json()["controlsNotPhi"] is True + + +########################################################### +# PHI re-affirmation invariant on published calibrations +########################################################### + + +@pytest.mark.parametrize("mock_publication_fetch", [CALIBRATION_PUBLICATIONS], indirect=["mock_publication_fetch"]) +def test_create_calibration_ignores_client_supplied_private_field( + client, setup_router_db, mock_publication_fetch, session, data_provider, data_files +): + # The create schema has no `private` field — creation always starts private, and publishing + # (which independently gates PHI-bearing controls) is the only path to making one public. + score_set, variant_urns = _score_set_with_variant_urns(client, session, data_provider, data_files) + + response = client.post( + "/api/v1/score-calibrations/", + json={ + **_calibration_payload(score_set["urn"]), + "private": False, + "controls": [{"variantUrn": variant_urns[0], "clinicalStatus": "pathogenic"}], + }, + ) + + assert response.status_code == 200, response.text + assert response.json()["private"] is True + + +@pytest.mark.parametrize("mock_publication_fetch", [CALIBRATION_PUBLICATIONS], indirect=["mock_publication_fetch"]) +def test_cannot_add_controls_to_published_calibration_without_phi_affirmation_as_admin( + client, setup_router_db, mock_publication_fetch, session, data_provider, data_files, admin_app_overrides +): + score_set, variant_urns = _score_set_with_variant_urns(client, session, data_provider, data_files) + calibration = create_test_score_calibration_in_score_set_via_client( + client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) + ) + publish_test_score_calibration_via_client(client, calibration["urn"]) + + with DependencyOverrider(admin_app_overrides): + response = client.put( + f"/api/v1/score-calibrations/{calibration['urn']}", + json=_calibration_payload( + score_set["urn"], controls=[{"variantUrn": variant_urns[0], "clinicalStatus": "pathogenic"}] + ), + ) + + assert response.status_code == 422 + assert "protected health information" in response.json()["detail"].lower() + + # The gate must not have attached the controls. + refreshed = session.query(CalibrationDbModel).where(CalibrationDbModel.urn == calibration["urn"]).one() + assert refreshed.controls == [] + + +@pytest.mark.parametrize("mock_publication_fetch", [CALIBRATION_PUBLICATIONS], indirect=["mock_publication_fetch"]) +def test_can_add_controls_to_published_calibration_with_phi_affirmation_as_admin( + client, setup_router_db, mock_publication_fetch, session, data_provider, data_files, admin_app_overrides +): + score_set, variant_urns = _score_set_with_variant_urns(client, session, data_provider, data_files) + calibration = create_test_score_calibration_in_score_set_via_client( + client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) + ) + publish_test_score_calibration_via_client(client, calibration["urn"]) + + with DependencyOverrider(admin_app_overrides): + response = client.put( + f"/api/v1/score-calibrations/{calibration['urn']}", + json=_calibration_payload( + score_set["urn"], + controls=[{"variantUrn": variant_urns[0], "clinicalStatus": "pathogenic"}], + controlsNotPhi=True, + ), + ) + + assert response.status_code == 200, response.text + assert response.json()["controlsNotPhi"] is True + assert len(response.json()["controls"]) == 1 + + +@pytest.mark.parametrize("mock_publication_fetch", [CALIBRATION_PUBLICATIONS], indirect=["mock_publication_fetch"]) +def test_cannot_replace_controls_on_published_calibration_without_reaffirming_as_admin( + client, setup_router_db, mock_publication_fetch, session, data_provider, data_files, admin_app_overrides +): + score_set, variant_urns = _score_set_with_variant_urns(client, session, data_provider, data_files) + calibration = _create_with_controls( + client, + score_set["urn"], + [{"variantUrn": variant_urns[0], "clinicalStatus": "pathogenic"}], + controlsNotPhi=True, + ) + publish_test_score_calibration_via_client(client, calibration["urn"]) + + # Replacing controls invalidates the prior affirmation just as it does on a private calibration, + # so an admin who doesn't re-affirm in the same request must be rejected here too. + with DependencyOverrider(admin_app_overrides): + response = client.put( + f"/api/v1/score-calibrations/{calibration['urn']}", + json=_calibration_payload( + score_set["urn"], controls=[{"variantUrn": variant_urns[1], "clinicalStatus": "benign"}] + ), + ) + + assert response.status_code == 422 + assert "protected health information" in response.json()["detail"].lower() + + # The gate must not have replaced the controls or reset the affirmation. + refreshed = session.query(CalibrationDbModel).where(CalibrationDbModel.urn == calibration["urn"]).one() + assert refreshed.controls_not_phi is True + assert len(refreshed.controls) == 1 + + +@pytest.mark.parametrize("mock_publication_fetch", [CALIBRATION_PUBLICATIONS], indirect=["mock_publication_fetch"]) +def test_cannot_decline_phi_affirmation_on_published_calibration_with_existing_controls_as_admin( + client, setup_router_db, mock_publication_fetch, session, data_provider, data_files, admin_app_overrides +): + score_set, variant_urns = _score_set_with_variant_urns(client, session, data_provider, data_files) + calibration = _create_with_controls( + client, + score_set["urn"], + [{"variantUrn": variant_urns[0], "clinicalStatus": "pathogenic"}], + controlsNotPhi=True, + ) + publish_test_score_calibration_via_client(client, calibration["urn"]) + + # Controls are left unchanged; only the affirmation is walked back to False. + with DependencyOverrider(admin_app_overrides): + response = client.put( + f"/api/v1/score-calibrations/{calibration['urn']}", + json=_calibration_payload(score_set["urn"], controlsNotPhi=False), + ) + + assert response.status_code == 422 + assert "protected health information" in response.json()["detail"].lower() + + +@pytest.mark.parametrize("mock_publication_fetch", [CALIBRATION_PUBLICATIONS], indirect=["mock_publication_fetch"]) +def test_can_update_published_calibration_leaving_affirmed_controls_untouched_as_admin( + client, setup_router_db, mock_publication_fetch, session, data_provider, data_files, admin_app_overrides +): + score_set, variant_urns = _score_set_with_variant_urns(client, session, data_provider, data_files) + calibration = _create_with_controls( + client, + score_set["urn"], + [{"variantUrn": variant_urns[0], "clinicalStatus": "pathogenic"}], + controlsNotPhi=True, + ) + publish_test_score_calibration_via_client(client, calibration["urn"]) + + # An edit that neither submits controls nor touches controlsNotPhi must leave the standing + # affirmation alone, so it should not trip the gate. + with DependencyOverrider(admin_app_overrides): + response = client.put( + f"/api/v1/score-calibrations/{calibration['urn']}", + json=_calibration_payload(score_set["urn"], notes="Updated notes"), + ) + + assert response.status_code == 200, response.text + assert response.json()["controlsNotPhi"] is True + assert len(response.json()["controls"]) == 1 From c29253ba9ceabc4c182d0ce97dd7871271d31186 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Fri, 18 Sep 2026 12:43:56 -0700 Subject: [PATCH 158/252] fix(calibration): make MONDO term resolution idempotent find_or_create_mondo_term looked up an existing term by the submitted code but inserted the canonical code returned by OLS. When OLS normalizes the input (case, or an alias/obsolete id), a resubmission missed the lookup and tried to insert the canonical code again, tripping UNIQUE(system, code) with an unhandled 500. Re-check for an existing row by the resolved canonical code before inserting, so lookup and insert key on the same value and the get-or-create stays idempotent. --- src/mavedb/lib/mondo.py | 11 ++++++++++- tests/lib/mondo/test_mondo.py | 16 ++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/src/mavedb/lib/mondo.py b/src/mavedb/lib/mondo.py index dbabbf01d..71a9c1223 100644 --- a/src/mavedb/lib/mondo.py +++ b/src/mavedb/lib/mondo.py @@ -183,7 +183,16 @@ async def find_or_create_mondo_term(db: Session, code: str) -> MondoTerm: if resolved is None: raise ValidationError(f"'{code}' is not a valid MONDO disease term.", custom_loc=["body", "disease"]) - term = MondoTerm(code=resolved["code"], system=MONDO_SYSTEM, label=resolved["label"]) # type: ignore[call-arg] + # OLS may normalize the submitted code (case, or an alias/obsolete id) to a different canonical code, + # so re-check existence by the resolved code before inserting: keying the lookup on the same value we + # insert under keeps this idempotent and avoids a UNIQUE(system, code) violation when the canonical + # row already exists under a different submitted spelling. + canonical_code = resolved["code"] + term = db.query(MondoTerm).filter(MondoTerm.system == MONDO_SYSTEM, MondoTerm.code == canonical_code).one_or_none() + if term is not None: + return term + + term = MondoTerm(code=canonical_code, system=MONDO_SYSTEM, label=resolved["label"]) # type: ignore[call-arg] db.add(term) db.flush() diff --git a/tests/lib/mondo/test_mondo.py b/tests/lib/mondo/test_mondo.py index b50550970..10d2a22f9 100644 --- a/tests/lib/mondo/test_mondo.py +++ b/tests/lib/mondo/test_mondo.py @@ -161,6 +161,22 @@ async def fake_fetch(code): assert again.id == term.id assert session.query(MondoTerm).filter(MondoTerm.code == "MONDO:0015263").count() == 1 + async def test_non_canonical_code_reuses_existing_canonical_row(self, session, monkeypatch): + # OLS normalizes both spellings to the same canonical code; the lookup must key on the resolved + # code so a second submission reuses the row instead of tripping UNIQUE(system, code). + async def fake_fetch(code): + return {"code": "MONDO:0015263", "label": "Brugada syndrome", "iri": mondo_iri("MONDO:0015263")} + + monkeypatch.setattr("mavedb.lib.mondo.fetch_mondo_term", fake_fetch) + + canonical = await find_or_create_mondo_term(session, "MONDO:0015263") + # A differently-spelled submission (here lowercase) resolves to the same canonical term. + aliased = await find_or_create_mondo_term(session, "mondo:0015263") + + assert aliased.id == canonical.id + assert aliased.code == "MONDO:0015263" + assert session.query(MondoTerm).filter(MondoTerm.code == "MONDO:0015263").count() == 1 + async def test_generic_code_resolves_without_ols(self, session, monkeypatch): async def fail_fetch(code): raise AssertionError("OLS should not be consulted for the generic term") From cbb0c1a0010523f9f33054fb0c05507bd63ffecc Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Fri, 18 Sep 2026 12:44:06 -0700 Subject: [PATCH 159/252] feat(score-set): freeze scores after publish A published score set's scores are the record other work cites; re-uploading them would silently change results already in circulation and anything derived from the variants they recreate (mappings, calibration bin membership). Restrict SET_SCORES so owners and contributors may only set scores while the set is private; admins keep the ability as an operational escape hatch. Corrections go through a superseding set. The combined update-with-variants endpoint checks the permission before applying the update, since it commits the metadata half before enqueuing variant creation. --- src/mavedb/lib/permissions/score_set.py | 14 ++++-- src/mavedb/routers/score_sets.py | 43 ++++++++++------ tests/lib/permissions/test_score_set.py | 8 +-- tests/routers/test_score_set.py | 67 +++++++++++++++++++++++++ 4 files changed, 108 insertions(+), 24 deletions(-) diff --git a/src/mavedb/lib/permissions/score_set.py b/src/mavedb/lib/permissions/score_set.py index ba35eaf91..cb61a6ad6 100644 --- a/src/mavedb/lib/permissions/score_set.py +++ b/src/mavedb/lib/permissions/score_set.py @@ -234,6 +234,12 @@ def _handle_set_scores_action( Only owners, contributors, and admins can modify the scores data within a ScoreSet. This is a critical operation that affects the scientific data. + Publishing freezes that data: a published score set's scores are the record other work cites, and + re-uploading them would silently change results already in circulation, along with anything + derived from the variants they create (mappings, and the bin membership of any calibration built + on the score set). Corrections to published data go through a superseding score set instead. + Admins keep the ability as an operational escape hatch, mirroring DELETE. + Args: user_data: The user's authentication data. entity: The ScoreSet entity whose scores are being modified. @@ -246,12 +252,12 @@ def _handle_set_scores_action( PermissionResponse: Permission result with appropriate HTTP status. """ ## Allow set scores access under the following conditions: - # The owner or contributors may set scores. - if user_is_owner or user_is_contributor: - return PermissionResponse(True) - # Users with these specific roles may set scores. + # Users with these specific roles may set scores on any score set. if roles_permitted(active_roles, [UserRole.admin]): return PermissionResponse(True) + # The owner or contributors may set scores only while the score set is still private. + if (user_is_owner or user_is_contributor) and private: + return PermissionResponse(True) return deny_action_for_entity(entity, private, user_data, user_is_contributor or user_is_owner, "score set") diff --git a/src/mavedb/routers/score_sets.py b/src/mavedb/routers/score_sets.py index 66fda2d15..a4dca8edb 100644 --- a/src/mavedb/routers/score_sets.py +++ b/src/mavedb/routers/score_sets.py @@ -2318,6 +2318,27 @@ async def update_score_set_with_variants( logger.info(msg="Failed to update score set; The requested score set does not exist.", extra=logging_context()) raise HTTPException(status_code=404, detail=f"score set with URN '{urn}' not found") + existing_score_columns_metadata = (existing_item.dataset_columns or {}).get("score_columns_metadata", {}) + existing_count_columns_metadata = (existing_item.dataset_columns or {}).get("count_columns_metadata", {}) + + did_score_columns_metadata_change = ( + dataset_column_metadata.get("score_columns_metadata", {}) != existing_score_columns_metadata + ) + did_count_columns_metadata_change = ( + dataset_column_metadata.get("count_columns_metadata", {}) != existing_count_columns_metadata + ) + request_sets_scores = ( + did_score_columns_metadata_change + or did_count_columns_metadata_change + or any([val is not None for val in score_set_variants_data.values()]) + ) + + # Checked before the update is applied, not just before the job is enqueued: score_set_update + # commits the metadata half of this request, so refusing afterwards would leave a caller who may + # not set scores with a half-applied update and no way to tell which half landed. + if request_sets_scores: + assert_permission(user_data, existing_item, Action.SET_SCORES) + itemUpdateResult = await score_set_update( db=db, urn=urn, @@ -2329,24 +2350,9 @@ async def update_score_set_with_variants( updatedItem = itemUpdateResult["item"] should_create_variants = itemUpdateResult.get("should_create_variants", False) - existing_score_columns_metadata = (existing_item.dataset_columns or {}).get("score_columns_metadata", {}) - existing_count_columns_metadata = (existing_item.dataset_columns or {}).get("count_columns_metadata", {}) - - did_score_columns_metadata_change = ( - dataset_column_metadata.get("score_columns_metadata", {}) != existing_score_columns_metadata - ) - did_count_columns_metadata_change = ( - dataset_column_metadata.get("count_columns_metadata", {}) != existing_count_columns_metadata - ) - # run variant creation job only if targets have changed (indicated by "should_create_variants"), new score # or count files were uploaded, or dataset column metadata has changed - if ( - should_create_variants - or did_score_columns_metadata_change - or did_count_columns_metadata_change - or any([val is not None for val in score_set_variants_data.values()]) - ): + if should_create_variants or request_sets_scores: assert_permission(user_data, updatedItem, Action.SET_SCORES) updatedItem.processing_state = ProcessingState.processing @@ -2431,6 +2437,11 @@ async def update_score_set( should_create_variants = itemUpdateResult["should_create_variants"] if should_create_variants: + # Structurally unreachable for a published score set, since score_set_update only sets this + # flag while the score set is private. Asserted anyway so every path that recreates variants + # states the same requirement, and a future change to that flag cannot open a hole here. + assert_permission(user_data, updatedItem, Action.SET_SCORES) + # Although this is also updated within the variant creation job, update it here # as well so that we can display the proper UI components (queue invocation delay # races the score set GET request). diff --git a/tests/lib/permissions/test_score_set.py b/tests/lib/permissions/test_score_set.py index 7198fba97..2d5a0c497 100644 --- a/tests/lib/permissions/test_score_set.py +++ b/tests/lib/permissions/test_score_set.py @@ -245,12 +245,12 @@ class TestScoreSetSetScoresActionHandler: # Admins can set scores on any ScoreSet PermissionTest("ScoreSet", "private", "admin", Action.SET_SCORES, True), PermissionTest("ScoreSet", "published", "admin", Action.SET_SCORES, True), - # Owners can set scores on any ScoreSet they own + # Owners can only set scores on ScoreSets they own that are still unpublished PermissionTest("ScoreSet", "private", "owner", Action.SET_SCORES, True), - PermissionTest("ScoreSet", "published", "owner", Action.SET_SCORES, True), - # Contributors can set scores on any ScoreSet they contribute to + PermissionTest("ScoreSet", "published", "owner", Action.SET_SCORES, False, 403), + # Contributors are likewise limited to unpublished ScoreSets PermissionTest("ScoreSet", "private", "contributor", Action.SET_SCORES, True), - PermissionTest("ScoreSet", "published", "contributor", Action.SET_SCORES, True), + PermissionTest("ScoreSet", "published", "contributor", Action.SET_SCORES, False, 403), # Mappers cannot set scores on ScoreSets PermissionTest("ScoreSet", "private", "mapper", Action.SET_SCORES, False, 404), PermissionTest("ScoreSet", "published", "mapper", Action.SET_SCORES, False, 403), diff --git a/tests/routers/test_score_set.py b/tests/routers/test_score_set.py index d91bdea13..4f3f08946 100644 --- a/tests/routers/test_score_set.py +++ b/tests/routers/test_score_set.py @@ -1222,6 +1222,73 @@ def test_add_score_set_variants_scores_only_endpoint(client, setup_router_db, da assert score_set == response_data +def test_add_score_set_variants_is_refused_on_a_published_score_set( + session, data_provider, client, setup_router_db, data_files, mock_s3_client +): + """Publishing freezes a score set's scores; the upload endpoint must refuse them afterwards. + + The UI only offers score editing while a score set is private, but that is not a guarantee: the + endpoint is reachable directly, and a re-upload would replace variants other records already + point at. + """ + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") + + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published = publish_score_set(client, score_set["urn"]) + + scores_csv_path = data_files / "scores.csv" + with ( + open(scores_csv_path, "rb") as scores_file, + patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as queue, + patch.object(mock_s3_client, "upload_fileobj", return_value=None), + ): + response = client.post( + f"/api/v1/score-sets/{published['urn']}/variants/data", + files={"scores_file": (scores_csv_path.name, scores_file, "text/csv")}, + ) + # The refusal must land before any work is queued or uploaded. + queue.assert_not_called() + + assert response.status_code == 403 + + +def test_patch_published_score_set_with_scores_file_is_refused_without_applying_the_update( + session, data_provider, client, setup_router_db, data_files, mock_s3_client +): + """A refused score upload must not leave the request's metadata half committed. + + The combined endpoint commits metadata before it reaches the enqueue step, so the SET_SCORES + check has to run before the update is applied rather than only before the job is queued. + """ + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") + + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published = publish_score_set(client, score_set["urn"]) + + scores_csv_path = data_files / "scores.csv" + with ( + open(scores_csv_path, "rb") as scores_file, + patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as queue, + patch.object(mock_s3_client, "upload_fileobj", return_value=None), + ): + response = client.patch( + f"/api/v1/score-sets-with-variants/{published['urn']}", + data={"title": "Retitled after publication"}, + files={"scores_file": (scores_csv_path.name, scores_file, "text/csv")}, + ) + queue.assert_not_called() + + assert response.status_code == 403 + + # The title edit rode along with the refused upload, so it must not have been applied either. + refreshed = client.get(f"/api/v1/score-sets/{published['urn']}").json() + assert refreshed["title"] == published["title"] + + def test_add_score_set_variants_scores_and_counts_endpoint( session, client, setup_router_db, data_files, mock_s3_client ): From ed1debee593ea2e976c042a269a612cd35bb6b66 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Fri, 18 Sep 2026 12:44:34 -0700 Subject: [PATCH 160/252] feat(calibration-controls): serve each control's functional-classification placement Expose which of the calibration's own functional classifications a control's variant falls into by bin membership, so the concordance view can cross a control's clinical status against where its score landed. Compute it as a correlated-subquery column_property on CalibrationControl rather than storing it, so it can never drift from the bin assignments it reports. Classified bins never overlap, so at most one applies; a classified bin is preferred over a 'not_specified' one, then lowest id breaks ties, for a deterministic result. --- src/mavedb/models/calibration_control.py | 46 ++++++- src/mavedb/view_models/calibration_control.py | 5 +- tests/lib/test_calibration_control.py | 115 +++++++++++++++++ tests/routers/test_calibration_controls.py | 118 +++++++++++++++++- tests/view_models/test_calibration_control.py | 2 + 5 files changed, 281 insertions(+), 5 deletions(-) diff --git a/src/mavedb/models/calibration_control.py b/src/mavedb/models/calibration_control.py index 363756616..144a1aeb0 100644 --- a/src/mavedb/models/calibration_control.py +++ b/src/mavedb/models/calibration_control.py @@ -10,13 +10,18 @@ from __future__ import annotations from datetime import date -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Optional -from sqlalchemy import Column, Date, Enum, ForeignKey, Integer, UniqueConstraint -from sqlalchemy.orm import Mapped, relationship +from sqlalchemy import Column, Date, Enum, ForeignKey, Integer, UniqueConstraint, case, select +from sqlalchemy.orm import Mapped, column_property, relationship from mavedb.db.base import Base from mavedb.models.enums.calibration_control_status import CalibrationControlStatus +from mavedb.models.enums.functional_classification import FunctionalClassification +from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification +from mavedb.models.score_calibration_functional_classification_variant_association import ( + score_calibration_functional_classification_variants_association_table, +) if TYPE_CHECKING: from mavedb.models.score_calibration import ScoreCalibration @@ -58,6 +63,41 @@ class CalibrationControl(Base): variant_id = Column(Integer, ForeignKey("variants.id"), nullable=False, index=True) variant: Mapped["Variant"] = relationship("Variant") + # Which of the calibration's own functional classifications contains this control's variant by bin + # membership, or NULL when it lands under none. Computed live from the membership association rather + # than stored, so it can never drift from the clinical bin assignments it reports. Mirrors the + # ``variant_count`` correlated-subquery pattern on ScoreCalibrationFunctionalClassification. Ranges may + # overlap when one is 'not_specified' (see ScoreCalibrationBase.ranges_do_not_overlap), so a variant can + # fall in several bins. Classified bins never overlap each other, so at most one applies; it is + # preferred over a 'not_specified' bin (which asserts no classification), then the lowest id breaks + # any remaining tie, for a deterministic result. + functional_classification_id: Mapped[Optional[int]] = column_property( + select(ScoreCalibrationFunctionalClassification.id) + .where( + ScoreCalibrationFunctionalClassification.calibration_id == calibration_id, + score_calibration_functional_classification_variants_association_table.c.functional_classification_id + == ScoreCalibrationFunctionalClassification.id, + score_calibration_functional_classification_variants_association_table.c.variant_id == variant_id, + ) + .order_by( + case( + ( + ScoreCalibrationFunctionalClassification.functional_classification + == FunctionalClassification.not_specified, + 1, + ), + else_=0, + ), + ScoreCalibrationFunctionalClassification.id, + ) + .limit(1) + .correlate_except( + ScoreCalibrationFunctionalClassification, + score_calibration_functional_classification_variants_association_table, + ) + .scalar_subquery() + ) + clinical_status = Column( Enum(CalibrationControlStatus, native_enum=False, validate_strings=True, length=32), nullable=False, diff --git a/src/mavedb/view_models/calibration_control.py b/src/mavedb/view_models/calibration_control.py index b6f48e020..9d1871ba6 100644 --- a/src/mavedb/view_models/calibration_control.py +++ b/src/mavedb/view_models/calibration_control.py @@ -7,7 +7,7 @@ """ from datetime import date -from typing import Any +from typing import Any, Optional from pydantic import model_validator @@ -54,6 +54,9 @@ class SavedCalibrationControl(CalibrationControlBase): _record_type_factory = record_type_validator()(set_record_type) id: int + # The calibration's own functional classification (range or class) which contains this control's + # variant, or None when the variant lands under none of them. + functional_classification_id: Optional[int] = None creation_date: date modification_date: date created_by: SavedUser diff --git a/tests/lib/test_calibration_control.py b/tests/lib/test_calibration_control.py index 7a04a1f28..4abc3ae6d 100644 --- a/tests/lib/test_calibration_control.py +++ b/tests/lib/test_calibration_control.py @@ -18,8 +18,10 @@ from mavedb.lib.validation.exceptions import ValidationError from mavedb.models.calibration_control import CalibrationControl from mavedb.models.enums.calibration_control_status import CalibrationControlStatus +from mavedb.models.enums.functional_classification import FunctionalClassification from mavedb.lib.mondo import get_generic_disease_term from mavedb.models.score_calibration import ScoreCalibration +from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification from mavedb.models.score_set import ScoreSet from mavedb.models.user import User from mavedb.models.variant import Variant @@ -123,6 +125,119 @@ def test_deleting_calibration_cascades_to_controls(session, setup_lib_db_with_va assert session.get(CalibrationControl, control_id) is None +############################################################################## +# functional_classification_id placement (computed column_property) +############################################################################## + + +def _classification(calibration, label, functional_classification, score_range): + return ScoreCalibrationFunctionalClassification( + calibration=calibration, + label=label, + functional_classification=functional_classification, + range=score_range, + inclusive_lower_bound=True, + inclusive_upper_bound=False, + ) + + +def test_control_placement_prefers_classified_bin_over_not_specified(session, setup_lib_db_with_variant): + """A control in a not_specified-vs-classified overlap resolves to the classified bin. + + Overlapping ranges are only possible when one bin is 'not_specified' (see + ScoreCalibrationBase.ranges_do_not_overlap), and membership is built per-range, so the control's + variant lands in both bins. The placement must surface the clinically meaningful classified bin even + when the not_specified bin has the lower id — the tiebreak a plain id ordering would wrongly pick. + """ + variant = setup_lib_db_with_variant + user = session.query(User).filter(User.username == TEST_USER["username"]).first() + calibration = _make_calibration(session, variant.score_set_id, user) + + # Create the not_specified bin first so it takes the lower id: this is the value a plain + # ``ORDER BY id`` would select, so asserting against the classified bin proves classified wins. + not_specified_bin = _classification( + calibration, "unclassified", FunctionalClassification.not_specified, [-2.0, 2.0] + ) + session.add(not_specified_bin) + session.commit() + + classified_bin = _classification(calibration, "abnormal", FunctionalClassification.abnormal, [0.0, 1.0]) + session.add(classified_bin) + session.commit() + + assert not_specified_bin.id < classified_bin.id + + # The variant sits in the overlap [0.0, 1.0], so it is a member of both bins. + not_specified_bin.variants = [variant] + classified_bin.variants = [variant] + session.commit() + + control = CalibrationControl( + calibration=calibration, + variant=variant, + clinical_status=CalibrationControlStatus.pathogenic, + created_by=user, + modified_by=user, + ) + session.add(control) + session.commit() + session.refresh(control) + + assert control.functional_classification_id == classified_bin.id + + +def test_control_placement_falls_back_to_not_specified_when_no_classified_bin(session, setup_lib_db_with_variant): + """With no classified bin containing the variant, placement honestly reports the not_specified bin.""" + variant = setup_lib_db_with_variant + user = session.query(User).filter(User.username == TEST_USER["username"]).first() + calibration = _make_calibration(session, variant.score_set_id, user) + + not_specified_bin = _classification( + calibration, "unclassified", FunctionalClassification.not_specified, [-2.0, 2.0] + ) + session.add(not_specified_bin) + session.commit() + not_specified_bin.variants = [variant] + session.commit() + + control = CalibrationControl( + calibration=calibration, + variant=variant, + clinical_status=CalibrationControlStatus.benign, + created_by=user, + modified_by=user, + ) + session.add(control) + session.commit() + session.refresh(control) + + assert control.functional_classification_id == not_specified_bin.id + + +def test_control_placement_is_null_when_variant_in_no_bin(session, setup_lib_db_with_variant): + """A control whose variant falls under none of the calibration's classifications has no placement.""" + variant = setup_lib_db_with_variant + user = session.query(User).filter(User.username == TEST_USER["username"]).first() + calibration = _make_calibration(session, variant.score_set_id, user) + + classified_bin = _classification(calibration, "abnormal", FunctionalClassification.abnormal, [0.0, 1.0]) + session.add(classified_bin) + session.commit() # bin exists but the variant is never added to its membership + + control = CalibrationControl( + calibration=calibration, + variant=variant, + clinical_status=CalibrationControlStatus.pathogenic, + created_by=user, + modified_by=user, + ) + session.add(control) + session.commit() + session.refresh(control) + + assert control.functional_classification_id is None + + ############################################################################## # validate_calibration_controls_in_score_set (#751) ############################################################################## diff --git a/tests/routers/test_calibration_controls.py b/tests/routers/test_calibration_controls.py index cf47e45c1..d75be1af3 100644 --- a/tests/routers/test_calibration_controls.py +++ b/tests/routers/test_calibration_controls.py @@ -8,7 +8,7 @@ import json -from sqlalchemy import select +from sqlalchemy import delete, select from mavedb.lib.validation.constants.general import ( calibration_control_status_column_name, @@ -17,6 +17,9 @@ from mavedb.models.calibration_control import CalibrationControl from mavedb.models.enums.calibration_control_status import CalibrationControlStatus from mavedb.models.score_calibration import ScoreCalibration as CalibrationDbModel +from mavedb.models.score_calibration_functional_classification_variant_association import ( + score_calibration_functional_classification_variants_association_table, +) from mavedb.models.score_set import ScoreSet as ScoreSetDbModel from mavedb.models.variant import Variant from tests.helpers.constants import ( @@ -520,3 +523,116 @@ def test_can_update_published_calibration_leaving_affirmed_controls_untouched_as assert response.status_code == 200, response.text assert response.json()["controlsNotPhi"] is True assert len(response.json()["controls"]) == 1 + + +########################################################### +# Control placement (functional_classification_id) — Option A +########################################################### + + +def _add_control_on_variant(session, calibration, variant, status=CalibrationControlStatus.pathogenic): + """Attach one control to ``calibration`` on a chosen ``variant`` at the ORM level.""" + session.add( + CalibrationControl( + calibration=calibration, + variant=variant, + clinical_status=status, + created_by=calibration.created_by, + modified_by=calibration.created_by, + ) + ) + session.commit() + + +def _placement_by_variant_id(calibration): + """Ground-truth ``{variant_id: functional_classification_id}`` from stored bin membership.""" + placement = {} + for classification in calibration.functional_classifications: + for variant in classification.variants: + placement[variant.id] = classification.id + return placement + + +@pytest.mark.parametrize("mock_publication_fetch", [CALIBRATION_PUBLICATIONS], indirect=["mock_publication_fetch"]) +def test_get_calibration_populates_control_placement( + client, setup_router_db, mock_publication_fetch, session, data_provider, data_files +): + """A control whose variant a range files reports that range's id as its placement.""" + calibration = _create_private_calibration(client, session, data_provider, data_files) + calibration_orm = session.query(CalibrationDbModel).where(CalibrationDbModel.urn == calibration["urn"]).one() + + placement = _placement_by_variant_id(calibration_orm) + assert placement, "fixture calibration should file at least one variant into a range" + + variant_id, expected_classification_id = next(iter(placement.items())) + variant = session.get(Variant, variant_id) + _add_control_on_variant(session, calibration_orm, variant) + + response = client.get(f"/api/v1/score-calibrations/{calibration['urn']}") + assert response.status_code == 200, response.text + + control = next(c for c in response.json()["controls"] if c["variantUrn"] == variant.urn) + assert control["functionalClassificationId"] == expected_classification_id + + +@pytest.mark.parametrize("mock_publication_fetch", [CALIBRATION_PUBLICATIONS], indirect=["mock_publication_fetch"]) +def test_control_placement_is_none_when_variant_has_no_bin( + client, setup_router_db, mock_publication_fetch, session, data_provider, data_files +): + """A control whose variant is filed under none of the calibration's ranges reports null placement.""" + calibration = _create_private_calibration(client, session, data_provider, data_files) + calibration_orm = session.query(CalibrationDbModel).where(CalibrationDbModel.urn == calibration["urn"]).one() + + variant = session.scalars(select(Variant).where(Variant.score_set_id == calibration_orm.score_set_id)).first() + _add_control_on_variant(session, calibration_orm, variant) + + # Strip any bin membership for this variant so its placement must resolve to None. + association = score_calibration_functional_classification_variants_association_table + session.execute(delete(association).where(association.c.variant_id == variant.id)) + session.commit() + + response = client.get(f"/api/v1/score-calibrations/{calibration['urn']}") + assert response.status_code == 200, response.text + + control = next(c for c in response.json()["controls"] if c["variantUrn"] == variant.urn) + assert control["functionalClassificationId"] is None + + +@pytest.mark.parametrize("mock_publication_fetch", [CALIBRATION_PUBLICATIONS], indirect=["mock_publication_fetch"]) +def test_control_placement_scoped_to_own_calibration( + client, setup_router_db, mock_publication_fetch, session, data_provider, data_files +): + """Placement reflects the control's own calibration, not another that happens to bin the variant.""" + calibration = _create_private_calibration(client, session, data_provider, data_files) + calibration_orm = session.query(CalibrationDbModel).where(CalibrationDbModel.urn == calibration["urn"]).one() + + # Capture plain ids/urns up front; creating the second calibration commits and expires ORM objects. + score_set_urn = calibration_orm.score_set.urn + placement = _placement_by_variant_id(calibration_orm) + assert placement + binned_variant_id = next(iter(placement)) + binned_variant_urn = session.get(Variant, binned_variant_id).urn + + # A second calibration on the same score set bins the same variant identically; clear only the + # second's membership for it, so the first calibration's binning must not leak into the second's + # placement. + second = create_test_score_calibration_in_score_set_via_client( + client, score_set_urn, deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) + ) + second_orm = session.query(CalibrationDbModel).where(CalibrationDbModel.urn == second["urn"]).one() + + association = score_calibration_functional_classification_variants_association_table + second_classification_ids = [fc.id for fc in second_orm.functional_classifications] + session.execute( + delete(association).where( + association.c.variant_id == binned_variant_id, + association.c.functional_classification_id.in_(second_classification_ids), + ) + ) + _add_control_on_variant(session, second_orm, session.get(Variant, binned_variant_id)) + + response = client.get(f"/api/v1/score-calibrations/{second['urn']}") + assert response.status_code == 200, response.text + + control = next(c for c in response.json()["controls"] if c["variantUrn"] == binned_variant_urn) + assert control["functionalClassificationId"] is None diff --git a/tests/view_models/test_calibration_control.py b/tests/view_models/test_calibration_control.py index 3f1bf5b5b..587782846 100644 --- a/tests/view_models/test_calibration_control.py +++ b/tests/view_models/test_calibration_control.py @@ -141,6 +141,8 @@ def test_saved_score_calibration_includes_controls_and_disease_concept(): assert len(calibration.controls) == 1 assert calibration.controls[0].variant_urn == TEST_CONTROL_VARIANT_URN assert calibration.controls[0].clinical_status is CalibrationControlStatus.pathogenic + # Placement is a computed column_property; an in-memory object never carries it, so it stays unset. + assert calibration.controls[0].functional_classification_id is None def test_saved_score_calibration_controls_default_to_empty_list(): From 0f6098947fc0a35898359d78ddaea38cbbef10cb Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Fri, 18 Sep 2026 12:44:46 -0700 Subject: [PATCH 161/252] feat(calibration-controls): preserve controls and class bins across a score re-upload Re-uploading a score set deletes and recreates its variants, breaking the foreign keys calibration controls and class-based bin membership hold into them (RESTRICT, so the delete would otherwise fail). Snapshot the irreproducible references by HGVS identity before the delete, then re-resolve them against the new variants; range-based bins are recomputed from the new scores instead. A dropped control resets controls_not_phi, and provenance (author, date) is carried across rather than reattributed to the re-uploader. Also consolidates control-variant validation and construction onto a single scoped variant query instead of one lookup to validate and another to build. --- src/mavedb/lib/score_calibrations.py | 366 ++++++++++++-- src/mavedb/lib/types/score_calibrations.py | 81 ++- .../jobs/variant_processing/creation.py | 46 ++ .../jobs/variant_processing/test_creation.py | 460 ++++++++++++++++++ 4 files changed, 906 insertions(+), 47 deletions(-) diff --git a/src/mavedb/lib/score_calibrations.py b/src/mavedb/lib/score_calibrations.py index 5d270e043..627d607b7 100644 --- a/src/mavedb/lib/score_calibrations.py +++ b/src/mavedb/lib/score_calibrations.py @@ -1,8 +1,10 @@ """Utilities for building and mutating score calibration ORM objects.""" +import logging import math from collections import Counter -from typing import Optional, Sequence, Union +from datetime import date +from typing import Optional, Sequence, Union, cast import pandas as pd from sqlalchemy import Float, and_, select @@ -11,7 +13,13 @@ from mavedb.lib.acmg import find_or_create_acmg_classification from mavedb.lib.identifiers import find_or_create_publication_identifier from mavedb.lib.mondo import resolve_disease_term -from mavedb.lib.types.score_calibrations import ClassificationDict +from mavedb.lib.types.score_calibrations import ( + CalibrationControlSnapshot, + CalibrationVariantLinkSnapshot, + CalibrationVariantRelinkReport, + ClassificationDict, + VariantIdentity, +) from mavedb.lib.validation.constants.general import ( calibration_class_column_name, calibration_variant_column_name, @@ -24,12 +32,71 @@ from mavedb.models.enums.score_calibration_relation import ScoreCalibrationRelation from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification +from mavedb.models.score_calibration_functional_classification_variant_association import ( + score_calibration_functional_classification_variants_association_table, +) from mavedb.models.score_calibration_publication_identifier import ScoreCalibrationPublicationIdentifierAssociation from mavedb.models.score_set import ScoreSet from mavedb.models.user import User from mavedb.models.variant import Variant from mavedb.view_models import score_calibration +logger = logging.getLogger(__name__) + + +def _fetch_control_variants_by_urn( + db: Session, + score_set: ScoreSet, + controls: Sequence[score_calibration.CalibrationControlCreate], +) -> dict[str, Variant]: + """Fetch the ``Variant`` rows for a submission's control URNs, scoped to the score set, keyed by URN. + + A single scoped query serves both membership validation (a URN absent from the result does not + belong to the score set) and row construction, so validating and building a submission share one + lookup rather than each issuing its own. + """ + submitted_urns = [control.variant_urn for control in controls] + return { + variant.urn: variant + for variant in db.scalars( + select(Variant).where(Variant.score_set_id == score_set.id, Variant.urn.in_(submitted_urns)) + ).all() + # Every returned row matched an IN over the submitted (non-null) URNs, so its urn is non-null; + # the guard both states that and narrows the key type from Optional[str]. + if variant.urn is not None + } + + +def _validate_control_submission( + controls: Sequence[score_calibration.CalibrationControlCreate], + variants_by_urn: dict[str, Variant], +) -> None: + """Duplicate and score-set-membership checks for a control submission, given its resolved variants. + + A control earns its role as calibration evidence from a variant the assay actually scored, so each + must reference a variant belonging to the calibration's own score set. Duplicate URNs within one + submission are caught here with a readable message rather than deferred to the + ``UNIQUE(calibration_id, variant_id)`` database constraint. + + Raises: + ValidationError: If any control URN is duplicated within the submission, absent from MaveDB, + or belongs to a different score set. + """ + counts = Counter(control.variant_urn for control in controls) + + duplicate_urns = {urn for urn, count in counts.items() if count > 1} + if duplicate_urns: + raise ValidationError( + f"Duplicate control variant URNs detected within the submission: {', '.join(sorted(duplicate_urns))}." + ) + + missing_urns = counts.keys() - variants_by_urn.keys() + if missing_urns: + raise ValidationError( + "The following control variants do not belong to the calibration's score set: " + f"{', '.join(sorted(missing_urns))}." + ) + def validate_calibration_controls_in_score_set( db: Session, @@ -38,16 +105,9 @@ def validate_calibration_controls_in_score_set( ) -> list[str]: """Validate calibration controls against a score set and return their variant URNs. - A control earns its role as calibration evidence from a variant the assay actually scored, - so each control must reference a variant belonging to the calibration's own score set. A - single scoped query resolves all submitted URNs at once; any URN it does not return either - does not exist in MaveDB or belongs to a different score set, and both are rejected together. - - Duplicate variant URNs within one submission are caught here with a readable message rather - than deferred to the ``UNIQUE(calibration_id, variant_id)`` database constraint. - - The validated URNs are returned so a caller that goes on to persist the controls can reuse - them (see :func:`build_calibration_controls`). + A thin wrapper over :func:`_fetch_control_variants_by_urn` and :func:`_validate_control_submission` + for callers that only need the validated URNs; :func:`build_calibration_controls` shares the same + two steps to avoid re-fetching the variants. Args: db: Database session used for the lookup. @@ -64,28 +124,9 @@ def validate_calibration_controls_in_score_set( if not controls: return [] - submitted_urns = [control.variant_urn for control in controls] - - counts = Counter(submitted_urns) - duplicate_urns = {urn for urn, count in counts.items() if count > 1} - if duplicate_urns: - raise ValidationError( - f"Duplicate control variant URNs detected within the submission: {', '.join(sorted(duplicate_urns))}." - ) - - existing_urns = set( - db.scalars( - select(Variant.urn).where(Variant.score_set_id == score_set.id, Variant.urn.in_(submitted_urns)) - ).all() - ) - missing_urns = set(counts) - existing_urns - if missing_urns: - raise ValidationError( - "The following control variants do not belong to the calibration's score set: " - f"{', '.join(sorted(missing_urns))}." - ) - - return submitted_urns + variants_by_urn = _fetch_control_variants_by_urn(db, score_set, controls) + _validate_control_submission(controls, variants_by_urn) + return [control.variant_urn for control in controls] def build_calibration_controls( @@ -96,11 +137,12 @@ def build_calibration_controls( ) -> list[CalibrationControl]: """Validate and construct transient ``CalibrationControl`` rows for a calibration. - Validation (see :func:`validate_calibration_controls_in_score_set`) returns the validated - variant URNs, which this function resolves to their ``Variant`` rows in one query and turns - into unattached ``CalibrationControl`` rows with audit fields set. The caller assigns them to - the calibration's ``controls`` collection and commits. Returns an empty list for ``None`` or - empty input. + Fetches the submission's ``Variant`` rows once, validates them (duplicates + score-set membership, + see :func:`_validate_control_submission`), then turns each control into an unattached + ``CalibrationControl`` with audit fields set. The single fetch is reused for validation and + construction, so this issues one variant query rather than one to validate and another to build. + The caller assigns the returned rows to the calibration's ``controls`` collection and commits. + Returns an empty list for ``None`` or empty input. Args: db: Database session. @@ -108,16 +150,11 @@ def build_calibration_controls( controls: Submitted controls, or ``None``/empty for none. user: The acting user, recorded on each control's audit fields. """ - submitted_urns = validate_calibration_controls_in_score_set(db, score_set, controls) if not controls: return [] - variants_by_urn = { - variant.urn: variant - for variant in db.scalars( - select(Variant).where(Variant.score_set_id == score_set.id, Variant.urn.in_(submitted_urns)) - ).all() - } + variants_by_urn = _fetch_control_variants_by_urn(db, score_set, controls) + _validate_control_submission(controls, variants_by_urn) return [ CalibrationControl( @@ -964,3 +1001,240 @@ def variant_classification_df_to_dict( classifications[functional_class].add(index_element) return {"indexed_by": index_column, "classifications": classifications} + + +def snapshot_calibration_variant_links(db: Session, score_set: ScoreSet) -> list[CalibrationVariantLinkSnapshot]: + """Record the calibration variant references a re-upload cannot reconstruct on its own. + + Re-uploading a score set's data deletes and recreates all of its ``Variant`` rows, which breaks + the foreign keys calibrations hold into them: ``calibration_controls.variant_id`` and the + functional-classification membership association. Neither carries an ``ON DELETE`` action, on + purpose — the resulting ``RESTRICT`` protects hand-entered controls from being destroyed by an + unrelated delete. Capturing the irreproducible references as :data:`VariantIdentity` tuples before + the delete lets :func:`restore_calibration_variant_links` re-point the survivors afterwards. + Variant URNs cannot serve as the key because they are renumbered on every upload. + + The rule for what to capture is whether the depositor asserted the reference or MaveDB derived it: + + * **Controls** carry clinical significance sourced from outside MaveDB, submitted inline or as a + ``controls_file``. Nothing in a score upload can regenerate them, so they are remembered. + * **Class-based bin membership** comes from the ``classes_file`` required of a class-based + calibration. That file is not part of a score upload either, so it too survives only by identity. + * **Range-based bin membership** is derived — purely a function of the variants' scores. It is + left out and recomputed from the new upload instead, which avoids holding an entry per variant + in memory and keeps membership honest when a re-upload moves a score across a threshold. + + The queries scope to variants in ``score_set``, which keeps the snapshot aligned with exactly the + rows the caller deletes. Every reference is already so scoped — controls are validated against the + calibration's own score set and bin membership is only ever populated from it — so the condition + holds the two halves together rather than filtering anything out. Were they to diverge, restore + would re-create a row whose original was never deleted and trip the unique constraint on + ``(calibration_id, variant_id)``. + + This function only reads. The caller performs the deletes. + + Args: + db: Database session. + score_set: The score set whose variants are about to be replaced. + + Returns: + One snapshot per calibration holding at least one irreproducible reference; empty when no + calibration has one. A score set whose calibrations use only range-based bins yields an empty + list and still needs :func:`restore_calibration_variant_links` called to re-bin them. + """ + calibration_ids = [calibration.id for calibration in score_set.score_calibrations] + if not calibration_ids: + return [] + + snapshots: dict[int, CalibrationVariantLinkSnapshot] = {} + + control_rows = db.execute( + select( + CalibrationControl.calibration_id, + CalibrationControl.clinical_status, + CalibrationControl.created_by_id, + CalibrationControl.creation_date, + Variant.hgvs_nt, + Variant.hgvs_pro, + Variant.hgvs_splice, + ) + .join(Variant, Variant.id == CalibrationControl.variant_id) + .where(CalibrationControl.calibration_id.in_(calibration_ids), Variant.score_set_id == score_set.id) + ).all() + + for calibration_id, clinical_status, created_by_id, creation_date, hgvs_nt, hgvs_pro, hgvs_splice in control_rows: + snapshot = snapshots.setdefault(calibration_id, CalibrationVariantLinkSnapshot(calibration_id=calibration_id)) + snapshot.controls.append( + CalibrationControlSnapshot( + identity=(hgvs_nt, hgvs_pro, hgvs_splice), + clinical_status=clinical_status, + created_by_id=created_by_id, + creation_date=creation_date, + ) + ) + + association = score_calibration_functional_classification_variants_association_table + membership_rows = db.execute( + select( + ScoreCalibrationFunctionalClassification.calibration_id, + ScoreCalibrationFunctionalClassification.id, + Variant.hgvs_nt, + Variant.hgvs_pro, + Variant.hgvs_splice, + ) + .join(association, association.c.functional_classification_id == ScoreCalibrationFunctionalClassification.id) + .join(Variant, Variant.id == association.c.variant_id) + .where( + ScoreCalibrationFunctionalClassification.calibration_id.in_(calibration_ids), + ScoreCalibrationFunctionalClassification.class_.is_not(None), + Variant.score_set_id == score_set.id, + ) + ).all() + + for calibration_id, classification_id, hgvs_nt, hgvs_pro, hgvs_splice in membership_rows: + snapshot = snapshots.setdefault(calibration_id, CalibrationVariantLinkSnapshot(calibration_id=calibration_id)) + snapshot.classification_members.setdefault(classification_id, []).append((hgvs_nt, hgvs_pro, hgvs_splice)) + + return list(snapshots.values()) + + +def restore_calibration_variant_links( + db: Session, + score_set: ScoreSet, + snapshots: Sequence[CalibrationVariantLinkSnapshot], + updater: User, +) -> CalibrationVariantRelinkReport: + """Re-establish a score set's calibration variant references after its variants are recreated. + + Each calibration is handled by the two mechanisms its references call for: + + * **Controls and class-based bin membership** are carried across by + :data:`VariantIdentity`. A reference whose identity is absent from the new upload describes a + variant the assay no longer scores, so it is dropped rather than guessed at; an identity + matching more than one new variant is ambiguous and dropped for the same reason. Dropping + rather than failing is deliberate: a re-upload that aborted on a vanished control would leave + depositors unable to correct their own data. + * **Range-based bin membership** is recomputed from the new scores via + :func:`variants_for_functional_classification`, the same helper that populated it originally. + Carrying the old membership across might leave a variant filed under a range its new score no + longer falls in, so the recorded membership would simply be wrong. + + A classification always has exactly one of ``range`` or ``class_`` set (enforced by the view + models), so every classification falls squarely into one branch or the other. + + Dropping a control changes the control set the submitter affirmed as free of protected health + information, so ``controls_not_phi`` is reset to ``None`` on that calibration — the same + re-acknowledgment rule applied when controls are replaced through the API (see + :func:`modify_score_calibration`). A calibration whose controls all relink keeps its affirmation. + Re-binning does not reset it: bin membership carries no clinical annotation and so no PHI. + + Changes are staged on the session; the caller commits. + + Args: + db: Database session. + score_set: The score set whose variants have just been recreated. + snapshots: Output of :func:`snapshot_calibration_variant_links`, taken before the delete. May + be empty while calibrations still need re-binning. + updater: The user who triggered the re-upload, recorded on the rows this relink touches. + + Returns: + Counts of relinked, dropped and re-binned references, for the caller's job log. + """ + report = CalibrationVariantRelinkReport() + if not score_set.score_calibrations: + return report + + # Only controls and class-based bin membership are re-resolved by identity; range-based bins are + # recomputed from the new scores below and never touch this map. Building it materializes every + # variant of the score set, so skip that entirely when nothing needs identity resolution — the + # common case of a re-upload whose calibrations use only range-based bins. + variants_by_identity: dict[VariantIdentity, Optional[Variant]] = {} + if any(snapshot.controls or snapshot.classification_members for snapshot in snapshots): + for new_variant in db.scalars(select(Variant).where(Variant.score_set_id == score_set.id)).all(): + identity = (new_variant.hgvs_nt, new_variant.hgvs_pro, new_variant.hgvs_splice) + # Upstream validation rejects duplicate variants, but a collision here would otherwise bind + # every reference to whichever row happened to be seen first. Mark it unresolvable instead. + variants_by_identity[identity] = None if identity in variants_by_identity else new_variant + + snapshots_by_calibration_id = {snapshot.calibration_id: snapshot for snapshot in snapshots} + + for calibration in score_set.score_calibrations: + calibration_id = cast(int, calibration.id) + snapshot = snapshots_by_calibration_id.get( + calibration_id, CalibrationVariantLinkSnapshot(calibration_id=calibration_id) + ) + + # The linkage rows were removed with Core deletes, which leave any loaded collections holding + # rows that no longer exist. Expire them so the ORM rebuilds from the post-delete state rather + # than issuing deletes for rows that are already gone. + db.expire(calibration, ["controls"]) + + dropped_controls = 0 + relinked_controls: list[CalibrationControl] = [] + for control in snapshot.controls: + control_variant = variants_by_identity.get(control.identity) + if control_variant is None: + dropped_controls += 1 + continue + + relinked_controls.append( + CalibrationControl( + calibration_id=calibration.id, + variant_id=control_variant.id, + clinical_status=control.clinical_status, + created_by_id=control.created_by_id, + creation_date=control.creation_date, + modified_by_id=updater.id, + modification_date=date.today(), + ) + ) + + db.add_all(relinked_controls) + report.controls_relinked += len(relinked_controls) + report.controls_dropped += dropped_controls + + dropped_members = 0 + for classification in calibration.functional_classifications: + db.expire(classification, ["variants"]) + + if classification.class_ is not None: + members: list[Variant] = [] + for identity in snapshot.classification_members.get(cast(int, classification.id), []): + member_variant = variants_by_identity.get(identity) + if member_variant is None: + dropped_members += 1 + continue + + members.append(member_variant) + + report.classification_members_relinked += len(members) + else: + members = variants_for_functional_classification(db, classification, use_sql=True) + report.classifications_rebinned += 1 + report.classification_members_rebinned += len(members) + + classification.variants = members + db.add(classification) + + report.classification_members_dropped += dropped_members + + if dropped_controls: + calibration.controls_not_phi = None + report.calibrations_pending_phi_reaffirmation.append(calibration_id) + + # Re-binning is the expected mechanical consequence of new scores, so it does not count as an + # edit. Losing a hand-entered reference does. + if dropped_controls or dropped_members: + calibration.modified_by = updater + db.add(calibration) + + if report.controls_dropped or report.classification_members_dropped: + logger.warning( + "Dropped %s calibration control(s) and %s class-based bin membership(s) from score set %s: their " + "variants are absent from the new upload.", + report.controls_dropped, + report.classification_members_dropped, + score_set.urn, + ) + + return report diff --git a/src/mavedb/lib/types/score_calibrations.py b/src/mavedb/lib/types/score_calibrations.py index d40edaf27..b6b4753bc 100644 --- a/src/mavedb/lib/types/score_calibrations.py +++ b/src/mavedb/lib/types/score_calibrations.py @@ -1,6 +1,85 @@ -from typing import TypedDict +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import date +from typing import Any, Optional, TypedDict + +from mavedb.models.enums.calibration_control_status import CalibrationControlStatus class ClassificationDict(TypedDict): indexed_by: str classifications: dict[str, set[str]] + + +VariantIdentity = tuple[Optional[str], Optional[str], Optional[str]] +"""A variant's ``(hgvs_nt, hgvs_pro, hgvs_splice)`` tuple. + +Variant URNs are positional within a score set (``{score_set.urn}#{n}``, renumbered from one on every +upload), so ``#5`` can name a different biological variant after a re-upload. The HGVS tuple is the +identity that survives one, and so is what calibration variant references are re-resolved against. +""" + + +@dataclass(frozen=True) +class CalibrationControlSnapshot: + """One calibration control, recorded by variant identity instead of variant id. + + Audit provenance is carried alongside the identity so a relinked control keeps the authorship and + date of the original submission rather than appearing to have been entered by whoever re-uploaded + the scores. + """ + + identity: VariantIdentity + clinical_status: CalibrationControlStatus + created_by_id: int + creation_date: date + + +@dataclass +class CalibrationVariantLinkSnapshot: + """The variant references held by one calibration that a re-upload cannot reconstruct. + + Only irreproducible references are captured. Range-based bin membership is omitted on purpose: it + is a function of the variants' scores, so it is recomputed from the new upload rather than + remembered (see :func:`~mavedb.lib.score_calibrations.restore_calibration_variant_links`). + """ + + calibration_id: int + controls: list[CalibrationControlSnapshot] = field(default_factory=list) + + # Class-based bin membership keyed by functional classification id. Classification rows survive a + # re-upload, so their ids remain valid addresses when the membership is restored. + classification_members: dict[int, list[VariantIdentity]] = field(default_factory=dict) + + +@dataclass +class CalibrationVariantRelinkReport: + """Outcome of re-establishing a score set's calibration variant references after a re-upload. + + Relinked and dropped counts describe references carried across by identity (controls and + class-based bin membership). Re-binned counts describe range-based membership recomputed from the + new scores, where "dropped" has no meaning — the bin simply holds whatever now falls in it. + """ + + controls_relinked: int = 0 + controls_dropped: int = 0 + classification_members_relinked: int = 0 + classification_members_dropped: int = 0 + classifications_rebinned: int = 0 + classification_members_rebinned: int = 0 + + # Calibrations whose ``controls_not_phi`` affirmation was cleared because a control was dropped. + calibrations_pending_phi_reaffirmation: list[int] = field(default_factory=list) + + def to_dict(self) -> dict[str, Any]: + """Return a JSON-serializable summary suitable for a job's logging context.""" + return { + "calibration_controls_relinked": self.controls_relinked, + "calibration_controls_dropped": self.controls_dropped, + "calibration_classification_members_relinked": self.classification_members_relinked, + "calibration_classification_members_dropped": self.classification_members_dropped, + "calibration_classifications_rebinned": self.classifications_rebinned, + "calibration_classification_members_rebinned": self.classification_members_rebinned, + "calibrations_pending_phi_reaffirmation": list(self.calibrations_pending_phi_reaffirmation), + } diff --git a/src/mavedb/worker/jobs/variant_processing/creation.py b/src/mavedb/worker/jobs/variant_processing/creation.py index a519bda07..480705ecb 100644 --- a/src/mavedb/worker/jobs/variant_processing/creation.py +++ b/src/mavedb/worker/jobs/variant_processing/creation.py @@ -13,14 +13,23 @@ from mavedb.data_providers.services import CSV_UPLOAD_S3_BUCKET_NAME, RESTDataProvider, s3_client from mavedb.lib.logging.context import format_raised_exception_info_as_dict +from mavedb.lib.score_calibrations import ( + restore_calibration_variant_links, + snapshot_calibration_variant_links, +) from mavedb.lib.score_sets import columns_for_dataset, create_variants, create_variants_data +from mavedb.lib.types.score_calibrations import CalibrationVariantLinkSnapshot from mavedb.lib.types.workflow import JobExecutionOutcome from mavedb.lib.validation.dataframe.dataframe import validate_and_standardize_dataframe_pair from mavedb.lib.validation.exceptions import ValidationError +from mavedb.models.calibration_control import CalibrationControl from mavedb.models.enums.job_pipeline import FailureCategory from mavedb.models.enums.mapping_state import MappingState from mavedb.models.enums.processing_state import ProcessingState from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.score_calibration_functional_classification_variant_association import ( + score_calibration_functional_classification_variants_association_table, +) from mavedb.models.score_set import ScoreSet from mavedb.models.user import User from mavedb.models.variant import Variant @@ -180,10 +189,30 @@ async def create_variants_for_score_set(ctx: dict, job_id: int, job_manager: Job } # Delete variants after validation occurs so we don't overwrite them in the case of a bad update. + calibration_link_snapshot: list[CalibrationVariantLinkSnapshot] = [] + relink_calibrations = False if score_set.variants: existing_variants = job_manager.db.scalars( select(Variant.id).where(Variant.score_set_id == score_set.id) ).all() + + # Calibration controls and functional classification bin membership reference variants + # without an ON DELETE action, so the variant delete below fails outright while those + # links stand. Record what the new upload cannot regenerate, clear the links, and + # re-establish them once the new variants exist. + relink_calibrations = bool(score_set.score_calibrations) + calibration_link_snapshot = snapshot_calibration_variant_links(job_manager.db, score_set) + job_manager.db.execute( + delete(CalibrationControl).where(CalibrationControl.variant_id.in_(existing_variants)) + ) + job_manager.db.execute( + delete(score_calibration_functional_classification_variants_association_table).where( + score_calibration_functional_classification_variants_association_table.c.variant_id.in_( + existing_variants + ) + ) + ) + job_manager.db.execute(delete(MappedVariant).where(MappedVariant.variant_id.in_(existing_variants))) job_manager.db.execute(delete(Variant).where(Variant.id.in_(existing_variants))) @@ -198,6 +227,23 @@ async def create_variants_for_score_set(ctx: dict, job_id: int, job_manager: Job variants_data = create_variants_data(validated_scores, validated_counts, None) create_variants(job_manager.db, score_set, variants_data) + # Scoped to the replacement path above: only links the delete broke need re-establishing, so a + # first upload leaves calibrations alone. An empty snapshot is not the same as nothing to do, + # though, since range-based bins are recomputed from the new scores rather than remembered. + if relink_calibrations: + # The new variants need ids and scores on record before anything can point at them. + job_manager.db.flush() + + relink_report = restore_calibration_variant_links( + job_manager.db, score_set, calibration_link_snapshot, updated_by + ) + job_manager.save_to_context(relink_report.to_dict()) + + logger.info( + msg="Re-established calibration variant references against the newly created variants.", + extra=job_manager.logging_context(), + ) + except ValidationError as e: job_manager.db.rollback() diff --git a/tests/worker/jobs/variant_processing/test_creation.py b/tests/worker/jobs/variant_processing/test_creation.py index d0d37562a..4c937eb6e 100644 --- a/tests/worker/jobs/variant_processing/test_creation.py +++ b/tests/worker/jobs/variant_processing/test_creation.py @@ -5,13 +5,19 @@ pytest.importorskip("arq") import math +from datetime import date, timedelta from unittest.mock import ANY, MagicMock, call, patch +from mavedb.lib.mondo import get_generic_disease_term +from mavedb.models.calibration_control import CalibrationControl +from mavedb.models.enums.calibration_control_status import CalibrationControlStatus from mavedb.models.enums.job_pipeline import JobStatus, PipelineStatus from mavedb.models.enums.mapping_state import MappingState from mavedb.models.enums.processing_state import ProcessingState from mavedb.models.job_run import JobRun from mavedb.models.pipeline import Pipeline +from mavedb.models.score_calibration import ScoreCalibration +from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification from mavedb.models.variant import Variant from mavedb.worker.jobs.variant_processing.creation import create_variants_for_score_set from mavedb.worker.lib.managers.job_manager import JobManager @@ -1188,3 +1194,457 @@ async def test_create_variants_for_score_set_with_arq_context_generic_exception_ ) for run in other_runs: assert run.status == JobStatus.SKIPPED + + +def _make_calibration(session, score_set, user, controls_not_phi=None): + """Persist a bare calibration on a score set; controls and bin membership need a parent.""" + calibration = ScoreCalibration( + title="Relink test calibration", + score_set_id=score_set.id, + disease_term=get_generic_disease_term(session), + controls_not_phi=controls_not_phi, + created_by=user, + modified_by=user, + ) + session.add(calibration) + session.commit() + session.refresh(calibration) + return calibration + + +def _variants_by_hgvs_nt(session, score_set): + return {variant.hgvs_nt: variant for variant in session.query(Variant).filter_by(score_set_id=score_set.id).all()} + + +def _without_variant(scores_df, counts_df, hgvs_nt): + """Return the score/count pair with one variant removed, simulating a corrected re-upload.""" + return ( + scores_df[scores_df["hgvs_nt"] != hgvs_nt].copy(), + counts_df[counts_df["hgvs_nt"] != hgvs_nt].copy(), + ) + + +async def _run_creation_job(mock_worker_ctx, mock_s3_client, job_run, scores_df, counts_df): + with ( + patch.object(mock_s3_client, "download_fileobj", return_value=None), + patch( + "mavedb.worker.jobs.variant_processing.creation.pd.read_csv", + side_effect=[scores_df, counts_df], + ), + ): + await create_variants_for_score_set(mock_worker_ctx, job_run.id) + + +@pytest.mark.integration +@pytest.mark.asyncio +class TestCreateVariantsForScoreSetCalibrationRelinking: + """Re-upload behavior for score sets whose calibrations reference variants. + + Calibration controls and functional classification bin membership hold plain foreign keys into + ``variants``, which a re-upload deletes and recreates wholesale. These tests pin both halves of + the fix: the delete no longer trips those constraints, and surviving references are re-resolved + against the new rows by HGVS identity rather than by the positional variant URN. + """ + + async def test_reupload_with_calibration_controls_does_not_violate_foreign_key( + self, + session, + with_independent_processing_runs, + with_populated_domain_data, + mock_worker_ctx, + mock_s3_client, + sample_score_dataframe, + sample_count_dataframe, + sample_score_set, + sample_user, + sample_independent_variant_creation_run, + ): + await _run_creation_job( + mock_worker_ctx, + mock_s3_client, + sample_independent_variant_creation_run, + sample_score_dataframe, + sample_count_dataframe, + ) + + calibration = _make_calibration(session, sample_score_set, sample_user) + session.add( + CalibrationControl( + calibration_id=calibration.id, + variant_id=_variants_by_hgvs_nt(session, sample_score_set)["c.1A>T"].id, + clinical_status=CalibrationControlStatus.pathogenic, + created_by=sample_user, + modified_by=sample_user, + ) + ) + sample_independent_variant_creation_run.status = JobStatus.PENDING + session.commit() + + # Regression: this second upload raised ForeignKeyViolation on calibration_controls_variant_id_fkey. + await _run_creation_job( + mock_worker_ctx, + mock_s3_client, + sample_independent_variant_creation_run, + sample_score_dataframe, + sample_count_dataframe, + ) + + session.refresh(sample_score_set) + assert sample_score_set.processing_state == ProcessingState.success + assert session.query(CalibrationControl).filter_by(calibration_id=calibration.id).count() == 1 + + async def test_controls_relink_to_new_variants_when_every_identity_persists( + self, + session, + with_independent_processing_runs, + with_populated_domain_data, + mock_worker_ctx, + mock_s3_client, + sample_score_dataframe, + sample_count_dataframe, + sample_score_set, + sample_user, + sample_extra_user, + sample_independent_variant_creation_run, + ): + await _run_creation_job( + mock_worker_ctx, + mock_s3_client, + sample_independent_variant_creation_run, + sample_score_dataframe, + sample_count_dataframe, + ) + + original_variant_ids = { + hgvs: variant.id for hgvs, variant in _variants_by_hgvs_nt(session, sample_score_set).items() + } + calibration = _make_calibration(session, sample_score_set, sample_user, controls_not_phi=True) + + # An older submission date proves the relinked row keeps its own provenance rather than + # inheriting the re-upload's. + submitted_on = date.today() - timedelta(days=30) + for hgvs_nt, status in ( + ("c.1A>T", CalibrationControlStatus.pathogenic), + ("c.2T>A", CalibrationControlStatus.benign), + ): + session.add( + CalibrationControl( + calibration_id=calibration.id, + variant_id=original_variant_ids[hgvs_nt], + clinical_status=status, + created_by=sample_extra_user, + modified_by=sample_extra_user, + creation_date=submitted_on, + modification_date=submitted_on, + ) + ) + + updated_scores = sample_score_dataframe.copy() + updated_scores["score"] += 10 + + sample_independent_variant_creation_run.status = JobStatus.PENDING + session.commit() + + await _run_creation_job( + mock_worker_ctx, + mock_s3_client, + sample_independent_variant_creation_run, + updated_scores, + sample_count_dataframe.copy(), + ) + + new_variants = _variants_by_hgvs_nt(session, sample_score_set) + assert set(new_variants) == set(original_variant_ids) + # Every variant row really was replaced, so a stale link would be visible below. + assert all(new_variants[hgvs].id != variant_id for hgvs, variant_id in original_variant_ids.items()) + + controls = session.query(CalibrationControl).filter_by(calibration_id=calibration.id).all() + assert len(controls) == 2 + assert {control.variant_id for control in controls} == { + new_variants["c.1A>T"].id, + new_variants["c.2T>A"].id, + } + assert {control.clinical_status for control in controls} == { + CalibrationControlStatus.pathogenic, + CalibrationControlStatus.benign, + } + assert all(control.created_by_id == sample_extra_user.id for control in controls) + assert all(control.creation_date == submitted_on for control in controls) + assert all(control.modified_by_id == sample_user.id for control in controls) + + # Nothing was dropped, so the submitter's PHI affirmation still describes this control set. + session.refresh(calibration) + assert calibration.controls_not_phi is True + + async def test_controls_absent_from_new_upload_are_dropped_and_reset_phi_affirmation( + self, + session, + with_independent_processing_runs, + with_populated_domain_data, + mock_worker_ctx, + mock_s3_client, + sample_score_dataframe, + sample_count_dataframe, + sample_score_set, + sample_user, + sample_independent_variant_creation_run, + ): + await _run_creation_job( + mock_worker_ctx, + mock_s3_client, + sample_independent_variant_creation_run, + sample_score_dataframe, + sample_count_dataframe, + ) + + original_variants = _variants_by_hgvs_nt(session, sample_score_set) + calibration = _make_calibration(session, sample_score_set, sample_user, controls_not_phi=True) + for hgvs_nt in ("c.1A>T", "c.4C>G"): + session.add( + CalibrationControl( + calibration_id=calibration.id, + variant_id=original_variants[hgvs_nt].id, + clinical_status=CalibrationControlStatus.pathogenic, + created_by=sample_user, + modified_by=sample_user, + ) + ) + + sample_independent_variant_creation_run.status = JobStatus.PENDING + session.commit() + + trimmed_scores, trimmed_counts = _without_variant(sample_score_dataframe, sample_count_dataframe, "c.4C>G") + + recorded_context: dict = {} + original_save_to_context = JobManager.save_to_context + + def _capture_context(self, ctx): + recorded_context.update(ctx) + return original_save_to_context(self, ctx) + + with patch.object(JobManager, "save_to_context", _capture_context): + await _run_creation_job( + mock_worker_ctx, + mock_s3_client, + sample_independent_variant_creation_run, + trimmed_scores, + trimmed_counts, + ) + + new_variants = _variants_by_hgvs_nt(session, sample_score_set) + assert "c.4C>G" not in new_variants + + controls = session.query(CalibrationControl).filter_by(calibration_id=calibration.id).all() + assert len(controls) == 1 + assert controls[0].variant_id == new_variants["c.1A>T"].id + + # A control set that lost a member is no longer the one the submitter affirmed as PHI-free. + session.refresh(calibration) + assert calibration.controls_not_phi is None + + assert recorded_context["calibration_controls_relinked"] == 1 + assert recorded_context["calibration_controls_dropped"] == 1 + assert recorded_context["calibrations_pending_phi_reaffirmation"] == [calibration.id] + + async def test_range_based_classification_membership_is_rebinned_against_new_scores( + self, + session, + with_independent_processing_runs, + with_populated_domain_data, + mock_worker_ctx, + mock_s3_client, + sample_score_dataframe, + sample_count_dataframe, + sample_score_set, + sample_user, + sample_independent_variant_creation_run, + ): + await _run_creation_job( + mock_worker_ctx, + mock_s3_client, + sample_independent_variant_creation_run, + sample_score_dataframe, + sample_count_dataframe, + ) + + # Sample scores: c.1A>T 0.3, c.2T>A 0.0, c.3G>C -1.65, c.4C>G unscored. + original_variants = _variants_by_hgvs_nt(session, sample_score_set) + calibration = _make_calibration(session, sample_score_set, sample_user) + normal = ScoreCalibrationFunctionalClassification( + calibration_id=calibration.id, + label="Normal function", + range=[0.0, None], + inclusive_lower_bound=True, + ) + normal.variants = [original_variants["c.1A>T"], original_variants["c.2T>A"]] + abnormal = ScoreCalibrationFunctionalClassification( + calibration_id=calibration.id, + label="Abnormal function", + range=[None, 0.0], + inclusive_upper_bound=False, + ) + abnormal.variants = [original_variants["c.3G>C"]] + session.add_all([normal, abnormal]) + + original_variant_ids = {variant.id for variant in original_variants.values()} + + sample_independent_variant_creation_run.status = JobStatus.PENDING + session.commit() + normal_id, abnormal_id = normal.id, abnormal.id + + # Shifting every score up by two moves c.3G>C across the threshold, so recorded membership and + # recomputed membership disagree and the test can tell which one was applied. + updated_scores = sample_score_dataframe.copy() + updated_scores["score"] += 2 + + recorded_context: dict = {} + original_save_to_context = JobManager.save_to_context + + def _capture_context(self, ctx): + recorded_context.update(ctx) + return original_save_to_context(self, ctx) + + with patch.object(JobManager, "save_to_context", _capture_context): + await _run_creation_job( + mock_worker_ctx, + mock_s3_client, + sample_independent_variant_creation_run, + updated_scores, + sample_count_dataframe.copy(), + ) + + session.expire_all() + normal = session.get(ScoreCalibrationFunctionalClassification, normal_id) + abnormal = session.get(ScoreCalibrationFunctionalClassification, abnormal_id) + + # Membership follows the new scores rather than what was recorded: c.3G>C (-1.65 -> 0.35) moves + # into the normal bin, emptying the abnormal one. c.4C>G stays out of both, having no score. + assert {variant.hgvs_nt for variant in normal.variants} == {"c.1A>T", "c.2T>A", "c.3G>C"} + assert normal.variant_count == 3 + assert abnormal.variants == [] + assert abnormal.variant_count == 0 + + # Re-binned membership points at the new variant rows, not the deleted ones. + new_variants = _variants_by_hgvs_nt(session, sample_score_set) + assert {variant.id for variant in normal.variants} <= {variant.id for variant in new_variants.values()} + assert {variant.id for variant in normal.variants}.isdisjoint(original_variant_ids) + + assert recorded_context["calibration_classifications_rebinned"] == 2 + assert recorded_context["calibration_classification_members_rebinned"] == 3 + # Range bins are recomputed, never relinked, so nothing is reported as carried across. + assert recorded_context["calibration_classification_members_relinked"] == 0 + + async def test_class_based_classification_membership_relinks_by_hgvs_identity( + self, + session, + with_independent_processing_runs, + with_populated_domain_data, + mock_worker_ctx, + mock_s3_client, + sample_score_dataframe, + sample_count_dataframe, + sample_score_set, + sample_user, + sample_independent_variant_creation_run, + ): + await _run_creation_job( + mock_worker_ctx, + mock_s3_client, + sample_independent_variant_creation_run, + sample_score_dataframe, + sample_count_dataframe, + ) + + original_variants = _variants_by_hgvs_nt(session, sample_score_set) + calibration = _make_calibration(session, sample_score_set, sample_user) + + # A class-based bin comes from an uploaded classes file that a score upload does not carry, so + # its membership can only survive by identity. + classification = ScoreCalibrationFunctionalClassification( + calibration_id=calibration.id, + label="Loss of function", + class_="loss_of_function", + ) + classification.variants = [original_variants["c.1A>T"], original_variants["c.4C>G"]] + session.add(classification) + + sample_independent_variant_creation_run.status = JobStatus.PENDING + session.commit() + classification_id = classification.id + + trimmed_scores, trimmed_counts = _without_variant(sample_score_dataframe, sample_count_dataframe, "c.4C>G") + + recorded_context: dict = {} + original_save_to_context = JobManager.save_to_context + + def _capture_context(self, ctx): + recorded_context.update(ctx) + return original_save_to_context(self, ctx) + + with patch.object(JobManager, "save_to_context", _capture_context): + await _run_creation_job( + mock_worker_ctx, + mock_s3_client, + sample_independent_variant_creation_run, + trimmed_scores, + trimmed_counts, + ) + + new_variants = _variants_by_hgvs_nt(session, sample_score_set) + assert "c.4C>G" not in new_variants + + session.expire_all() + classification = session.get(ScoreCalibrationFunctionalClassification, classification_id) + + # The surviving member is relinked to its new row; the one absent from the upload is dropped. + assert [variant.id for variant in classification.variants] == [new_variants["c.1A>T"].id] + assert classification.variant_count == 1 + + assert recorded_context["calibration_classification_members_relinked"] == 1 + assert recorded_context["calibration_classification_members_dropped"] == 1 + assert recorded_context["calibration_classifications_rebinned"] == 0 + + async def test_reupload_without_calibrations_is_unaffected( + self, + session, + with_independent_processing_runs, + with_populated_domain_data, + mock_worker_ctx, + mock_s3_client, + sample_score_dataframe, + sample_count_dataframe, + sample_score_set, + sample_independent_variant_creation_run, + ): + await _run_creation_job( + mock_worker_ctx, + mock_s3_client, + sample_independent_variant_creation_run, + sample_score_dataframe, + sample_count_dataframe, + ) + + sample_independent_variant_creation_run.status = JobStatus.PENDING + session.commit() + + recorded_context: dict = {} + original_save_to_context = JobManager.save_to_context + + def _capture_context(self, ctx): + recorded_context.update(ctx) + return original_save_to_context(self, ctx) + + with patch.object(JobManager, "save_to_context", _capture_context): + await _run_creation_job( + mock_worker_ctx, + mock_s3_client, + sample_independent_variant_creation_run, + sample_score_dataframe, + sample_count_dataframe, + ) + + session.refresh(sample_score_set) + assert sample_score_set.processing_state == ProcessingState.success + assert sample_score_set.num_variants == sample_score_dataframe.shape[0] + # The relink path stays out of the way entirely when there is nothing to relink. + assert "calibration_controls_relinked" not in recorded_context From 5a13ff199acc3bcbc02c185c1d1cafa93db00c0d Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Fri, 18 Sep 2026 16:46:09 -0700 Subject: [PATCH 162/252] fix(scripts): let untried score sets claim slots before retrying failures plan_enqueue walked run_score_set_pipelines' cohort in a single fixed pass (gene cluster, then URN), and a FAILED score set looked identical to a never-run one. A handful of chronically-failing score sets early in that order would therefore win the same concurrency slots on every re-invocation forever, starving score sets later in the order that had never been attempted, and stalling the whole cohort behind a known error. Untried entries now always fill available slots first, in cohort order; only leftover slots go to entries whose latest pipeline failed, also in cohort order so cache-coherent per-gene fill still holds within each pass. latest_pipeline_by_score_set computes each score set's latest pipeline once and is shared between planning and the report table. --- src/mavedb/scripts/run_score_set_pipelines.py | 84 ++++++++++++----- tests/scripts/test_run_score_set_pipelines.py | 94 +++++++++++++++++++ 2 files changed, 157 insertions(+), 21 deletions(-) diff --git a/src/mavedb/scripts/run_score_set_pipelines.py b/src/mavedb/scripts/run_score_set_pipelines.py index 01770253e..221ee675f 100644 --- a/src/mavedb/scripts/run_score_set_pipelines.py +++ b/src/mavedb/scripts/run_score_set_pipelines.py @@ -15,6 +15,12 @@ it (by hand, cron, or /loop) to keep driving progress; the heavy pipeline work happens entirely in the worker, with the Pipeline/JobRun tables as durable state. +Cohort order (gene cluster, then URN) is fixed across invocations, so a score set whose +latest run failed keeps its same spot on every re-run. Slots therefore always go to +untried score sets first; a failed one is only retried with whatever slots are left +over, so a handful of chronic failures can't camp on the concurrency window and starve +score sets that have never been attempted (see plan_enqueue). + Usage: # Preview what would be enqueued, without enqueuing anything. poetry run python -m mavedb.scripts.run_score_set_pipelines map_annotate_score_set \\ @@ -251,6 +257,7 @@ def plan_enqueue( *, in_flight_score_set_ids: set[int], current_score_set_ids: set[int], + failed_score_set_ids: frozenset[int] = frozenset(), slots: int, limit: Optional[int], ) -> list[tuple[ScoreSet, str, EnqueueDecision]]: @@ -259,29 +266,46 @@ def plan_enqueue( Slots are spent in cohort order, which cluster_cohort has already grouped by gene: a window therefore stays on one gene until that gene is exhausted, rather than splitting its concurrency across two genes where half the running pipelines warm a - ClinGen cache the other half never reads. + ClinGen cache the other half never reads. That ordering is fixed across invocations, + so a score set whose latest run failed sits at the same spot on every re-run. Filling + slots in a single pass over that order would let a handful of chronically-failing + score sets early in cluster/URN order camp on the window forever, since a FAILED + entry looks exactly like a never-run one to a single scan and keeps winning the same + slots run after run — starving entries later in the order that have never been + attempted. Untried entries are therefore filled first, in cohort order; only leftover + slots go to failed entries, also in cohort order (preserving cache-coherent + within-cluster fill for the retry pass too). Skip-current/skip-in-flight decisions never consume a slot; only "enqueue" does. """ - plan: list[tuple[ScoreSet, str, EnqueueDecision]] = [] + decisions: dict[Optional[int], EnqueueDecision] = {} remaining_slots = slots enqueued_count = 0 + def fill(entries: list[CohortEntry]) -> None: + nonlocal remaining_slots, enqueued_count + for entry in entries: + if remaining_slots <= 0 or (limit is not None and enqueued_count >= limit): + decisions[entry.score_set.id] = "skip_cap" + else: + decisions[entry.score_set.id] = "enqueue" + remaining_slots -= 1 + enqueued_count += 1 + + eligible: list[CohortEntry] = [] for entry in ordered_cohort: - score_set = entry.score_set - - if score_set.id in current_score_set_ids: - plan.append((score_set, entry.cluster_key, "skip_current")) - elif score_set.id in in_flight_score_set_ids: - plan.append((score_set, entry.cluster_key, "skip_in_flight")) - elif remaining_slots <= 0 or (limit is not None and enqueued_count >= limit): - plan.append((score_set, entry.cluster_key, "skip_cap")) + score_set_id = entry.score_set.id + if score_set_id in current_score_set_ids: + decisions[score_set_id] = "skip_current" + elif score_set_id in in_flight_score_set_ids: + decisions[score_set_id] = "skip_in_flight" else: - plan.append((score_set, entry.cluster_key, "enqueue")) - remaining_slots -= 1 - enqueued_count += 1 + eligible.append(entry) - return plan + fill([entry for entry in eligible if entry.score_set.id not in failed_score_set_ids]) + fill([entry for entry in eligible if entry.score_set.id in failed_score_set_ids]) + + return [(entry.score_set, entry.cluster_key, decisions[entry.score_set.id]) for entry in ordered_cohort] # --------------------------------------------------------------------------- @@ -383,6 +407,17 @@ def pipelines_by_score_set( return result +def latest_pipeline_by_score_set(db: Session, *, tracked_name: str, score_set_ids: list[int]) -> dict[int, Pipeline]: + """The most recently created tracked_name pipeline for each score set that has one. + + Shared by enqueue planning (was this score set's last attempt a failure?) and + reporting (what's its current status?), so a large cohort pays for this join once + per invocation instead of once per use. + """ + by_score_set = pipelines_by_score_set(db, tracked_name=tracked_name, score_set_ids=score_set_ids) + return {ss_id: max(pipelines, key=lambda p: p.created_at) for ss_id, pipelines in by_score_set.items()} + + def representative_error(db: Session, pipeline_id: int) -> Optional[str]: """Latest JobRun.error_message for a FAILED job under this pipeline.""" job_run = db.scalars( @@ -563,6 +598,7 @@ def render_report( *, tracked_name: str, ordered_cohort: Sequence[CohortEntry], + latest_by_score_set: dict[int, Pipeline], in_flight_rows: list[tuple[Pipeline, Optional[int]]], current_since: Optional[datetime.date], ) -> tuple[str, list[str]]: @@ -573,20 +609,16 @@ def render_report( lines: list[str] = [] failed_urns: list[str] = [] - score_set_ids: list[int] = [entry.score_set.id for entry in ordered_cohort] # type: ignore[misc] - latest_by_score_set = pipelines_by_score_set(db, tracked_name=tracked_name, score_set_ids=score_set_ids) - lines.append(f"Cohort report for '{tracked_name}' ({len(ordered_cohort)} score sets):") lines.append(f"{'URN':<40} {'CLUSTER':<20} {'STATUS':<12} ERROR") for entry in ordered_cohort: score_set = entry.score_set cluster = entry.cluster_key or "-" - pipelines = latest_by_score_set.get(entry.score_set.id, []) # type: ignore[arg-type] - if not pipelines: + latest = latest_by_score_set.get(entry.score_set.id) # type: ignore[arg-type] + if latest is None: lines.append(f"{score_set.urn:<40} {cluster:<20} {'no run':<12}") continue - latest = max(pipelines, key=lambda p: p.created_at) error = "" if is_failure(latest.status): failed_urns.append(score_set.urn) # type: ignore[arg-type] @@ -796,19 +828,27 @@ async def main( run_pipeline_name = None current_since_date = current_since.date() if current_since else None + cohort_score_set_ids: list[int] = [entry.score_set.id for entry in ordered_cohort] # type: ignore[misc] current_score_set_ids: set[int] = set() if current_since_date is not None: succeeded = pipelines_by_score_set( db, tracked_name=effective_name, - score_set_ids=[entry.score_set.id for entry in ordered_cohort], # type: ignore[misc] + score_set_ids=cohort_score_set_ids, statuses=[PipelineStatus.SUCCEEDED], ) for ss_id, pipelines in succeeded.items(): if any(is_current(p.status, p.finished_at, current_since_date) for p in pipelines): current_score_set_ids.add(ss_id) + latest_by_score_set = latest_pipeline_by_score_set( + db, tracked_name=effective_name, score_set_ids=cohort_score_set_ids + ) + failed_score_set_ids = frozenset( + ss_id for ss_id, pipeline in latest_by_score_set.items() if is_failure(pipeline.status) + ) + in_flight_rows = in_flight_pipelines(db, tracked_name=effective_name) in_flight_score_set_ids = {ss_id for _p, ss_id in in_flight_rows if ss_id is not None} @@ -817,6 +857,7 @@ async def main( ordered_cohort, in_flight_score_set_ids=in_flight_score_set_ids, current_score_set_ids=current_score_set_ids, + failed_score_set_ids=failed_score_set_ids, slots=slots, limit=limit, ) @@ -862,6 +903,7 @@ async def main( db, tracked_name=effective_name, ordered_cohort=ordered_cohort, + latest_by_score_set=latest_by_score_set, in_flight_rows=in_flight_rows, current_since=current_since_date, ) diff --git a/tests/scripts/test_run_score_set_pipelines.py b/tests/scripts/test_run_score_set_pipelines.py index 49498df11..b5251a7d4 100644 --- a/tests/scripts/test_run_score_set_pipelines.py +++ b/tests/scripts/test_run_score_set_pipelines.py @@ -22,6 +22,7 @@ in_flight_pipelines, is_current, is_failure, + latest_pipeline_by_score_set, pipelines_by_score_set, plan_enqueue, render_cluster_table, @@ -287,6 +288,61 @@ def test_limit_caps_below_slots(self): ) assert sorted(self._decisions(plan).values()) == ["enqueue", "skip_cap", "skip_cap"] + def test_untried_entry_wins_a_slot_over_an_earlier_failed_entry(self): + """Cohort order (id 1 before id 2) must not decide this: id 1 previously failed, + id 2 has never run, and with only one slot the untried one has to get it — + otherwise a chronic failure at the front of the order retries forever while an + untried entry behind it never gets a turn.""" + plan = plan_enqueue( + self._cohort(2), + in_flight_score_set_ids=set(), + current_score_set_ids=set(), + failed_score_set_ids=frozenset({1}), + slots=1, + limit=None, + ) + decisions = self._decisions(plan) + assert decisions[1] == "skip_cap" + assert decisions[2] == "enqueue" + + def test_failed_entry_gets_leftover_slots_after_untried_are_placed(self): + plan = plan_enqueue( + self._cohort(2), + in_flight_score_set_ids=set(), + current_score_set_ids=set(), + failed_score_set_ids=frozenset({1}), + slots=2, + limit=None, + ) + assert set(self._decisions(plan).values()) == {"enqueue"} + + def test_limit_also_applies_across_the_two_priority_passes(self): + plan = plan_enqueue( + self._cohort(2), + in_flight_score_set_ids=set(), + current_score_set_ids=set(), + failed_score_set_ids=frozenset({1}), + slots=2, + limit=1, + ) + decisions = self._decisions(plan) + assert decisions[2] == "enqueue" + assert decisions[1] == "skip_cap" + + def test_current_and_in_flight_still_take_precedence_over_failed_status(self): + cohort = self._cohort(2) + plan = plan_enqueue( + cohort, + in_flight_score_set_ids={1}, + current_score_set_ids={2}, + failed_score_set_ids=frozenset({1, 2}), + slots=5, + limit=None, + ) + decisions = self._decisions(plan) + assert decisions[1] == "skip_in_flight" + assert decisions[2] == "skip_current" + def test_slots_fill_one_cluster_before_moving_to_the_next(self): """Two slots against two clusters must both land on the same gene, otherwise each running pipeline warms a ClinGen cache the other never reads.""" @@ -295,6 +351,21 @@ def test_slots_fill_one_cluster_before_moving_to_the_next(self): enqueued = {key for _ss, key, decision in plan if decision == "enqueue"} assert len(enqueued) == 1 + def test_failed_retry_pass_still_fills_one_cluster_before_the_next(self): + """The retry pass is a subsequence of the same cluster-ordered cohort, so it + inherits cache-coherent fill without any special-casing.""" + cohort = self._cohort(4, {1: "BRCA1", 2: "BRCA1", 3: "TP53", 4: "TP53"}) + plan = plan_enqueue( + cohort, + in_flight_score_set_ids=set(), + current_score_set_ids=set(), + failed_score_set_ids=frozenset({1, 2, 3, 4}), + slots=2, + limit=None, + ) + enqueued = {key for _ss, key, decision in plan if decision == "enqueue"} + assert len(enqueued) == 1 + def test_unknown_cluster_yields_to_real_clusters(self): cohort = self._cohort(3, {3: "TP53"}) plan = plan_enqueue(cohort, in_flight_score_set_ids=set(), current_score_set_ids=set(), slots=1, limit=None) @@ -506,6 +577,29 @@ def test_phase_and_base_pipeline_tracked_independently(self, session, make_score assert score_set.id not in succeeded +@pytest.mark.integration +class TestLatestPipelineByScoreSet: + def test_score_set_with_no_pipeline_is_absent(self, session, make_score_set): + score_set = make_score_set() + + result = latest_pipeline_by_score_set(session, tracked_name="test_pipeline", score_set_ids=[score_set.id]) + assert score_set.id not in result + + def test_most_recently_created_pipeline_wins(self, session, make_score_set): + score_set = make_score_set() + older = _make_pipeline( + session, status=PipelineStatus.FAILED, created_at=datetime(2024, 1, 1, tzinfo=timezone.utc) + ) + _make_job_run(session, pipeline_id=older.id, score_set_id=score_set.id) + newer = _make_pipeline( + session, status=PipelineStatus.SUCCEEDED, created_at=datetime(2024, 6, 1, tzinfo=timezone.utc) + ) + _make_job_run(session, pipeline_id=newer.id, score_set_id=score_set.id) + + result = latest_pipeline_by_score_set(session, tracked_name="test_pipeline", score_set_ids=[score_set.id]) + assert result[score_set.id].id == newer.id + + @pytest.mark.integration class TestResolveCohort: def test_published_only_filters_unpublished(self, session, make_score_set): From be449bb1ebfb6c1961b270c0f55d1f226808e33b Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Fri, 18 Sep 2026 16:56:17 -0700 Subject: [PATCH 163/252] build(deps): pin variant-annotation to git tag instead of local path The editable sibling path dependency only resolved when variant-annotation happened to be checked out next to this repo, so it couldn't be installed in CI or a fresh clone. Point it at the bbi-lab/variant-annotation v0.1.0-dev.1 tag instead, which resolves the same way everywhere and drops the need for the Dockerfile's --no-directory workaround. --- Dockerfile | 2 +- poetry.lock | 12 +++++++----- pyproject.toml | 8 ++++---- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index bc5a52c45..422e2bf83 100644 --- a/Dockerfile +++ b/Dockerfile @@ -80,7 +80,7 @@ RUN curl -sSL https://install.python-poetry.org | python3 - COPY poetry.lock pyproject.toml ./ # installs runtime dependencies to $VIRTUAL_ENV -RUN poetry install --no-root --extras server --no-directory +RUN poetry install --no-root --extras server COPY alembic /code/alembic COPY alembic.ini /code/alembic.ini COPY src /code/src diff --git a/poetry.lock b/poetry.lock index ffa4c5618..c7fe295d6 100644 --- a/poetry.lock +++ b/poetry.lock @@ -4872,14 +4872,14 @@ test = ["aiohttp (>=3.10.5)", "flake8 (>=6.1,<7.0)", "mypy (>=0.800)", "psutil", [[package]] name = "variant-annotation" -version = "0.1.0" +version = "0.1.0-dev.1" description = "Genetic variant mapping and annotation pipeline" optional = true python-versions = ">=3.11" groups = ["main"] markers = "extra == \"server\"" files = [] -develop = true +develop = false [package.dependencies] biopython = "*" @@ -4900,8 +4900,10 @@ publish = ["build", "twine"] tests = ["pytest"] [package.source] -type = "directory" -url = "../variant-annotation" +type = "git" +url = "https://github.com/bbi-lab/variant-annotation.git" +reference = "v0.1.0-dev.1" +resolved_reference = "199e1a0d78050aaea8798744352065190aeed617" [[package]] name = "variant-translation" @@ -5233,4 +5235,4 @@ server = ["aiocache", "alembic", "alembic-utils", "arq", "authlib", "biocommons" [metadata] lock-version = "2.1" python-versions = "^3.11" -content-hash = "186d92c69885b37db17b63bba4e2e56a87224bd52a6468bb68b4e40061bebb44" +content-hash = "5644e132fe7ac94a34e5ad512912b6dda1374ebe5f2ff802e2117acfdea9b5f2" diff --git a/pyproject.toml b/pyproject.toml index 4644b0ad2..6944cfc41 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,7 +64,7 @@ starlette = { version = "~0.49.0", optional = true } starlette-context = { version = "^0.3.6", optional = true } slack-sdk = { version = "~3.21.3", optional = true } uvicorn = { extras = ["standard"], version = "*", optional = true } -variant-annotation = { path = "../variant-annotation", develop = true, optional = true } +variant-annotation = { git = "https://github.com/bbi-lab/variant-annotation.git", tag = "v0.1.0-dev.1", optional = true } watchtower = { version = "~3.2.0", optional = true } asyncclick = "^8.3.0.7" filelock = "^3.29.0" @@ -106,9 +106,9 @@ plugins = [ ] mypy_path = "mypy_stubs" -# variant-annotation is installed as a PEP 660 editable sibling package whose import -# hook mypy cannot follow, and it ships no py.typed marker. Treat it as untyped rather -# than maintaining drift-prone local stubs for an actively-developed internal library. +# variant-annotation is a git-pinned dependency (not yet published to PyPI) that ships +# no py.typed marker. Treat it as untyped rather than maintaining drift-prone local +# stubs for an actively-developed internal library. [[tool.mypy.overrides]] module = "variant_annotation.*" ignore_missing_imports = true From ab892504a9d95d104a079a2d3275f9039a22b33e Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Mon, 21 Sep 2026 09:17:24 -0700 Subject: [PATCH 164/252] fix(refget): close the review's inverted-range and bounds-checking gaps Address the code-review feedback on this PR: - Reject a Range header whose last-byte-pos is exactly one less than its first-byte-pos (e.g. "bytes=5-4"). Converting it to a half-open bound collapsed it to start == end, letting it slip past every downstream check and return a 206 with an inverted, spec-invalid Content-Range. - Per the refget spec, only a start beyond the sequence length must be rejected; there's no rule requiring a lone over-long end to be rejected too, so it's now clamped to the sequence length instead (previously both were rejected with 416, a spec deviation this PR introduced when it started validating the one-sided case at all). - Align seqrepo.py's newly-added bounds validation with refget.py's: same boundary math (a start equal to the sequence length -- an empty tail slice -- is legal in both), same one-sided/two-sided split. The 422 status family is unchanged since seqrepo.py predates and isn't bound by the refget spec. Closes the malformed-range hole and the seqrepo.py/refget.py boundary disagreement flagged in review; the over-length-range clamping decision was confirmed against the refget spec's explicit start/end rules. --- src/mavedb/routers/refget.py | 21 +++++++++++++++++++-- src/mavedb/routers/seqrepo.py | 20 ++++++++++++++------ tests/routers/test_refget.py | 27 +++++++++++++++++++-------- tests/routers/test_seqrepo.py | 8 +++++++- 4 files changed, 59 insertions(+), 17 deletions(-) diff --git a/src/mavedb/routers/refget.py b/src/mavedb/routers/refget.py index 70e18e2bf..9e406c97a 100644 --- a/src/mavedb/routers/refget.py +++ b/src/mavedb/routers/refget.py @@ -158,7 +158,12 @@ def get_sequence( logger.error(msg="Invalid range header format", extra=logging_context()) raise HTTPException(status_code=400, detail="Invalid range header format") + # Rejects "bytes=5-4"-style headers (RFC 9110 S14.1.1): after conversion to a half-open + # bound, start == end here would otherwise silently pass as a valid empty range. start, end = int(m.group(1)), int(m.group(2)) + 1 + if start >= end: + logger.error(msg="Invalid range header format", extra=logging_context()) + raise HTTPException(status_code=400, detail="Invalid range header format") save_to_logging_context({"requested_refget_start": start, "requested_refget_end": end}) if start is not None and end is not None: @@ -190,8 +195,9 @@ def get_sequence( seq_start = start if start is not None else 0 seq_end = end if end is not None else seq_len - if start is not None or end is not None: - if seq_start >= seq_len: + # Refget spec: only a start "larger than" the sequence length is invalid (strict inequality). + if start is not None and end is not None: + if seq_start > seq_len: raise HTTPException( status_code=416, detail="Invalid coordinates: start > sequence length", @@ -209,6 +215,17 @@ def get_sequence( detail="Invalid coordinates: must obey 0 <= start <= end <= sequence_length", headers={"Content-Range": f"bytes */{seq_len}"}, ) + # The refget spec only requires rejecting a start beyond the sequence length; there's no + # equivalent rule for a lone end (see below). + elif start is not None: + if not 0 <= seq_start <= seq_len: + raise HTTPException( + status_code=400, + detail=f"Invalid coordinates: start must satisfy 0 <= start <= {seq_len}", + ) + # Clamp the end coordinate to the valid range [0, seq_len]. + elif end is not None: + seq_end = max(0, min(seq_end, seq_len)) # Content-Length must match bytes actually streamed. Overstating it aborts the response mid-stream # once the ASGI server has already committed the status line. diff --git a/src/mavedb/routers/seqrepo.py b/src/mavedb/routers/seqrepo.py index 9cd271312..91d81c9a2 100644 --- a/src/mavedb/routers/seqrepo.py +++ b/src/mavedb/routers/seqrepo.py @@ -79,13 +79,21 @@ def get_sequence( seq_start = start if start is not None else 0 seq_end = end if end is not None else seq_len - if start is not None or end is not None: - if not 0 <= seq_start < seq_len: + # Mirrors refget.py's two-sided rule: start == seq_len (an empty tail slice) is legal. + if start is not None and end is not None: + if not 0 <= seq_start <= seq_end <= seq_len: + logger.error(msg="Invalid coordinates: range lies outside the sequence.", extra=logging_context()) + raise HTTPException( + status_code=422, detail=f"Invalid coordinates: must obey 0 <= start <= end <= {seq_len}" + ) + # Mirrors refget.py: a start beyond the sequence length is rejected. + elif start is not None: + if not 0 <= seq_start <= seq_len: logger.error(msg="Invalid coordinates: start lies outside the sequence.", extra=logging_context()) - raise HTTPException(status_code=422, detail=f"Invalid coordinates: must obey 0 <= start < {seq_len}") - if not seq_start <= seq_end <= seq_len: - logger.error(msg="Invalid coordinates: end lies outside the sequence.", extra=logging_context()) - raise HTTPException(status_code=422, detail=f"Invalid coordinates: must obey start <= end <= {seq_len}") + raise HTTPException(status_code=422, detail=f"Invalid coordinates: must obey 0 <= start <= {seq_len}") + # Mirrors refget.py: an over-long end is clamped rather than rejected. + elif end is not None: + seq_end = max(0, min(seq_end, seq_len)) return StreamingResponse(sequence_generator(sr, seq_id, seq_start, seq_end), media_type="text/plain") diff --git a/tests/routers/test_refget.py b/tests/routers/test_refget.py index 762f60be4..c6afd8623 100644 --- a/tests/routers/test_refget.py +++ b/tests/routers/test_refget.py @@ -128,23 +128,21 @@ def test_get_sequence_range_header_and_query_params_conflict(client): def test_get_sequence_invalid_query_range_only_start_negative(client): resp = client.get(f"/api/v1/refget/sequence/{VALID_ENSEMBL_IDENTIFIER}", params={"start": -1}) - assert resp.status_code == 416 + assert resp.status_code == 400 assert "Invalid coordinates" in resp.text - assert "Content-Range" in resp.headers def test_get_sequence_invalid_query_range_only_start_too_large(client): resp = client.get(f"/api/v1/refget/sequence/{VALID_ENSEMBL_IDENTIFIER}", params={"start": 7}) - assert resp.status_code == 416 + assert resp.status_code == 400 assert "Invalid coordinates" in resp.text - assert "Content-Range" in resp.headers -def test_get_sequence_invalid_query_range_only_end_too_large(client): +def test_get_sequence_only_end_too_large_clamps(client): + # Unlike start, an over-long lone end is clamped rather than rejected. resp = client.get(f"/api/v1/refget/sequence/{VALID_ENSEMBL_IDENTIFIER}", params={"end": 10}) - assert resp.status_code == 416 - assert "Invalid coordinates" in resp.text - assert "Content-Range" in resp.headers + assert resp.status_code == 200 + assert resp.text == "GGGG" @pytest.mark.parametrize( @@ -238,3 +236,16 @@ def test_get_sequence_range_header_invalid(client): resp = client.get(f"/api/v1/refget/sequence/{VALID_ENSEMBL_IDENTIFIER}", headers=headers) assert resp.status_code == 400 assert "Invalid range header format" in resp.text + + +def test_get_sequence_range_header_invalid_inverted_by_one(client): + resp = client.get(f"/api/v1/refget/sequence/{VALID_ENSEMBL_IDENTIFIER}", headers={"Range": "bytes=5-4"}) + assert resp.status_code == 400 + assert "Invalid range header format" in resp.text + + +def test_get_sequence_range_header_single_byte(client): + # Boundary case adjacent to the inverted range above: first-byte-pos == last-byte-pos is valid. + resp = client.get(f"/api/v1/refget/sequence/{VALID_ENSEMBL_IDENTIFIER}", headers={"Range": "bytes=1-1"}) + assert resp.status_code == 206 + assert resp.text == "G" diff --git a/tests/routers/test_seqrepo.py b/tests/routers/test_seqrepo.py index c9163c072..cfa8da063 100644 --- a/tests/routers/test_seqrepo.py +++ b/tests/routers/test_seqrepo.py @@ -81,7 +81,6 @@ def test_get_sequence_invalid_coords(client): "start=10&end=12", "start=1&end=12", "start=10", - "end=12", "start=-1&end=2", ], ) @@ -91,6 +90,13 @@ def test_get_sequence_coords_outside_sequence(client, query): assert "Invalid coordinates" in resp.text +def test_get_sequence_only_end_too_large_clamps(client): + # Mirrors refget.py: an over-long lone end is clamped rather than rejected. + resp = client.get(f"/api/v1/seqrepo/sequence/{VALID_ENSEMBL_IDENTIFIER}?end=12") + assert resp.status_code == 200 + assert resp.text == "GGGG" + + @pytest.mark.parametrize("entry", TEST_SEQREPO_INITIAL_STATE) def test_get_metadata_success(client, entry): alias = list(entry.keys())[0] From 4cf4287653c004a7e94d9ff4abce8073e008977d Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Mon, 21 Sep 2026 11:43:18 -0700 Subject: [PATCH 165/252] fix(score-sets): index functional classification variant fk to unblock score set deletion Deleting a score set cascades through SQLAlchemy to delete each of its variants, and Postgres checks every foreign key into variants.id to enforce referential integrity on each one. score_calibration_functional_classification_variants carries variant_id only as the trailing column of its composite primary key (functional_classification_id, variant_id), so that check has no index it can use and instead sequentially scans the whole table once per variant being deleted. The table is global, growing with every calibration ever created across MaveDB, so the cost compounds with both the deleted score set's variant count and the table's total size. This is the mechanism behind #677 ("unpublished score set deletion often fails"): deletion runs synchronously in the request, and for any score set with a nontrivial number of variants the per-variant sequential scan is slow enough to time out, non-deterministically depending on how large the association table has grown. Confirmed with EXPLAIN against a synthetic 200k-row table: the FK check plan flips from Seq Scan to Index Scan, and deleting 500 variants drops from ~2.6s to ~2.4ms. The migration builds the index CONCURRENTLY, outside a transaction, since the table may already be large enough in production that a blocking build would itself be disruptive. --- ...ndex_functional_classification_variant_.py | 38 +++++++++++++++++++ ...onal_classification_variant_association.py | 3 +- 2 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 alembic/versions/15c40c367731_index_functional_classification_variant_.py diff --git a/alembic/versions/15c40c367731_index_functional_classification_variant_.py b/alembic/versions/15c40c367731_index_functional_classification_variant_.py new file mode 100644 index 000000000..773a5ed9c --- /dev/null +++ b/alembic/versions/15c40c367731_index_functional_classification_variant_.py @@ -0,0 +1,38 @@ +"""index functional classification variant fk + +variant_id is only the trailing column of score_calibration_functional_classification_variants' +composite primary key, so Postgres has no index it can use to look up rows by variant_id alone. +Every Variant delete (including the per-variant cascade when a score set is deleted) forces a +sequential scan of this table to enforce that FK, which dominates the delete for any score set +with a nontrivial number of variants. See mavedb-api#677 ("unpublished score set deletion often +fails"). + +Built CONCURRENTLY, outside a transaction, so this doesn't hold an exclusive lock on the table for +the build's duration. + +Revision ID: 15c40c367731 +Revises: c4b18d0f7a92 +Create Date: 2026-09-21 11:19:39.396343 + +""" + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "15c40c367731" +down_revision = "c4b18d0f7a92" +branch_labels = None +depends_on = None + +INDEX_NAME = "ix_score_calib_func_classification_variants_variant_id" +TABLE_NAME = "score_calibration_functional_classification_variants" + + +def upgrade(): + with op.get_context().autocommit_block(): + op.execute(f"CREATE INDEX CONCURRENTLY IF NOT EXISTS {INDEX_NAME} ON {TABLE_NAME} (variant_id)") + + +def downgrade(): + with op.get_context().autocommit_block(): + op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {INDEX_NAME}") diff --git a/src/mavedb/models/score_calibration_functional_classification_variant_association.py b/src/mavedb/models/score_calibration_functional_classification_variant_association.py index 61f074bd2..e9475f798 100644 --- a/src/mavedb/models/score_calibration_functional_classification_variant_association.py +++ b/src/mavedb/models/score_calibration_functional_classification_variant_association.py @@ -1,6 +1,6 @@ """SQLAlchemy association table for variants belonging to functional classifications.""" -from sqlalchemy import Column, ForeignKey, Table +from sqlalchemy import Column, ForeignKey, Index, Table from mavedb.db.base import Base @@ -11,4 +11,5 @@ "functional_classification_id", ForeignKey("score_calibration_functional_classifications.id"), primary_key=True ), Column("variant_id", ForeignKey("variants.id"), primary_key=True), + Index("ix_score_calib_func_classification_variants_variant_id", "variant_id"), ) From b25ce83f215f32ad2a86613b874e81fb4984c88b Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Mon, 21 Sep 2026 10:43:46 -0700 Subject: [PATCH 166/252] fix(tests): stop server-extras imports from crashing core-dependencies collection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four test modules broke collection under the "Pytest on Core Dependencies" CI job (no fastapi/starlette/hgvs/variant-annotation installed): - tests/lib/annotation/conftest.py hard-imported VariantAnnotationContext, which now reaches mavedb.lib.cat_vrs -> logging.context -> fastapi. Every test module in the directory already requires psycopg2 (also a server extra) for its DB fixtures except test_flatten.py, which needs neither — so falling back to None on ImportError, matching the try/except already used for conftest_optional.py, costs nothing under core deps and keeps that one file collectible. - tests/lib/test_deprecation.py only exercises deprecation_headers (no logging.context dependency), but the module import drags fastapi in through record_deprecated_usage; needs its own importorskip("fastapi") since it isn't otherwise psycopg2-gated. - tests/lib/test_vep.py and tests/lib/test_vrs_utils.py were missing importorskip guards for variant_annotation/hgvs, the established pattern used elsewhere in the suite. tests/db/test_mixins.py had no guard at all and errored at fixture setup (not collection), which the interrupted run above was masking. --- tests/db/test_mixins.py | 4 ++++ tests/lib/annotation/conftest.py | 15 +++++++++++++-- tests/lib/test_deprecation.py | 4 ++++ tests/lib/test_vep.py | 7 +++++++ tests/lib/test_vrs_utils.py | 3 +++ 5 files changed, 31 insertions(+), 2 deletions(-) diff --git a/tests/db/test_mixins.py b/tests/db/test_mixins.py index 938d2062e..cd7ef373d 100644 --- a/tests/db/test_mixins.py +++ b/tests/db/test_mixins.py @@ -10,9 +10,13 @@ the gap bug relied on, so the explicit ``at`` is what actually proves the timestamp is threaded. """ +# ruff: noqa: E402 from datetime import datetime, timedelta, timezone import pytest + +pytest.importorskip("psycopg2") + from sqlalchemy import Column, ForeignKey, Index, Integer, select, text from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import aliased, declarative_base, relationship diff --git a/tests/lib/annotation/conftest.py b/tests/lib/annotation/conftest.py index 34fde44b7..bedc43af3 100644 --- a/tests/lib/annotation/conftest.py +++ b/tests/lib/annotation/conftest.py @@ -10,8 +10,6 @@ import pytest -from mavedb.lib.annotation.context import VariantAnnotationContext -from mavedb.lib.vrs import vrs_object_from_mapped_variant from tests.helpers.constants import PRIVATE_CALIBRATION_OWNER_ID from tests.helpers.mocks.factories import ( create_mock_mapped_variant, @@ -19,6 +17,19 @@ create_mock_mapped_variant_with_pathogenicity_calibration_score_set, ) +# VariantAnnotationContext (via mavedb.lib.cat_vrs) needs logging.context, which pulls in fastapi, an +# optional "server" extra. Every test module in this directory that uses the fixtures built from it +# already requires psycopg2 (also a "server" extra) for its DB session fixture, so this costs nothing +# under "core dependencies" CI beyond what those tests already needed — but without the fallback, a +# missing fastapi crashes collection for the whole directory, including files like test_flatten.py that +# need neither. +try: + from mavedb.lib.annotation.context import VariantAnnotationContext + from mavedb.lib.vrs import vrs_object_from_mapped_variant +except ImportError: + VariantAnnotationContext = None + vrs_object_from_mapped_variant = None + # Permission related helpers coupled to logging context. try: from .conftest_optional import * # noqa: F403 diff --git a/tests/lib/test_deprecation.py b/tests/lib/test_deprecation.py index 027e26404..c15246b67 100644 --- a/tests/lib/test_deprecation.py +++ b/tests/lib/test_deprecation.py @@ -3,6 +3,10 @@ import pytest +# mavedb.lib.deprecation imports logging.context for record_deprecated_usage, which pulls in +# fastapi (an optional "server" extra) even though the functions tested here don't need it. +pytest.importorskip("fastapi") + from mavedb.lib import deprecation from mavedb.lib.deprecation import deprecation_headers diff --git a/tests/lib/test_vep.py b/tests/lib/test_vep.py index 94251c0ed..98b51a0d3 100644 --- a/tests/lib/test_vep.py +++ b/tests/lib/test_vep.py @@ -7,10 +7,17 @@ and re-keys the result by input HGVS, the release lookup, and the DB linker (run against the DB). """ +# ruff: noqa: E402 + from datetime import date, timedelta from unittest.mock import MagicMock, patch import pytest + +pytest.importorskip("variant_annotation") +pytest.importorskip("psycopg2") +pytest.importorskip("fastapi") + from sqlalchemy import select from variant_annotation.lib.vep import ( diff --git a/tests/lib/test_vrs_utils.py b/tests/lib/test_vrs_utils.py index 07823c20e..931a00eb8 100644 --- a/tests/lib/test_vrs_utils.py +++ b/tests/lib/test_vrs_utils.py @@ -7,6 +7,9 @@ import pytest pytest.importorskip("ga4gh.vrs") +# mavedb.lib.vrs_utils imports ga4gh.vrs.extras.translator.AlleleTranslator, which needs hgvs (an +# optional "server" extra) even though ga4gh.vrs itself does not. +pytest.importorskip("hgvs") from ga4gh.core.models import iriReference from ga4gh.vrs.models import ( From 764375f697b261bb2f611934f77baa799387e33d Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Mon, 21 Sep 2026 10:43:56 -0700 Subject: [PATCH 167/252] fix(mypy): resolve the SQLAlchemy-plugin and Optional-narrowing errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit models/gnomad_variant.py and models/clinical_control.py mixed a Mapped[] annotation with a legacy Column() call for their Float and nullable-String attributes; the SQLAlchemy mypy plugin only infers the left/right-hand types correctly from mapped_column(), so switched those declarations to it (the other Column() fields in each class weren't affected and are left as-is). lib/gnomad.py and scripts/audit_allele_identifiers.py each read an Optional/wider-union value that a query or the callee's own contract already guarantees is narrower at that point (a WHERE clause excluding nulls; vrs_object_from_mapped_variant only ever producing an Allele or CisPhasedBlock despite its wider declared return type) — assert the invariant mypy can't see rather than loosening the callee's real contract. --- src/mavedb/lib/gnomad.py | 4 ++++ src/mavedb/models/clinical_control.py | 4 ++-- src/mavedb/models/gnomad_variant.py | 8 ++++---- src/mavedb/scripts/audit_allele_identifiers.py | 15 +++++++++++++-- 4 files changed, 23 insertions(+), 8 deletions(-) diff --git a/src/mavedb/lib/gnomad.py b/src/mavedb/lib/gnomad.py index 830cc62d9..712792dbd 100644 --- a/src/mavedb/lib/gnomad.py +++ b/src/mavedb/lib/gnomad.py @@ -271,6 +271,10 @@ def link_gnomad_variants_to_alleles( func.regexp_replace(Allele.clingen_allele_id, _CAID_LEADING_ZERO_RE, r"\1\2").in_(target_caids) ) ).all(): + # clingen_allele_id is nullable on the model, but the WHERE above only matches rows where + # regexp_replace(clingen_allele_id, ...) is non-null (SQL NULL fails .in_()), so every row + # returned here has one. + assert allele.clingen_allele_id is not None alleles_by_caid[normalize_caid(allele.clingen_allele_id)].append(allele) verdicts: dict[int, GnomadLinkVerdict] = {} diff --git a/src/mavedb/models/clinical_control.py b/src/mavedb/models/clinical_control.py index 388444a09..56ad491ca 100644 --- a/src/mavedb/models/clinical_control.py +++ b/src/mavedb/models/clinical_control.py @@ -2,7 +2,7 @@ from typing import TYPE_CHECKING, Optional from sqlalchemy import Column, Date, Integer, String, UniqueConstraint -from sqlalchemy.orm import Mapped, relationship +from sqlalchemy.orm import Mapped, mapped_column, relationship from mavedb.db.base import Base from mavedb.models.clinical_control_mapped_variant import mapped_variants_clinical_controls_association_table @@ -33,7 +33,7 @@ class ClinvarControl(Base): db_version: Mapped[str] = Column(String, nullable=False, index=True) # ClinVar Variation ID (variation level link). - clinvar_variation_id: Mapped[Optional[str]] = Column(String, nullable=True) + clinvar_variation_id: Mapped[Optional[str]] = mapped_column(String, nullable=True) creation_date = Column(Date, nullable=False, default=date.today) modification_date = Column(Date, nullable=False, default=date.today, onupdate=date.today) diff --git a/src/mavedb/models/gnomad_variant.py b/src/mavedb/models/gnomad_variant.py index bb9a1faf6..c0904f614 100644 --- a/src/mavedb/models/gnomad_variant.py +++ b/src/mavedb/models/gnomad_variant.py @@ -2,7 +2,7 @@ from typing import TYPE_CHECKING, Optional from sqlalchemy import Column, Date, Float, Integer, String -from sqlalchemy.orm import Mapped, relationship +from sqlalchemy.orm import Mapped, mapped_column, relationship from mavedb.db.base import Base from mavedb.models.gnomad_variant_mapped_variant import gnomad_variants_mapped_variants_association_table @@ -23,10 +23,10 @@ class GnomADVariant(Base): allele_count: Mapped[int] = Column(Integer, nullable=False) allele_number: Mapped[int] = Column(Integer, nullable=False) - allele_frequency: Mapped[float] = Column(Float, nullable=False) + allele_frequency: Mapped[float] = mapped_column(Float, nullable=False) - faf95_max: Mapped[Optional[float]] = Column(Float, nullable=True) - faf95_max_ancestry: Mapped[Optional[str]] = Column(String, nullable=True) + faf95_max: Mapped[Optional[float]] = mapped_column(Float, nullable=True) + faf95_max_ancestry: Mapped[Optional[str]] = mapped_column(String, nullable=True) creation_date = Column(Date, nullable=False, default=date.today) modification_date = Column(Date, nullable=False, default=date.today, onupdate=date.today) diff --git a/src/mavedb/scripts/audit_allele_identifiers.py b/src/mavedb/scripts/audit_allele_identifiers.py index a05f366dc..5b8fb1380 100644 --- a/src/mavedb/scripts/audit_allele_identifiers.py +++ b/src/mavedb/scripts/audit_allele_identifiers.py @@ -41,7 +41,8 @@ from typing import Any, Optional from ga4gh.core import ga4gh_identify -from ga4gh.vrs.models import SequenceLocation +from ga4gh.vrs.models import Allele as VrsAllele +from ga4gh.vrs.models import CisPhasedBlock, SequenceLocation import asyncclick as click from sqlalchemy import select @@ -65,7 +66,13 @@ def recomputed_identifier(post_mapped: dict[str, Any]) -> Optional[str]: the shim's tolerance costs nothing here. """ try: - return identify_variation(vrs_object_from_mapped_variant(dict(post_mapped)).root) + variation = vrs_object_from_mapped_variant(dict(post_mapped)).root + # vrs_object_from_mapped_variant only ever produces one of these two (its own contract), but its + # declared return type is the full VRS MolecularVariation union, wider than identify_variation + # accepts. + if not isinstance(variation, (VrsAllele, CisPhasedBlock)): + return None + return identify_variation(variation) except Exception: logger.debug("Could not hydrate an allele for identification", exc_info=True) return None @@ -88,6 +95,8 @@ def audit(db: Session, limit: Optional[int], repair: bool) -> None: for allele in db.scalars(query): counts["examined"] += 1 + # post_mapped is nullable on the model, but the query above only selects rows where it isn't. + assert allele.post_mapped is not None computed = recomputed_identifier(allele.post_mapped) if computed is None: @@ -158,6 +167,8 @@ def audit_pre_mapped(db: Session, limit: Optional[int]) -> None: for record in db.scalars(query): counts["examined"] += 1 + # pre_mapped is nullable on the model, but the query above only selects rows where it isn't. + assert record.pre_mapped is not None computed = recomputed_identifier(record.pre_mapped) if computed is None: From e085ba4eb79a8349ee09ad6bc70f4743911c1f80 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Mon, 21 Sep 2026 10:20:21 -0700 Subject: [PATCH 168/252] fix(calibration): give directly-constructed test calibrations a disease term score_calibrations.disease_term_id is now NOT NULL; two helpers in tests/lib/csv/test_variant.py built ScoreCalibration rows without one, tripping the constraint the first time they were exercised against a real session. --- tests/lib/csv/test_variant.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/lib/csv/test_variant.py b/tests/lib/csv/test_variant.py index 6b475d6fd..13b7f5de2 100644 --- a/tests/lib/csv/test_variant.py +++ b/tests/lib/csv/test_variant.py @@ -18,6 +18,7 @@ available_variant_csv_namespaces, get_variant_csv, ) +from mavedb.lib.mondo import get_generic_disease_term from mavedb.lib.permissions.principal import Principal from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer from mavedb.models.acmg_classification import ACMGClassification @@ -63,6 +64,7 @@ def _add_pathogenicity_calibration(db, score_set, variants_in_abnormal_range, ur primary=True, private=False, calibration_metadata={}, + disease_term=get_generic_disease_term(db), created_by_id=score_set.created_by_id, modified_by_id=score_set.modified_by_id, ) @@ -120,6 +122,7 @@ def _add_rangeless_calibration(db, score_set, urn, title): primary=True, private=False, calibration_metadata={}, + disease_term=get_generic_disease_term(db), created_by_id=score_set.created_by_id, modified_by_id=score_set.modified_by_id, ) From e1732e7c02fa8ea1649a3994fd8285877f400d7c Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Mon, 21 Sep 2026 10:21:01 -0700 Subject: [PATCH 169/252] refactor(mondo): split OLS integration out of mondo.py for core-dependency imports mavedb.lib.mondo imported mavedb.lib.logging.context at module level, which pulls in fastapi/starlette (the optional "server" extra) via mavedb.lib.logging.logged_route. tests/helpers/mocks/factories.py imports mondo.py for its generic-term constants and is itself hard-imported by tests/lib/conftest.py, so this broke collection for all of tests/lib/ plus every view_models test that reaches ScoreCalibration, under the "Pytest on Core Dependencies" CI job (no server extras installed). logging.context is only actually needed by the two OLS-network functions (search_mondo, fetch_mondo_term); everything else in the module (constants, MappableConcept serialization, get_generic_disease_term) has no such dependency. Moved the network functions, plus find_or_create_mondo_term and resolve_disease_term (which call them), to a new mondo_ols.py; mondo.py keeps the vocabulary surface and stays importable without server extras. See #459 for the deeper fix (making logging.context itself core-compatible), which this intentionally does not attempt. --- src/mavedb/lib/mondo.py | 143 +++-------------------- src/mavedb/lib/mondo_ols.py | 138 ++++++++++++++++++++++ src/mavedb/lib/score_calibrations.py | 2 +- src/mavedb/routers/diseases.py | 3 +- tests/lib/mondo/network/test_mondo.py | 5 +- tests/lib/mondo/test_mondo.py | 17 ++- tests/routers/test_score_calibrations.py | 2 +- 7 files changed, 168 insertions(+), 142 deletions(-) create mode 100644 src/mavedb/lib/mondo_ols.py diff --git a/src/mavedb/lib/mondo.py b/src/mavedb/lib/mondo.py index 71a9c1223..7906146c8 100644 --- a/src/mavedb/lib/mondo.py +++ b/src/mavedb/lib/mondo.py @@ -1,44 +1,37 @@ -"""MONDO disease-term resolution and search. +"""MONDO disease-term vocabulary: constants, MappableConcept serialization, and the generic term. Calibrations carry a disease/disorder context drawn from the Monarch Disease Ontology (MONDO). This -module owns the MONDO vocabulary integration: get-or-create of :class:`MondoTerm` rows (mirroring -:func:`mavedb.lib.taxonomies.find_or_create_taxonomy`), a typeahead search + validation against the -EBI OLS4 service, and conversion of a stored term to the GA4GH ``MappableConcept`` served on the wire. +module owns the vocabulary's static surface: the generic "disease or disorder" root, conversion of a +stored :class:`MondoTerm` (or an OLS suggestion) to the GA4GH ``MappableConcept`` served on the wire, and +get-or-create of the generic term (mirroring :func:`mavedb.lib.taxonomies.find_or_create_taxonomy`). -The generic term (``MONDO:0000001``, "disease or disorder") is the write-time default for a calibration -with no specific disease, so ``disease_term_id`` is never null. The annotation layer serializes whatever -term a calibration carries — generic or specific — as its VA-Spec disease condition (see +The generic term (``MONDO:0000001``) is the write-time default for a calibration with no specific +disease, so ``disease_term_id`` is never null. The annotation layer serializes whatever term a +calibration carries — generic or specific — as its VA-Spec disease condition (see :func:`mavedb.lib.annotation.condition.calibration_disease_condition`). + +Typeahead search and OLS-backed validation of a submitted code live in :mod:`mavedb.lib.mondo_ols`, +kept separate because they need :mod:`mavedb.lib.logging.context` (an optional "server" dependency, +tracked for a core-compatible rework in +`#459 `_); this module has no such dependency, +so callers that only need the constants or serialization (e.g. the score calibration view model) stay +importable under core dependencies. """ -import logging -from typing import Any, Optional, TypedDict +from typing import TypedDict -import httpx from ga4gh.core.models import Coding, MappableConcept, iriReference from sqlalchemy.orm import Session -from mavedb.lib.exceptions import MondoServiceError -from mavedb.lib.logging.context import format_raised_exception_info_as_dict, logging_context, save_to_logging_context -from mavedb.lib.validation.exceptions import ValidationError from mavedb.models.mondo_term import MondoTerm -logger = logging.getLogger(__name__) - # The Monarch Disease Ontology (MONDO), the controlled vocabulary for disease terms used across MaveDB. MONDO_SYSTEM = "https://purl.obolibrary.org/obo/mondo.owl" -# EBI Ontology Lookup Service (OLS4) — the source for MONDO term search and validation. -OLS_BASE_URL = "https://www.ebi.ac.uk/ols4/api" -MONDO_ONTOLOGY = "mondo" -# OLS can be slow; the default httpx 5s timeout is too tight for its autocomplete on short queries. -OLS_TIMEOUT = 10.0 # The generic "disease or disorder" root, used when a calibration names no specific disease. MONDO_GENERIC_CODE = "MONDO:0000001" MONDO_GENERIC_LABEL = "disease or disorder" -DEFAULT_SEARCH_LIMIT = 20 - class MondoSuggestion(TypedDict): """A single MONDO term resolved from OLS: the CURIE code, its label, and its resolvable IRI.""" @@ -93,64 +86,6 @@ def generic_disease_mappable_concept() -> MappableConcept: ) -async def search_mondo(query: str, limit: int = DEFAULT_SEARCH_LIMIT) -> list[MondoSuggestion]: - """Search MONDO disease classes by free text via OLS, for the disease typeahead.""" - if not query or not query.strip(): - return [] - - # OLS4's /select endpoint is the autocomplete-optimized path; /search times out on broad prefixes - # (e.g. a two-letter query), which is often what a typeahead sends. - params: dict[str, Any] = { - "q": query.strip(), - "ontology": MONDO_ONTOLOGY, - "rows": limit, - "fieldList": "iri,label,obo_id", - } - save_to_logging_context({"mondo_search_query": query, "mondo_search_limit": limit}) - try: - async with httpx.AsyncClient(timeout=OLS_TIMEOUT) as client: - response = await client.get(f"{OLS_BASE_URL}/select", params=params) - response.raise_for_status() - docs = response.json().get("response", {}).get("docs", []) - except httpx.HTTPError as exc: - save_to_logging_context(format_raised_exception_info_as_dict(exc)) - logger.error(msg="MONDO/OLS search request failed.", exc_info=exc, extra=logging_context()) - raise MondoServiceError("Disease ontology service temporarily unavailable") from exc - - return [ - MondoSuggestion(code=doc["obo_id"], label=doc["label"], iri=doc.get("iri", mondo_iri(doc["obo_id"]))) - for doc in docs - if doc.get("obo_id") and doc.get("label") - ] - - -async def fetch_mondo_term(code: str) -> Optional[MondoSuggestion]: - """Resolve one MONDO CURIE to its canonical label + IRI via OLS, or None if it does not exist.""" - params: dict[str, Any] = { - "q": code, - "ontology": MONDO_ONTOLOGY, - "queryFields": "obo_id", - "exact": "true", - "rows": 1, - "fieldList": "iri,label,obo_id", - } - try: - async with httpx.AsyncClient(timeout=OLS_TIMEOUT) as client: - response = await client.get(f"{OLS_BASE_URL}/search", params=params) - response.raise_for_status() - docs = response.json().get("response", {}).get("docs", []) - except httpx.HTTPError as exc: - save_to_logging_context(format_raised_exception_info_as_dict(exc)) - logger.error(msg="MONDO/OLS term lookup failed.", exc_info=exc, extra=logging_context()) - raise MondoServiceError("Disease ontology service temporarily unavailable") from exc - - if not docs: - return None - - doc = docs[0] - return MondoSuggestion(code=doc["obo_id"], label=doc["label"], iri=doc.get("iri", mondo_iri(code))) - - def get_generic_disease_term(db: Session) -> MondoTerm: """Get-or-create the generic disease term. Known-canonical, so it needs no OLS round trip.""" term = ( @@ -162,51 +97,3 @@ def get_generic_disease_term(db: Session) -> MondoTerm: db.flush() return term - - -async def find_or_create_mondo_term(db: Session, code: str) -> MondoTerm: - """Find an existing MONDO term by code, or create one after validating it against OLS. - - A new code is validated against OLS and its canonical label stored. An unknown code raises - ``ValidationError``; an OLS outage raises ``MondoServiceError`` (via :func:`fetch_mondo_term`) so a - term is never persisted without a validated label — a controlled vocabulary must stay validated. - """ - # The generic term is canonical and seeded — resolve it directly, never via OLS. - if code == MONDO_GENERIC_CODE: - return get_generic_disease_term(db) - - term = db.query(MondoTerm).filter(MondoTerm.system == MONDO_SYSTEM, MondoTerm.code == code).one_or_none() - if term is not None: - return term - - resolved = await fetch_mondo_term(code) - if resolved is None: - raise ValidationError(f"'{code}' is not a valid MONDO disease term.", custom_loc=["body", "disease"]) - - # OLS may normalize the submitted code (case, or an alias/obsolete id) to a different canonical code, - # so re-check existence by the resolved code before inserting: keying the lookup on the same value we - # insert under keeps this idempotent and avoids a UNIQUE(system, code) violation when the canonical - # row already exists under a different submitted spelling. - canonical_code = resolved["code"] - term = db.query(MondoTerm).filter(MondoTerm.system == MONDO_SYSTEM, MondoTerm.code == canonical_code).one_or_none() - if term is not None: - return term - - term = MondoTerm(code=canonical_code, system=MONDO_SYSTEM, label=resolved["label"]) # type: ignore[call-arg] - db.add(term) - db.flush() - - return term - - -async def resolve_disease_term(db: Session, code: Optional[str]) -> MondoTerm: - """Resolve a calibration's submitted disease code to a stored MONDO term. - - ``None`` (no disease specified) resolves to the generic "disease or disorder" term; any other code - is validated/created via :func:`find_or_create_mondo_term`. Shared by the score calibration create - and update paths so the write-time default lives in one place. - """ - if code is None: - return get_generic_disease_term(db) - - return await find_or_create_mondo_term(db, code) diff --git a/src/mavedb/lib/mondo_ols.py b/src/mavedb/lib/mondo_ols.py new file mode 100644 index 000000000..c6a58bcec --- /dev/null +++ b/src/mavedb/lib/mondo_ols.py @@ -0,0 +1,138 @@ +"""MONDO disease-term search and validation against the EBI Ontology Lookup Service (OLS4). + +Split out from :mod:`mavedb.lib.mondo`: this module resolves and validates a submitted MONDO code +against the live OLS4 service, which needs :mod:`mavedb.lib.logging.context` for structured request +logging — and that, in turn, needs ``starlette``, an optional "server" dependency (see +`#459 `_ for making logging context available to +core dependencies directly). Keeping the OLS calls here, separate from :mod:`mavedb.lib.mondo`'s +constants and serialization, lets callers that only need the static vocabulary stay importable without +the server extras. +""" + +import logging +from typing import Any, Optional + +import httpx +from sqlalchemy.orm import Session + +from mavedb.lib.exceptions import MondoServiceError +from mavedb.lib.logging.context import format_raised_exception_info_as_dict, logging_context, save_to_logging_context +from mavedb.lib.mondo import MONDO_GENERIC_CODE, MONDO_SYSTEM, MondoSuggestion, get_generic_disease_term, mondo_iri +from mavedb.lib.validation.exceptions import ValidationError +from mavedb.models.mondo_term import MondoTerm + +logger = logging.getLogger(__name__) + +# EBI Ontology Lookup Service (OLS4) — the source for MONDO term search and validation. +OLS_BASE_URL = "https://www.ebi.ac.uk/ols4/api" +MONDO_ONTOLOGY = "mondo" +# OLS can be slow; the default httpx 5s timeout is too tight for its autocomplete on short queries. +OLS_TIMEOUT = 10.0 + +DEFAULT_SEARCH_LIMIT = 20 + + +async def search_mondo(query: str, limit: int = DEFAULT_SEARCH_LIMIT) -> list[MondoSuggestion]: + """Search MONDO disease classes by free text via OLS, for the disease typeahead.""" + if not query or not query.strip(): + return [] + + # OLS4's /select endpoint is the autocomplete-optimized path; /search times out on broad prefixes + # (e.g. a two-letter query), which is often what a typeahead sends. + params: dict[str, Any] = { + "q": query.strip(), + "ontology": MONDO_ONTOLOGY, + "rows": limit, + "fieldList": "iri,label,obo_id", + } + save_to_logging_context({"mondo_search_query": query, "mondo_search_limit": limit}) + try: + async with httpx.AsyncClient(timeout=OLS_TIMEOUT) as client: + response = await client.get(f"{OLS_BASE_URL}/select", params=params) + response.raise_for_status() + docs = response.json().get("response", {}).get("docs", []) + except httpx.HTTPError as exc: + save_to_logging_context(format_raised_exception_info_as_dict(exc)) + logger.error(msg="MONDO/OLS search request failed.", exc_info=exc, extra=logging_context()) + raise MondoServiceError("Disease ontology service temporarily unavailable") from exc + + return [ + MondoSuggestion(code=doc["obo_id"], label=doc["label"], iri=doc.get("iri", mondo_iri(doc["obo_id"]))) + for doc in docs + if doc.get("obo_id") and doc.get("label") + ] + + +async def fetch_mondo_term(code: str) -> Optional[MondoSuggestion]: + """Resolve one MONDO CURIE to its canonical label + IRI via OLS, or None if it does not exist.""" + params: dict[str, Any] = { + "q": code, + "ontology": MONDO_ONTOLOGY, + "queryFields": "obo_id", + "exact": "true", + "rows": 1, + "fieldList": "iri,label,obo_id", + } + try: + async with httpx.AsyncClient(timeout=OLS_TIMEOUT) as client: + response = await client.get(f"{OLS_BASE_URL}/search", params=params) + response.raise_for_status() + docs = response.json().get("response", {}).get("docs", []) + except httpx.HTTPError as exc: + save_to_logging_context(format_raised_exception_info_as_dict(exc)) + logger.error(msg="MONDO/OLS term lookup failed.", exc_info=exc, extra=logging_context()) + raise MondoServiceError("Disease ontology service temporarily unavailable") from exc + + if not docs: + return None + + doc = docs[0] + return MondoSuggestion(code=doc["obo_id"], label=doc["label"], iri=doc.get("iri", mondo_iri(code))) + + +async def find_or_create_mondo_term(db: Session, code: str) -> MondoTerm: + """Find an existing MONDO term by code, or create one after validating it against OLS. + + A new code is validated against OLS and its canonical label stored. An unknown code raises + ``ValidationError``; an OLS outage raises ``MondoServiceError`` (via :func:`fetch_mondo_term`) so a + term is never persisted without a validated label — a controlled vocabulary must stay validated. + """ + # The generic term is canonical and seeded — resolve it directly, never via OLS. + if code == MONDO_GENERIC_CODE: + return get_generic_disease_term(db) + + term = db.query(MondoTerm).filter(MondoTerm.system == MONDO_SYSTEM, MondoTerm.code == code).one_or_none() + if term is not None: + return term + + resolved = await fetch_mondo_term(code) + if resolved is None: + raise ValidationError(f"'{code}' is not a valid MONDO disease term.", custom_loc=["body", "disease"]) + + # OLS may normalize the submitted code (case, or an alias/obsolete id) to a different canonical code, + # so re-check existence by the resolved code before inserting: keying the lookup on the same value we + # insert under keeps this idempotent and avoids a UNIQUE(system, code) violation when the canonical + # row already exists under a different submitted spelling. + canonical_code = resolved["code"] + term = db.query(MondoTerm).filter(MondoTerm.system == MONDO_SYSTEM, MondoTerm.code == canonical_code).one_or_none() + if term is not None: + return term + + term = MondoTerm(code=canonical_code, system=MONDO_SYSTEM, label=resolved["label"]) # type: ignore[call-arg] + db.add(term) + db.flush() + + return term + + +async def resolve_disease_term(db: Session, code: Optional[str]) -> MondoTerm: + """Resolve a calibration's submitted disease code to a stored MONDO term. + + ``None`` (no disease specified) resolves to the generic "disease or disorder" term; any other code + is validated/created via :func:`find_or_create_mondo_term`. Shared by the score calibration create + and update paths so the write-time default lives in one place. + """ + if code is None: + return get_generic_disease_term(db) + + return await find_or_create_mondo_term(db, code) diff --git a/src/mavedb/lib/score_calibrations.py b/src/mavedb/lib/score_calibrations.py index 627d607b7..4e9a7b5e8 100644 --- a/src/mavedb/lib/score_calibrations.py +++ b/src/mavedb/lib/score_calibrations.py @@ -12,7 +12,7 @@ from mavedb.lib.acmg import find_or_create_acmg_classification from mavedb.lib.identifiers import find_or_create_publication_identifier -from mavedb.lib.mondo import resolve_disease_term +from mavedb.lib.mondo_ols import resolve_disease_term from mavedb.lib.types.score_calibrations import ( CalibrationControlSnapshot, CalibrationVariantLinkSnapshot, diff --git a/src/mavedb/routers/diseases.py b/src/mavedb/routers/diseases.py index 6f2d5e70c..f0f2f9daa 100644 --- a/src/mavedb/routers/diseases.py +++ b/src/mavedb/routers/diseases.py @@ -5,7 +5,8 @@ from mavedb.lib.logging.context import save_to_logging_context from mavedb.lib.logging.logged_route import LoggedRoute -from mavedb.lib.mondo import DEFAULT_SEARCH_LIMIT, mondo_suggestion_to_mappable_concept, search_mondo +from mavedb.lib.mondo import mondo_suggestion_to_mappable_concept +from mavedb.lib.mondo_ols import DEFAULT_SEARCH_LIMIT, search_mondo from mavedb.routers.shared import GATEWAY_ERROR_RESPONSES, PUBLIC_ERROR_RESPONSES, ROUTER_BASE_PREFIX TAG_NAME = "Diseases" diff --git a/tests/lib/mondo/network/test_mondo.py b/tests/lib/mondo/network/test_mondo.py index 0daaf2866..8e056927f 100644 --- a/tests/lib/mondo/network/test_mondo.py +++ b/tests/lib/mondo/network/test_mondo.py @@ -9,10 +9,11 @@ import pytest -# starlette is required for logging context functionality pulled in by mavedb.lib.mondo. +# starlette is required for logging context functionality pulled in by mavedb.lib.mondo_ols. pytest.importorskip("starlette") -from mavedb.lib.mondo import MONDO_GENERIC_CODE, fetch_mondo_term, search_mondo +from mavedb.lib.mondo import MONDO_GENERIC_CODE +from mavedb.lib.mondo_ols import fetch_mondo_term, search_mondo # Brugada syndrome: a stable, long-established MONDO disease term used as a known-good fixture. BRUGADA_CODE = "MONDO:0015263" diff --git a/tests/lib/mondo/test_mondo.py b/tests/lib/mondo/test_mondo.py index 10d2a22f9..409fe8102 100644 --- a/tests/lib/mondo/test_mondo.py +++ b/tests/lib/mondo/test_mondo.py @@ -1,4 +1,5 @@ -"""Tests for mavedb.lib.mondo: MONDO term resolution, search, and MappableConcept serialization.""" +"""Tests for mavedb.lib.mondo and mavedb.lib.mondo_ols: MONDO term resolution, search, and +MappableConcept serialization.""" # ruff: noqa: E402 @@ -13,15 +14,13 @@ MONDO_GENERIC_CODE, MONDO_GENERIC_LABEL, MONDO_SYSTEM, - fetch_mondo_term, - find_or_create_mondo_term, generic_disease_mappable_concept, get_generic_disease_term, mondo_iri, mondo_suggestion_to_mappable_concept, mondo_term_to_mappable_concept, - search_mondo, ) +from mavedb.lib.mondo_ols import fetch_mondo_term, find_or_create_mondo_term, search_mondo from mavedb.lib.validation.exceptions import ValidationError from mavedb.models.mondo_term import MondoTerm @@ -149,7 +148,7 @@ async def test_creates_from_ols_and_is_idempotent(self, session, monkeypatch): async def fake_fetch(code): return {"code": code, "label": "Brugada syndrome", "iri": mondo_iri(code)} - monkeypatch.setattr("mavedb.lib.mondo.fetch_mondo_term", fake_fetch) + monkeypatch.setattr("mavedb.lib.mondo_ols.fetch_mondo_term", fake_fetch) term = await find_or_create_mondo_term(session, "MONDO:0015263") assert term.code == "MONDO:0015263" @@ -167,7 +166,7 @@ async def test_non_canonical_code_reuses_existing_canonical_row(self, session, m async def fake_fetch(code): return {"code": "MONDO:0015263", "label": "Brugada syndrome", "iri": mondo_iri("MONDO:0015263")} - monkeypatch.setattr("mavedb.lib.mondo.fetch_mondo_term", fake_fetch) + monkeypatch.setattr("mavedb.lib.mondo_ols.fetch_mondo_term", fake_fetch) canonical = await find_or_create_mondo_term(session, "MONDO:0015263") # A differently-spelled submission (here lowercase) resolves to the same canonical term. @@ -181,7 +180,7 @@ async def test_generic_code_resolves_without_ols(self, session, monkeypatch): async def fail_fetch(code): raise AssertionError("OLS should not be consulted for the generic term") - monkeypatch.setattr("mavedb.lib.mondo.fetch_mondo_term", fail_fetch) + monkeypatch.setattr("mavedb.lib.mondo_ols.fetch_mondo_term", fail_fetch) term = await find_or_create_mondo_term(session, MONDO_GENERIC_CODE) assert term.code == MONDO_GENERIC_CODE assert term.label == MONDO_GENERIC_LABEL @@ -190,7 +189,7 @@ async def test_rejects_unknown_code(self, session, monkeypatch): async def fake_fetch(code): return None - monkeypatch.setattr("mavedb.lib.mondo.fetch_mondo_term", fake_fetch) + monkeypatch.setattr("mavedb.lib.mondo_ols.fetch_mondo_term", fake_fetch) with pytest.raises(ValidationError): await find_or_create_mondo_term(session, "MONDO:9999999") @@ -199,7 +198,7 @@ async def test_ols_outage_propagates_service_error(self, session, monkeypatch): async def fake_fetch(code): raise MondoServiceError("ols down") - monkeypatch.setattr("mavedb.lib.mondo.fetch_mondo_term", fake_fetch) + monkeypatch.setattr("mavedb.lib.mondo_ols.fetch_mondo_term", fake_fetch) with pytest.raises(MondoServiceError): await find_or_create_mondo_term(session, "MONDO:0015263") diff --git a/tests/routers/test_score_calibrations.py b/tests/routers/test_score_calibrations.py index e5bb5c486..62a7660db 100644 --- a/tests/routers/test_score_calibrations.py +++ b/tests/routers/test_score_calibrations.py @@ -195,7 +195,7 @@ async def fake_fetch(code): "iri": f"https://purl.obolibrary.org/obo/{code.replace(':', '_')}", } - monkeypatch.setattr("mavedb.lib.mondo.fetch_mondo_term", fake_fetch) + monkeypatch.setattr("mavedb.lib.mondo_ols.fetch_mondo_term", fake_fetch) experiment = create_experiment(client) score_set = create_seq_score_set_with_mapped_variants( From 4138248ae6978d543b14a1660cc2dc33c4490ea2 Mon Sep 17 00:00:00 2001 From: Estelle Da Date: Wed, 23 Sep 2026 11:33:09 +1000 Subject: [PATCH 170/252] Modify the find_generic_article function. --- src/mavedb/lib/external_publications.py | 10 +--------- src/mavedb/lib/identifiers.py | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/src/mavedb/lib/external_publications.py b/src/mavedb/lib/external_publications.py index 96c4f8dd7..326102d77 100644 --- a/src/mavedb/lib/external_publications.py +++ b/src/mavedb/lib/external_publications.py @@ -565,12 +565,4 @@ def _fetch(self, url: str, return_format: str = "json") -> Any: response.raise_for_status() - if return_format == "json": - if not response.text or not response.text.strip(): - return [] - try: - return json.loads(response.text) - except json.JSONDecodeError: - return [] - - return response.text + return json.loads(response.text) if return_format == "json" else response.text diff --git a/src/mavedb/lib/identifiers.py b/src/mavedb/lib/identifiers.py index d5bfdbd46..d04fd3fae 100644 --- a/src/mavedb/lib/identifiers.py +++ b/src/mavedb/lib/identifiers.py @@ -1,3 +1,4 @@ +import logging import os from typing import Mapping, Optional, Union @@ -23,6 +24,8 @@ from mavedb.models.uniprot_identifier import UniprotIdentifier from mavedb.models.uniprot_offset import UniprotOffset +logger = logging.getLogger(__name__) + # XXX these classes all have an "identifier" attribute but there's no superclass # to unify them ... @@ -322,7 +325,17 @@ async def find_generic_article( ).scalar_one_or_none() if not existing_publication: - external_publication = await db_specific_fetches[publication_db](identifier) + try: + external_publication = await db_specific_fetches[publication_db](identifier) + except Exception: + logger.warning( + "Failed to fetch identifier %r from %s while fanning out over candidate databases.", + identifier, + publication_db, + exc_info=True, + ) + external_publication = None + found_articles[publication_db] = ( ExternalPublication(identifier, publication_db, external_publication) if external_publication From a068959831ffd3bdd4673550a7448a9a3903621a Mon Sep 17 00:00:00 2001 From: Estelle Da Date: Wed, 23 Sep 2026 12:14:50 +1000 Subject: [PATCH 171/252] Debug a test. --- src/mavedb/lib/identifiers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/mavedb/lib/identifiers.py b/src/mavedb/lib/identifiers.py index d04fd3fae..c8d5d6acf 100644 --- a/src/mavedb/lib/identifiers.py +++ b/src/mavedb/lib/identifiers.py @@ -1,3 +1,4 @@ +import json import logging import os from typing import Mapping, Optional, Union @@ -327,7 +328,7 @@ async def find_generic_article( if not existing_publication: try: external_publication = await db_specific_fetches[publication_db](identifier) - except Exception: + except json.JSONDecodeError: logger.warning( "Failed to fetch identifier %r from %s while fanning out over candidate databases.", identifier, From 8a28b20b0e0687f2477ae8061d62dcebaa5253dd Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Thu, 24 Sep 2026 14:12:17 -0700 Subject: [PATCH 172/252] feat(reverse-translation): record UTA outages as api_error and reconnect UTA - Bump variant-annotation to v0.1.0-dev.2, which retries transient UTA connection failures in reverse translation - Record TranslationErrorReason.UPSTREAM_UNAVAILABLE as failed/api_error so outages stay distinguishable from genuine translation errors - uta_transcript_source yields the reconnecting UtaClient.from_url client --- poetry.lock | 8 +-- pyproject.toml | 2 +- src/mavedb/models/enums/event_reason.py | 2 +- .../variant_processing/reverse_translation.py | 14 +++-- src/mavedb/worker/lib/translation_ports.py | 11 ++-- .../test_reverse_translation.py | 53 ++++++++++++++++++- 6 files changed, 74 insertions(+), 16 deletions(-) diff --git a/poetry.lock b/poetry.lock index c7fe295d6..7e9cdf708 100644 --- a/poetry.lock +++ b/poetry.lock @@ -4872,7 +4872,7 @@ test = ["aiohttp (>=3.10.5)", "flake8 (>=6.1,<7.0)", "mypy (>=0.800)", "psutil", [[package]] name = "variant-annotation" -version = "0.1.0-dev.1" +version = "0.1.0-dev.2" description = "Genetic variant mapping and annotation pipeline" optional = true python-versions = ">=3.11" @@ -4902,8 +4902,8 @@ tests = ["pytest"] [package.source] type = "git" url = "https://github.com/bbi-lab/variant-annotation.git" -reference = "v0.1.0-dev.1" -resolved_reference = "199e1a0d78050aaea8798744352065190aeed617" +reference = "v0.1.0-dev.2" +resolved_reference = "073b7aa07c44fad1059725733ff8b11ac0dd2d4b" [[package]] name = "variant-translation" @@ -5235,4 +5235,4 @@ server = ["aiocache", "alembic", "alembic-utils", "arq", "authlib", "biocommons" [metadata] lock-version = "2.1" python-versions = "^3.11" -content-hash = "5644e132fe7ac94a34e5ad512912b6dda1374ebe5f2ff802e2117acfdea9b5f2" +content-hash = "72f5ecf8eaa4554e8db078dc2910b7ec656dd0a4b9774180c6794a4ec841aa5c" diff --git a/pyproject.toml b/pyproject.toml index 6944cfc41..498fd6cfe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,7 +64,7 @@ starlette = { version = "~0.49.0", optional = true } starlette-context = { version = "^0.3.6", optional = true } slack-sdk = { version = "~3.21.3", optional = true } uvicorn = { extras = ["standard"], version = "*", optional = true } -variant-annotation = { git = "https://github.com/bbi-lab/variant-annotation.git", tag = "v0.1.0-dev.1", optional = true } +variant-annotation = { git = "https://github.com/bbi-lab/variant-annotation.git", tag = "v0.1.0-dev.2", optional = true } watchtower = { version = "~3.2.0", optional = true } asyncclick = "^8.3.0.7" filelock = "^3.29.0" diff --git a/src/mavedb/models/enums/event_reason.py b/src/mavedb/models/enums/event_reason.py index 9a9ffdbd3..3bcb5ee5c 100644 --- a/src/mavedb/models/enums/event_reason.py +++ b/src/mavedb/models/enums/event_reason.py @@ -49,7 +49,7 @@ class EventReason(str, Enum): ) # failed — errored - API_ERROR = "api_error" # network/timeout/upstream error (ClinVar, VEP, LDH, CAR no-response) + API_ERROR = "api_error" # network/timeout/upstream error (ClinVar, VEP, LDH, CAR no-response, RT UTA outage) SERVICE_REJECTED = "service_rejected" # external service refused the input (CAR) MALFORMED_RESPONSE = "malformed_response" # unparseable/contractless response (CAR) CAID_CONFLICT = "caid_conflict" # returned identifier conflicts with the stored one (CAR) diff --git a/src/mavedb/worker/jobs/variant_processing/reverse_translation.py b/src/mavedb/worker/jobs/variant_processing/reverse_translation.py index a33a70008..ad5e64656 100644 --- a/src/mavedb/worker/jobs/variant_processing/reverse_translation.py +++ b/src/mavedb/worker/jobs/variant_processing/reverse_translation.py @@ -544,9 +544,10 @@ async def reverse_translate_variants_for_score_set( ) # The library types each error's reason: NOT_TRANSLATABLE is a benign structural gap (the - # protein consequence's edit type — del/ins/delins/fs/ext/stop-loss — has no DNA equivalence - # class to construct), everything else is a genuine failure. Split on that typed reason rather - # than pattern-matching the engine's error text. + # protein consequence's edit type — ins/delins/dup/fs/ext/stop-loss — has no DNA equivalence + # class to construct), UPSTREAM_UNAVAILABLE is a UTA outage that outlasted the library's + # retries, and FAILED is a genuine translation error. Split on that typed reason rather than + # pattern-matching the engine's error text. for error in errors: _rec, variant = variant_input_map[id(error.input)] if error.reason is TranslationErrorReason.NOT_TRANSLATABLE: @@ -567,7 +568,12 @@ async def reverse_translate_variants_for_score_set( annotation_manager, variant_id=variant.id, disposition=Disposition.FAILED, - reason=EventReason.TRANSLATION_ERROR, + # An upstream outage says nothing about the variant, so keep it out of translation_error. + reason=( + EventReason.API_ERROR + if error.reason is TranslationErrorReason.UPSTREAM_UNAVAILABLE + else EventReason.TRANSLATION_ERROR + ), metadata={"hgvs_input": error.input.hgvs, "error_message": error.error}, ) diff --git a/src/mavedb/worker/lib/translation_ports.py b/src/mavedb/worker/lib/translation_ports.py index b715aace4..1fd34809f 100644 --- a/src/mavedb/worker/lib/translation_ports.py +++ b/src/mavedb/worker/lib/translation_ports.py @@ -17,7 +17,7 @@ from collections.abc import Generator from typing import Any -from variant_annotation.lib.clients.uta import UtaClient, connect_uta +from variant_annotation.lib.clients.uta import UtaClient @contextlib.contextmanager @@ -25,14 +25,15 @@ def uta_transcript_source() -> Generator[UtaClient]: """Yield a UTA-backed TranscriptSource over a connection scoped to the block. Backs both the NP_→NM_ association lookup and the WT-codon read used by - WtCodonMode.ALL (TranscriptSource.codon_at). The connection is closed on exit, - so callers must use the client within the ``with`` block. + WtCodonMode.ALL (TranscriptSource.codon_at). The client reconnects when UTA drops + the connection and closes it on exit, so callers must use the client within the + ``with`` block. """ uta_db_url = (os.environ.get("UTA_DB_URL") or "").strip() if not uta_db_url: raise RuntimeError("UTA_DB_URL must be set to resolve transcript facts (NP_→NM_ associations and WT codons).") - with contextlib.closing(connect_uta(uta_db_url)) as conn: - yield UtaClient(conn) + with UtaClient.from_url(uta_db_url) as client: + yield client class WorkerCoordinateTranslator: diff --git a/tests/worker/jobs/variant_processing/test_reverse_translation.py b/tests/worker/jobs/variant_processing/test_reverse_translation.py index c4d16d055..0cb425563 100644 --- a/tests/worker/jobs/variant_processing/test_reverse_translation.py +++ b/tests/worker/jobs/variant_processing/test_reverse_translation.py @@ -66,6 +66,7 @@ def fake_construct( results_by_hgvs: dict, errors_by_hgvs: dict | None = None, not_translatable_by_hgvs: dict | None = None, + upstream_unavailable_by_hgvs: dict | None = None, ): """Build a stand-in for ``construct_equivalent_variants``. @@ -85,10 +86,12 @@ def fake_construct( ``errors_by_hgvs`` maps an assay HGVS to a genuine-failure message (reason ``FAILED``); ``not_translatable_by_hgvs`` maps one to a benign non-translatable message (reason - ``NOT_TRANSLATABLE``), standing in for an edit type the library screens out up front. + ``NOT_TRANSLATABLE``), standing in for an edit type the library screens out up front; + ``upstream_unavailable_by_hgvs`` maps one to a UTA outage message (reason ``UPSTREAM_UNAVAILABLE``). """ errors_by_hgvs = errors_by_hgvs or {} not_translatable_by_hgvs = not_translatable_by_hgvs or {} + upstream_unavailable_by_hgvs = upstream_unavailable_by_hgvs or {} def _construct(inputs, *, transcripts, coordinates, config): results, errors = [], [] @@ -101,6 +104,14 @@ def _construct(inputs, *, transcripts, coordinates, config): reason=TranslationErrorReason.NOT_TRANSLATABLE, ) ) + elif inp.hgvs in upstream_unavailable_by_hgvs: + errors.append( + TranslationError( + input=inp, + error=upstream_unavailable_by_hgvs[inp.hgvs], + reason=TranslationErrorReason.UPSTREAM_UNAVAILABLE, + ) + ) elif inp.hgvs in errors_by_hgvs: errors.append(TranslationError(input=inp, error=errors_by_hgvs[inp.hgvs])) elif inp.hgvs in results_by_hgvs: @@ -1235,6 +1246,46 @@ async def test_non_translatable_skip_coexists_with_success_and_failure( assert len(skipped) == 1 assert skipped[0].disposition == "not_applicable" + async def test_upstream_outage_is_failed_as_api_error_not_translation_error( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """A UTA outage that outlasts the library's retries is FAILED with reason api_error, so it + stays distinguishable from a genuine translation_error when auditing coverage gaps.""" + variant_ok = Variant(score_set_id=sample_score_set.id, urn="variant:1", hgvs_nt="NM_000000.1:c.1A>G", data={}) + variant_down = Variant(score_set_id=sample_score_set.id, urn="variant:2", hgvs_nt="NM_000000.1:c.2G>T", data={}) + session.add_all([variant_ok, variant_down]) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + c_candidate = "NM_000001.1:c.5A>G" + outage = "reverse-translate-variants failed: server closed the connection unexpectedly" + construct = fake_construct( + {"NM_000000.1:c.1A>G": [(c_candidate, None)]}, + upstream_unavailable_by_hgvs={"NM_000000.1:c.2G>T": outage}, + ) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", fake_translate({c_candidate: "ga4gh:VA.coding"})), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 1, "failed": 1, "skipped": 0, "alleles_created": 1} + + failed = _cross_level_events(session, sample_score_set.id, disposition="failed") + assert len(failed) == 1 + assert failed[0].reason == "api_error" + assert failed[0].event_metadata["error_message"] == outage + assert _cross_level_events(session, sample_score_set.id, reason="translation_error") == [] + async def test_partial_candidate_translation_failure_keeps_success_with_metadata( self, session, From 5d239636ac98aebe5948a1dbf5c682ee281fb7fb Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Mon, 28 Sep 2026 15:33:05 -0700 Subject: [PATCH 173/252] feat(rls): carry score_set_id on mapping_records and mapping_record_alleles (#833) Both tables reached their score set only through variants, and an RLS policy that delegates through a table that size costs ~700 ms per lookup. Each now carries score_set_id under a composite foreign key to its parent's (id, score set), so the column can't disagree with the variant it belongs to. The composite keys replace the single-column variant_id and mapping_record_id foreign keys. - New migration 45e8ec1db15e rather than an edit to the create-table migration, since staging has already run RT's migrations. It backfills from variants, then sets NOT NULL. In production the tables are empty when it runs. - The variants (id, scoreset_id) unique index must be built CONCURRENTLY by hand in production before the deploy; the migration's IF NOT EXISTS then does nothing. - The new columns use the newer tables' score_set_id naming; variants keeps scoreset_id. - The mapping job, reverse translation and the reshape script set score_set_id on every row; the reshape script looks score sets up once per variant chunk. - Variant.score_set_id and MappingRecord.score_set_id are annotated Mapped[int], which they already were at the database level. --- ...ate_mapped_variants_to_allele_substrate.py | 6 ++ ...5e_carry_score_set_id_on_mapping_tables.py | 92 +++++++++++++++++++ src/mavedb/models/mapping_record.py | 16 +++- src/mavedb/models/mapping_record_allele.py | 16 ++-- src/mavedb/models/variant.py | 10 +- .../worker/jobs/variant_processing/mapping.py | 2 + .../variant_processing/reverse_translation.py | 1 + tests/helpers/util/annotation.py | 2 + tests/lib/csv/test_variant.py | 73 ++++++++++++++- tests/lib/test_allele_measurements.py | 8 +- tests/lib/test_alleles.py | 12 ++- tests/lib/test_cat_vrs.py | 12 ++- tests/lib/test_score_set_variants.py | 2 + tests/lib/test_variant_annotations_script.py | 10 +- tests/lib/test_variant_detail.py | 2 + tests/routers/test_clingen_allele.py | 35 +++++-- tests/routers/test_variant.py | 26 +++++- tests/worker/jobs/conftest.py | 8 ++ .../jobs/external_services/test_clingen.py | 2 + .../external_services/test_clingen_cache.py | 6 ++ .../jobs/variant_processing/test_mapping.py | 7 +- 21 files changed, 317 insertions(+), 31 deletions(-) create mode 100644 alembic/versions/45e8ec1db15e_carry_score_set_id_on_mapping_tables.py diff --git a/alembic/manual_migrations/migrate_mapped_variants_to_allele_substrate.py b/alembic/manual_migrations/migrate_mapped_variants_to_allele_substrate.py index 0d9ddc0ec..ea0e239bc 100644 --- a/alembic/manual_migrations/migrate_mapped_variants_to_allele_substrate.py +++ b/alembic/manual_migrations/migrate_mapped_variants_to_allele_substrate.py @@ -800,6 +800,7 @@ def _backfill_annotation_timelines( def _migrate_variant( db: Session, variant_id: int, + score_set_id: int, mvs: list[MappedVariant], stats: Counter, skipped: dict[str, list[int]], @@ -910,6 +911,7 @@ def _migrate_variant( record = MappingRecord( variant_id=variant_id, + score_set_id=score_set_id, vrs_digest=pre_mapped_digest, pre_mapped=pre_mapped_document, assay_level=level, @@ -1124,6 +1126,9 @@ def do_migration( ) ).all() ) + score_set_ids: dict[int, int] = dict( + db.execute(sa.select(Variant.id, Variant.score_set_id).where(Variant.id.in_(chunk))).tuples().all() + ) candidate_digests = {(mv.post_mapped or {}).get("id") for mvs in mvs_by_variant.values() for mv in mvs} candidate_digests.discard(None) allele_cache: dict[str, Allele] = {} @@ -1143,6 +1148,7 @@ def do_migration( variant_touched_allele_ids = _migrate_variant( db, variant_id, + score_set_ids[variant_id], mvs_by_variant[variant_id], stats, skipped, diff --git a/alembic/versions/45e8ec1db15e_carry_score_set_id_on_mapping_tables.py b/alembic/versions/45e8ec1db15e_carry_score_set_id_on_mapping_tables.py new file mode 100644 index 000000000..b0a68c059 --- /dev/null +++ b/alembic/versions/45e8ec1db15e_carry_score_set_id_on_mapping_tables.py @@ -0,0 +1,92 @@ +"""carry score_set_id on mapping_records and mapping_record_alleles + +Revision ID: 45e8ec1db15e +Revises: d1c9b7a3e2f4 +Create Date: 2026-09-28 + +Adds a ``score_set_id`` column to ``mapping_records`` and ``mapping_record_alleles`` so an RLS policy can +check a row's score set directly (#833). Both tables otherwise reach their score set only through +``variants``, and delegating through a table that size costs ~700 ms per lookup. + +Each column sits under a composite foreign key to its parent's id and score set, so it can't disagree +with the variant it belongs to. The composite keys replace the single-column ``variant_id`` and +``mapping_record_id`` foreign keys. + +**Before deploying to production**, build the ``variants`` index by hand, since a migration can't build it +concurrently (#877). The ``IF NOT EXISTS`` below then does nothing there, and builds the index on an empty +or small database:: + + CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS uq_variants_id_scoreset ON variants (id, scoreset_id); + +The backfills are sized for staging, where the mapping tables already hold data. In production the tables +are empty when this runs. +""" + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "45e8ec1db15e" +down_revision = "d1c9b7a3e2f4" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.execute("CREATE UNIQUE INDEX IF NOT EXISTS uq_variants_id_scoreset ON variants (id, scoreset_id)") + + op.execute("ALTER TABLE mapping_records ADD COLUMN score_set_id integer") + op.execute("UPDATE mapping_records mr SET score_set_id = v.scoreset_id FROM variants v WHERE v.id = mr.variant_id") + op.execute("ALTER TABLE mapping_records ALTER COLUMN score_set_id SET NOT NULL") + op.execute("CREATE UNIQUE INDEX uq_mapping_records_id_score_set ON mapping_records (id, score_set_id)") + op.drop_constraint("fk_mapping_records_variant_id", "mapping_records", type_="foreignkey") + op.create_foreign_key( + "fk_mapping_records_variant_score_set", + "mapping_records", + "variants", + ["variant_id", "score_set_id"], + ["id", "scoreset_id"], + ) + + op.execute("ALTER TABLE mapping_record_alleles ADD COLUMN score_set_id integer") + op.execute( + "UPDATE mapping_record_alleles mra SET score_set_id = mr.score_set_id " + "FROM mapping_records mr WHERE mr.id = mra.mapping_record_id" + ) + op.execute("ALTER TABLE mapping_record_alleles ALTER COLUMN score_set_id SET NOT NULL") + op.drop_constraint("fk_mapping_record_alleles_mapping_record_id", "mapping_record_alleles", type_="foreignkey") + op.create_foreign_key( + "fk_mapping_record_alleles_mapping_record_score_set", + "mapping_record_alleles", + "mapping_records", + ["mapping_record_id", "score_set_id"], + ["id", "score_set_id"], + ondelete="CASCADE", + ) + + +def downgrade() -> None: + op.drop_constraint( + "fk_mapping_record_alleles_mapping_record_score_set", "mapping_record_alleles", type_="foreignkey" + ) + op.create_foreign_key( + "fk_mapping_record_alleles_mapping_record_id", + "mapping_record_alleles", + "mapping_records", + ["mapping_record_id"], + ["id"], + ondelete="CASCADE", + ) + op.drop_column("mapping_record_alleles", "score_set_id") + + op.drop_constraint("fk_mapping_records_variant_score_set", "mapping_records", type_="foreignkey") + op.create_foreign_key( + "fk_mapping_records_variant_id", + "mapping_records", + "variants", + ["variant_id"], + ["id"], + ) + op.drop_index("uq_mapping_records_id_score_set", table_name="mapping_records") + op.drop_column("mapping_records", "score_set_id") + + op.drop_index("uq_variants_id_scoreset", table_name="variants") diff --git a/src/mavedb/models/mapping_record.py b/src/mavedb/models/mapping_record.py index dda9ef16e..ad7d0151c 100644 --- a/src/mavedb/models/mapping_record.py +++ b/src/mavedb/models/mapping_record.py @@ -1,7 +1,7 @@ from datetime import date from typing import TYPE_CHECKING, Any, Optional -from sqlalchemy import Boolean, Column, Date, Enum, ForeignKey, Index, Integer, String, func, text +from sqlalchemy import Boolean, Column, Date, Enum, ForeignKey, ForeignKeyConstraint, Index, Integer, String, func, text from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.orm import Mapped, relationship @@ -22,7 +22,11 @@ class MappingRecord(ValidTime, Base): id: Mapped[int] = Column(Integer, primary_key=True) - variant_id = Column(Integer, ForeignKey("variants.id"), nullable=False) + variant_id = Column(Integer, nullable=False) + # Denormalized from the variant so an RLS policy can check the score set directly; delegating + # through variants costs ~700 ms per lookup at production size (#833). The composite foreign key + # below keeps it equal to the variant's score set. + score_set_id: Mapped[int] = Column(Integer, nullable=False) variant: Mapped["Variant"] = relationship("Variant", back_populates="mapping_records") # Digest of the pre-mapped (assayed-level) VRS representation, indexed for @@ -98,6 +102,14 @@ def _transcript_expression(cls): __retire_cascade__ = ("allele_links",) __table_args__ = ( + ForeignKeyConstraint( + [variant_id, score_set_id], + ["variants.id", "variants.scoreset_id"], + name="fk_mapping_records_variant_score_set", + link_to_name=True, + ), + # Target of mapping_record_alleles' composite foreign key. + Index("uq_mapping_records_id_score_set", id, score_set_id, unique=True), Index("ix_mapping_records_vrs_digest", "vrs_digest"), Index("ix_mapping_records_variant_id", "variant_id"), # At most one live mapping record per variant — promotes to the database the invariant the diff --git a/src/mavedb/models/mapping_record_allele.py b/src/mavedb/models/mapping_record_allele.py index ce57f259d..79a4f7573 100644 --- a/src/mavedb/models/mapping_record_allele.py +++ b/src/mavedb/models/mapping_record_allele.py @@ -1,6 +1,6 @@ from typing import TYPE_CHECKING -from sqlalchemy import Boolean, Column, ForeignKey, Index, Integer, text +from sqlalchemy import Boolean, Column, ForeignKey, ForeignKeyConstraint, Index, Integer, text from sqlalchemy.orm import Mapped, relationship from mavedb.db.base import Base @@ -15,11 +15,9 @@ class MappingRecordAllele(ValidTime, Base): __tablename__ = "mapping_record_alleles" id: Mapped[int] = Column(Integer, primary_key=True) - mapping_record_id: Mapped[int] = Column( - Integer, - ForeignKey("mapping_records.id", ondelete="CASCADE"), - nullable=False, - ) + mapping_record_id: Mapped[int] = Column(Integer, nullable=False) + # Denormalized from the mapping record for the RLS policy, like MappingRecord.score_set_id (#833). + score_set_id: Mapped[int] = Column(Integer, nullable=False) allele_id: Mapped[int] = Column( Integer, ForeignKey("alleles.id", ondelete="RESTRICT"), @@ -38,6 +36,12 @@ class MappingRecordAllele(ValidTime, Base): allele: Mapped["Allele"] = relationship("Allele", back_populates="mapping_record_links") __table_args__ = ( + ForeignKeyConstraint( + [mapping_record_id, score_set_id], + ["mapping_records.id", "mapping_records.score_set_id"], + name="fk_mapping_record_alleles_mapping_record_score_set", + ondelete="CASCADE", + ), Index( "ix_mapping_record_alleles_mapping_record_id", "mapping_record_id", diff --git a/src/mavedb/models/variant.py b/src/mavedb/models/variant.py index dd46c430c..bcbe33ca6 100644 --- a/src/mavedb/models/variant.py +++ b/src/mavedb/models/variant.py @@ -1,7 +1,7 @@ from datetime import date from typing import TYPE_CHECKING, List -from sqlalchemy import Column, Date, ForeignKey, Integer, String +from sqlalchemy import Column, Date, ForeignKey, Index, Integer, String from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.orm import Mapped, relationship @@ -21,7 +21,7 @@ class Variant(Base): urn = Column(String(64), index=True, nullable=True, unique=True) data = Column(JSONB, nullable=False) - score_set_id = Column("scoreset_id", Integer, ForeignKey("scoresets.id"), index=True, nullable=False) + score_set_id: Mapped[int] = Column("scoreset_id", Integer, ForeignKey("scoresets.id"), index=True, nullable=False) # TODO examine if delete-orphan is necessary, explore cascade score_set: Mapped["ScoreSet"] = relationship(back_populates="variants") @@ -42,3 +42,9 @@ class Variant(Base): # Bidirectional relationship with ScoreCalibrationFunctionalClassification is left # purposefully undefined for performance reasons. + + __table_args__ = ( + # Target of mapping_records' composite (variant_id, score_set_id) foreign key. The records + # carry their score set so an RLS policy can check it without joining through variants (#833). + Index("uq_variants_id_scoreset", id, score_set_id, unique=True), + ) diff --git a/src/mavedb/worker/jobs/variant_processing/mapping.py b/src/mavedb/worker/jobs/variant_processing/mapping.py index f6385a994..7f50ecad7 100644 --- a/src/mavedb/worker/jobs/variant_processing/mapping.py +++ b/src/mavedb/worker/jobs/variant_processing/mapping.py @@ -396,6 +396,7 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan mapping_record = MappingRecord( variant_id=variant.id, + score_set_id=variant.score_set_id, vrs_digest=pre_mapped_allele.get("id"), pre_mapped=pre_mapped_allele or None, assay_level=sequence_level, @@ -465,6 +466,7 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan job_manager.db.add( MappingRecordAllele( mapping_record_id=mapping_record.id, + score_set_id=mapping_record.score_set_id, allele_id=authoritative_allele.id, is_authoritative=True, ) diff --git a/src/mavedb/worker/jobs/variant_processing/reverse_translation.py b/src/mavedb/worker/jobs/variant_processing/reverse_translation.py index ad5e64656..7ee7fe7c0 100644 --- a/src/mavedb/worker/jobs/variant_processing/reverse_translation.py +++ b/src/mavedb/worker/jobs/variant_processing/reverse_translation.py @@ -491,6 +491,7 @@ async def reverse_translate_variants_for_score_set( new_links.append( MappingRecordAllele( mapping_record_id=rec.id, + score_set_id=rec.score_set_id, allele_id=allele.id, is_authoritative=False, projection_group=projection_group, diff --git a/tests/helpers/util/annotation.py b/tests/helpers/util/annotation.py index 24b9c11f3..7f92067cc 100644 --- a/tests/helpers/util/annotation.py +++ b/tests/helpers/util/annotation.py @@ -78,6 +78,7 @@ def seed_mapping_record( record = MappingRecord( variant_id=variant.id, + score_set_id=variant.score_set_id, assay_level=assay_level, hgvs_assay_level=hgvs_assay_level, mapping_api_version=mapping_api_version, @@ -104,6 +105,7 @@ def seed_mapping_record( links: list[Any] = [ MappingRecordAllele( mapping_record_id=record.id, + score_set_id=record.score_set_id, allele_id=allele.id, is_authoritative=spec.is_authoritative, projection_group=spec.projection_group, diff --git a/tests/lib/csv/test_variant.py b/tests/lib/csv/test_variant.py index 74f5ecf66..ed2ad3d00 100644 --- a/tests/lib/csv/test_variant.py +++ b/tests/lib/csv/test_variant.py @@ -1039,7 +1039,69 @@ class TestOneLiveMappingPerVariantIsGuaranteed: def test_a_second_live_mapping_record_is_rejected(self, session, setup_lib_db_with_mapped_variant): variant = setup_lib_db_with_mapped_variant.variant # The fixture already seeded one live record for this variant. - session.add(MappingRecord(variant_id=variant.id, assay_level="genomic", mapping_api_version="test.0.0")) + session.add( + MappingRecord( + variant_id=variant.id, + score_set_id=variant.score_set_id, + assay_level="genomic", + mapping_api_version="test.0.0", + ) + ) + + with pytest.raises(IntegrityError): + session.commit() + session.rollback() + + @staticmethod + def _other_score_set(session, score_set): + other = ScoreSet( + **{k: v for k, v in TEST_SEQ_SCORESET.items() if k != "target_genes"}, + urn="urn:mavedb:00000001-a-3", + experiment_id=score_set.experiment_id, + licence_id=score_set.licence_id, + created_by_id=score_set.created_by_id, + modified_by_id=score_set.modified_by_id, + ) + session.add(other) + session.commit() + return other + + def test_a_record_cannot_claim_another_score_set(self, session, setup_lib_db_with_mapped_variant): + """``scoreset_id`` is carried for row-level security, so it must match the variant's score set. + + The other score set exists, so the rejection comes from the composite key on + ``(variant_id, scoreset_id)``, not from a missing parent. + """ + variant = setup_lib_db_with_mapped_variant.variant + other = self._other_score_set(session, variant.score_set) + live = session.scalars( + select(MappingRecord).where(MappingRecord.variant_id == variant.id).where(MappingRecord.valid_to.is_(None)) + ).one() + live.retire(session) + session.add( + MappingRecord( + variant_id=variant.id, + score_set_id=other.id, + assay_level="genomic", + mapping_api_version="test.1.0", + ) + ) + + with pytest.raises(IntegrityError): + session.commit() + session.rollback() + + def test_an_allele_link_cannot_claim_another_score_set(self, session, setup_lib_db_with_mapped_variant): + """A link's ``scoreset_id`` must match its mapping record's, by the composite key on the link.""" + variant = setup_lib_db_with_mapped_variant.variant + other = self._other_score_set(session, variant.score_set) + live = session.scalars( + select(MappingRecord).where(MappingRecord.variant_id == variant.id).where(MappingRecord.valid_to.is_(None)) + ).one() + allele = Allele(vrs_digest="lib-csv-unlinked", level="genomic") + session.add(allele) + session.commit() + session.add(MappingRecordAllele(mapping_record_id=live.id, score_set_id=other.id, allele_id=allele.id)) with pytest.raises(IntegrityError): session.commit() @@ -1053,7 +1115,14 @@ def test_a_superseded_record_does_not_block_a_new_live_one(self, session, setup_ ).one() live.retire(session) - session.add(MappingRecord(variant_id=variant.id, assay_level="genomic", mapping_api_version="test.1.0")) + session.add( + MappingRecord( + variant_id=variant.id, + score_set_id=variant.score_set_id, + assay_level="genomic", + mapping_api_version="test.1.0", + ) + ) session.commit() still_live = session.scalars( diff --git a/tests/lib/test_allele_measurements.py b/tests/lib/test_allele_measurements.py index 319caea27..167d6ba42 100644 --- a/tests/lib/test_allele_measurements.py +++ b/tests/lib/test_allele_measurements.py @@ -74,6 +74,7 @@ def _allele(session, digest, *, level="cdna", clingen_allele_id=None): def _record(session, variant, *, assay_level="cdna", hgvs_assay_level=None): record = MappingRecord( variant_id=variant.id, + score_set_id=variant.score_set_id, assay_level=assay_level, hgvs_assay_level=hgvs_assay_level, mapping_api_version="test.0.0", @@ -85,7 +86,12 @@ def _record(session, variant, *, assay_level="cdna", hgvs_assay_level=None): def _link(session, record, allele, *, is_authoritative=False): session.add( - MappingRecordAllele(mapping_record_id=record.id, allele_id=allele.id, is_authoritative=is_authoritative) + MappingRecordAllele( + mapping_record_id=record.id, + score_set_id=record.score_set_id, + allele_id=allele.id, + is_authoritative=is_authoritative, + ) ) session.commit() diff --git a/tests/lib/test_alleles.py b/tests/lib/test_alleles.py index 3e3a15055..f01f69378 100644 --- a/tests/lib/test_alleles.py +++ b/tests/lib/test_alleles.py @@ -42,7 +42,11 @@ def _variant(session, score_set, suffix): def _record(session, variant, *, assay_level="genomic", valid_from=None): record = MappingRecord( - variant_id=variant.id, assay_level=assay_level, mapping_api_version="test.0.0", valid_from=valid_from + variant_id=variant.id, + score_set_id=variant.score_set_id, + assay_level=assay_level, + mapping_api_version="test.0.0", + valid_from=valid_from, ) session.add(record) session.commit() @@ -51,7 +55,11 @@ def _record(session, variant, *, assay_level="genomic", valid_from=None): def _link(session, record, allele, *, is_authoritative=False, valid_from=None): link = MappingRecordAllele( - mapping_record_id=record.id, allele_id=allele.id, is_authoritative=is_authoritative, valid_from=valid_from + mapping_record_id=record.id, + score_set_id=record.score_set_id, + allele_id=allele.id, + is_authoritative=is_authoritative, + valid_from=valid_from, ) session.add(link) session.commit() diff --git a/tests/lib/test_cat_vrs.py b/tests/lib/test_cat_vrs.py index 1ee2b9ba0..92d427d64 100644 --- a/tests/lib/test_cat_vrs.py +++ b/tests/lib/test_cat_vrs.py @@ -291,7 +291,11 @@ def _db_variant(session, score_set, suffix): def _db_record(session, variant, *, assay_level, valid_from=None): record = MappingRecord( - variant_id=variant.id, assay_level=assay_level, mapping_api_version="test.0.0", valid_from=valid_from + variant_id=variant.id, + score_set_id=variant.score_set_id, + assay_level=assay_level, + mapping_api_version="test.0.0", + valid_from=valid_from, ) session.add(record) session.commit() @@ -300,7 +304,11 @@ def _db_record(session, variant, *, assay_level, valid_from=None): def _db_link(session, record, allele, *, is_authoritative=False, valid_from=None): link = MappingRecordAllele( - mapping_record_id=record.id, allele_id=allele.id, is_authoritative=is_authoritative, valid_from=valid_from + mapping_record_id=record.id, + score_set_id=record.score_set_id, + allele_id=allele.id, + is_authoritative=is_authoritative, + valid_from=valid_from, ) session.add(link) session.commit() diff --git a/tests/lib/test_score_set_variants.py b/tests/lib/test_score_set_variants.py index 6ac84bb12..934322954 100644 --- a/tests/lib/test_score_set_variants.py +++ b/tests/lib/test_score_set_variants.py @@ -64,6 +64,7 @@ def _allele(session, digest, *, level="cdna", clingen_allele_id=None, hgvs_c=Non def _record(session, variant, *, assay_level="cdna", hgvs_assay_level=None, valid_from=None): record = MappingRecord( variant_id=variant.id, + score_set_id=variant.score_set_id, assay_level=assay_level, hgvs_assay_level=hgvs_assay_level, mapping_api_version="test.0.0", @@ -77,6 +78,7 @@ def _record(session, variant, *, assay_level="cdna", hgvs_assay_level=None, vali def _link(session, record, allele, *, is_authoritative=True, projection_group=None, valid_from=None): link = MappingRecordAllele( mapping_record_id=record.id, + score_set_id=record.score_set_id, allele_id=allele.id, is_authoritative=is_authoritative, projection_group=projection_group, diff --git a/tests/lib/test_variant_annotations_script.py b/tests/lib/test_variant_annotations_script.py index 494bce22b..1a654bc3e 100644 --- a/tests/lib/test_variant_annotations_script.py +++ b/tests/lib/test_variant_annotations_script.py @@ -26,10 +26,16 @@ def _variant_mapped_to_allele(session, score_set, allele): session.add(variant) session.commit() - record = MappingRecord(variant_id=variant.id, assay_level="genomic", mapping_api_version="test.0.0") + record = MappingRecord( + variant_id=variant.id, score_set_id=variant.score_set_id, assay_level="genomic", mapping_api_version="test.0.0" + ) session.add(record) session.commit() - session.add(MappingRecordAllele(mapping_record_id=record.id, allele_id=allele.id, is_authoritative=True)) + session.add( + MappingRecordAllele( + mapping_record_id=record.id, score_set_id=record.score_set_id, allele_id=allele.id, is_authoritative=True + ) + ) session.commit() return variant diff --git a/tests/lib/test_variant_detail.py b/tests/lib/test_variant_detail.py index 8950dd6ef..3d6700ef4 100644 --- a/tests/lib/test_variant_detail.py +++ b/tests/lib/test_variant_detail.py @@ -84,6 +84,7 @@ def _allele(session, digest, *, level="cdna", clingen_allele_id=None, hgvs_g=Non def _record(session, variant, *, assay_level="cdna", hgvs_assay_level=None): record = MappingRecord( variant_id=variant.id, + score_set_id=variant.score_set_id, assay_level=assay_level, hgvs_assay_level=hgvs_assay_level, mapping_api_version="test.0.0", @@ -96,6 +97,7 @@ def _record(session, variant, *, assay_level="cdna", hgvs_assay_level=None): def _link(session, record, allele, *, is_authoritative=False, projection_group=None): link = MappingRecordAllele( mapping_record_id=record.id, + score_set_id=record.score_set_id, allele_id=allele.id, is_authoritative=is_authoritative, projection_group=projection_group, diff --git a/tests/routers/test_clingen_allele.py b/tests/routers/test_clingen_allele.py index 7ac06e709..4157c27db 100644 --- a/tests/routers/test_clingen_allele.py +++ b/tests/routers/test_clingen_allele.py @@ -37,6 +37,7 @@ def _seed_cdna_measurement(session, variant_urn, *, clingen_allele_id="CA123", v variant = session.scalar(select(VariantDbModel).where(VariantDbModel.urn == variant_urn)) record = MappingRecord( variant_id=variant.id, + score_set_id=variant.score_set_id, assay_level="cdna", hgvs_assay_level="NM_000546.6:c.1216G>A", mapping_api_version="test.0.0", @@ -47,7 +48,11 @@ def _seed_cdna_measurement(session, variant_urn, *, clingen_allele_id="CA123", v measured = Allele(vrs_digest=vrs_digest, level="cdna", clingen_allele_id=clingen_allele_id) session.add(measured) session.commit() - session.add(MappingRecordAllele(mapping_record_id=record.id, allele_id=measured.id, is_authoritative=True)) + session.add( + MappingRecordAllele( + mapping_record_id=record.id, score_set_id=record.score_set_id, allele_id=measured.id, is_authoritative=True + ) + ) session.commit() @@ -122,14 +127,22 @@ def test_nucleotide_siblings_shown(client, session, data_provider, data_files, s # Two coding measurements encoding the same protein consequence (PA9), each carrying its own CA. for suffix, caid, digest in ((1, "CA111", "cdna-1"), (2, "CA222", "cdna-2")): variant = session.scalar(select(VariantDbModel).where(VariantDbModel.urn == f"{score_set['urn']}#{suffix}")) - record = MappingRecord(variant_id=variant.id, assay_level="cdna", mapping_api_version="test.0.0") + record = MappingRecord( + variant_id=variant.id, score_set_id=variant.score_set_id, assay_level="cdna", mapping_api_version="test.0.0" + ) session.add(record) session.commit() nt = Allele(vrs_digest=digest, level="cdna", clingen_allele_id=caid) session.add(nt) session.commit() - session.add(MappingRecordAllele(mapping_record_id=record.id, allele_id=nt.id, is_authoritative=True)) - session.add(MappingRecordAllele(mapping_record_id=record.id, allele_id=prot.id)) + session.add( + MappingRecordAllele( + mapping_record_id=record.id, score_set_id=record.score_set_id, allele_id=nt.id, is_authoritative=True + ) + ) + session.add( + MappingRecordAllele(mapping_record_id=record.id, score_set_id=record.score_set_id, allele_id=prot.id) + ) session.commit() response = client.get("/api/v1/clingen-alleles/CA111/measurements") @@ -153,14 +166,22 @@ def test_direct_measurements_sort_before_related(client, session, data_provider, for suffix, caid, digest in ((1, "CA111", "cdna-1"), (2, "CA222", "cdna-2")): variant = session.scalar(select(VariantDbModel).where(VariantDbModel.urn == f"{score_set['urn']}#{suffix}")) - record = MappingRecord(variant_id=variant.id, assay_level="cdna", mapping_api_version="test.0.0") + record = MappingRecord( + variant_id=variant.id, score_set_id=variant.score_set_id, assay_level="cdna", mapping_api_version="test.0.0" + ) session.add(record) session.commit() nt = Allele(vrs_digest=digest, level="cdna", clingen_allele_id=caid) session.add(nt) session.commit() - session.add(MappingRecordAllele(mapping_record_id=record.id, allele_id=nt.id, is_authoritative=True)) - session.add(MappingRecordAllele(mapping_record_id=record.id, allele_id=prot.id)) + session.add( + MappingRecordAllele( + mapping_record_id=record.id, score_set_id=record.score_set_id, allele_id=nt.id, is_authoritative=True + ) + ) + session.add( + MappingRecordAllele(mapping_record_id=record.id, score_set_id=record.score_set_id, allele_id=prot.id) + ) session.commit() response = client.get("/api/v1/clingen-alleles/CA111/measurements") diff --git a/tests/routers/test_variant.py b/tests/routers/test_variant.py index a650e3d45..121c1b4bb 100644 --- a/tests/routers/test_variant.py +++ b/tests/routers/test_variant.py @@ -72,6 +72,7 @@ def _seed_mapping(session, variant_urn): variant = session.scalar(select(VariantDbModel).where(VariantDbModel.urn == variant_urn)) record = MappingRecord( variant_id=variant.id, + score_set_id=variant.score_set_id, assay_level="cdna", hgvs_assay_level="NM_000546.6:c.1216G>A", mapping_api_version="test.0.0", @@ -103,14 +104,31 @@ def _seed_mapping(session, variant_urn): # The measured cdna link and its genomic projection share a projection_group; the protein # apex is in no pair (group None); the cousin sits in its own projection group. MappingRecordAllele( - mapping_record_id=record.id, allele_id=measured.id, is_authoritative=True, projection_group=0 + mapping_record_id=record.id, + score_set_id=record.score_set_id, + allele_id=measured.id, + is_authoritative=True, + projection_group=0, + ), + MappingRecordAllele( + mapping_record_id=record.id, + score_set_id=record.score_set_id, + allele_id=genomic.id, + is_authoritative=False, + projection_group=0, ), MappingRecordAllele( - mapping_record_id=record.id, allele_id=genomic.id, is_authoritative=False, projection_group=0 + mapping_record_id=record.id, + score_set_id=record.score_set_id, + allele_id=protein.id, + is_authoritative=False, ), - MappingRecordAllele(mapping_record_id=record.id, allele_id=protein.id, is_authoritative=False), MappingRecordAllele( - mapping_record_id=record.id, allele_id=cousin.id, is_authoritative=False, projection_group=1 + mapping_record_id=record.id, + score_set_id=record.score_set_id, + allele_id=cousin.id, + is_authoritative=False, + projection_group=1, ), VepAlleleConsequence( allele_id=measured.id, diff --git a/tests/worker/jobs/conftest.py b/tests/worker/jobs/conftest.py index 873d09ff0..6f90825d6 100644 --- a/tests/worker/jobs/conftest.py +++ b/tests/worker/jobs/conftest.py @@ -311,6 +311,7 @@ def setup_sample_alleles_with_caid(session, with_populated_domain_data, sample_l mapping_record = MappingRecord( variant_id=variant.id, + score_set_id=variant.score_set_id, assay_level="genomic", mapping_api_version="pytest.0.0", ) @@ -320,6 +321,7 @@ def setup_sample_alleles_with_caid(session, with_populated_domain_data, sample_l session.add( MappingRecordAllele( mapping_record_id=mapping_record.id, + score_set_id=mapping_record.score_set_id, allele_id=allele.id, is_authoritative=True, ) @@ -358,6 +360,7 @@ def setup_rt_derived_allele_with_caid(session, setup_sample_alleles_with_caid): session.add( MappingRecordAllele( mapping_record_id=mapping_record.id, + score_set_id=mapping_record.score_set_id, allele_id=rt_allele.id, is_authoritative=False, ) @@ -1176,6 +1179,7 @@ def setup_sample_alleles_for_vep(session, with_populated_domain_data, mock_worke mapping_record = MappingRecord( variant_id=variant.id, + score_set_id=variant.score_set_id, assay_level="cdna", mapping_api_version="pytest.0.0", ) @@ -1185,6 +1189,7 @@ def setup_sample_alleles_for_vep(session, with_populated_domain_data, mock_worke session.add( MappingRecordAllele( mapping_record_id=mapping_record.id, + score_set_id=mapping_record.score_set_id, allele_id=allele.id, is_authoritative=True, ) @@ -1221,6 +1226,7 @@ def setup_sample_protein_allele_for_vep(session, with_populated_domain_data, moc mapping_record = MappingRecord( variant_id=variant.id, + score_set_id=variant.score_set_id, assay_level="protein", mapping_api_version="pytest.0.0", ) @@ -1230,6 +1236,7 @@ def setup_sample_protein_allele_for_vep(session, with_populated_domain_data, moc session.add( MappingRecordAllele( mapping_record_id=mapping_record.id, + score_set_id=mapping_record.score_set_id, allele_id=allele.id, is_authoritative=True, ) @@ -1262,6 +1269,7 @@ def setup_rt_derived_allele_for_vep(session, setup_sample_alleles_for_vep): session.add( MappingRecordAllele( mapping_record_id=mapping_record.id, + score_set_id=mapping_record.score_set_id, allele_id=rt_allele.id, is_authoritative=False, ) diff --git a/tests/worker/jobs/external_services/test_clingen.py b/tests/worker/jobs/external_services/test_clingen.py index 5431227d6..0281545ae 100644 --- a/tests/worker/jobs/external_services/test_clingen.py +++ b/tests/worker/jobs/external_services/test_clingen.py @@ -275,6 +275,7 @@ async def test_submit_score_set_mappings_to_car_event_per_allele( session.add( MappingRecordAllele( mapping_record_id=a_link.mapping_record_id, + score_set_id=a_link.score_set_id, allele_id=derived_allele.id, is_authoritative=False, ) @@ -939,6 +940,7 @@ async def _add_derived_alleles_with_distinct_hgvs(self, session, count): session.add( MappingRecordAllele( mapping_record_id=a_link.mapping_record_id, + score_set_id=a_link.score_set_id, allele_id=derived.id, is_authoritative=False, ) diff --git a/tests/worker/jobs/external_services/test_clingen_cache.py b/tests/worker/jobs/external_services/test_clingen_cache.py index 6d95b917a..625bb0c90 100644 --- a/tests/worker/jobs/external_services/test_clingen_cache.py +++ b/tests/worker/jobs/external_services/test_clingen_cache.py @@ -47,6 +47,7 @@ def _make_allele_with_caid(session, score_set_id: int, urn_suffix: str, caid: st mapping_record = MappingRecord( variant_id=variant.id, + score_set_id=variant.score_set_id, assay_level="genomic", mapping_api_version="1.0.0", mapped_date=date.today(), @@ -56,6 +57,7 @@ def _make_allele_with_caid(session, score_set_id: int, urn_suffix: str, caid: st link = MappingRecordAllele( mapping_record_id=mapping_record.id, + score_set_id=mapping_record.score_set_id, allele_id=allele.id, is_authoritative=True, ) @@ -224,6 +226,7 @@ async def test_only_warms_current_mapped_variants( old_mapping_record = MappingRecord( variant_id=variant.id, + score_set_id=variant.score_set_id, assay_level="genomic", mapping_api_version="0.9.0", mapped_date=date(2023, 1, 1), @@ -235,6 +238,7 @@ async def test_only_warms_current_mapped_variants( old_link = MappingRecordAllele( mapping_record_id=old_mapping_record.id, + score_set_id=old_mapping_record.score_set_id, allele_id=old_allele.id, is_authoritative=True, ) @@ -254,6 +258,7 @@ async def test_only_warms_current_mapped_variants( current_mapping_record = MappingRecord( variant_id=variant.id, + score_set_id=variant.score_set_id, assay_level="genomic", mapping_api_version="1.0.0", mapped_date=date.today(), @@ -263,6 +268,7 @@ async def test_only_warms_current_mapped_variants( current_link = MappingRecordAllele( mapping_record_id=current_mapping_record.id, + score_set_id=current_mapping_record.score_set_id, allele_id=current_allele.id, is_authoritative=True, ) diff --git a/tests/worker/jobs/variant_processing/test_mapping.py b/tests/worker/jobs/variant_processing/test_mapping.py index 659cfed84..88d4af978 100644 --- a/tests/worker/jobs/variant_processing/test_mapping.py +++ b/tests/worker/jobs/variant_processing/test_mapping.py @@ -1014,6 +1014,7 @@ async def dummy_mapping_job(): session.commit() mapping_record = MappingRecord( variant_id=variant.id, + score_set_id=variant.score_set_id, assay_level=SequenceLevel.genomic, mapped_date=date(2023, 1, 1), mapping_api_version="v1.0.0", @@ -1026,7 +1027,10 @@ async def dummy_mapping_job(): session.add(prior_allele) session.commit() prior_link = MappingRecordAllele( - mapping_record_id=mapping_record.id, allele_id=prior_allele.id, is_authoritative=True + mapping_record_id=mapping_record.id, + score_set_id=mapping_record.score_set_id, + allele_id=prior_allele.id, + is_authoritative=True, ) session.add(prior_link) session.commit() @@ -1541,6 +1545,7 @@ async def test_map_variants_for_score_set_updates_current_mapped_variants( for variant in variants: mapping_record = MappingRecord( variant_id=variant.id, + score_set_id=variant.score_set_id, assay_level=SequenceLevel.genomic, mapped_date=date(2023, 1, 1), mapping_api_version="v1.0.0", From 7acf76199af7e157bf8582360f406ea5bb292dde Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Mon, 28 Sep 2026 16:53:29 -0700 Subject: [PATCH 174/252] fix(tests): submit a versioned RefSeq accession in the UniProt network test UniProt ID mapping now returns an empty result for unversioned RefSeq accessions, so the polling job exhausted its retries waiting on NM_003345. --- .../jobs/external_services/network/test_uniprot.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/worker/jobs/external_services/network/test_uniprot.py b/tests/worker/jobs/external_services/network/test_uniprot.py index 506eb20f0..742344c62 100644 --- a/tests/worker/jobs/external_services/network/test_uniprot.py +++ b/tests/worker/jobs/external_services/network/test_uniprot.py @@ -5,10 +5,12 @@ pytest.importorskip("arq") from mavedb.models.enums.job_pipeline import JobStatus, PipelineStatus -from tests.helpers.constants import TEST_REFSEQ_IDENTIFIER pytestmark = pytest.mark.usefixtures("patch_db_session_ctxmgr") +# UniProt ID mapping returns no results for unversioned RefSeq accessions. +VERSIONED_REFSEQ_ACCESSION = "NM_003345.5" + @pytest.mark.asyncio @pytest.mark.integration @@ -31,7 +33,7 @@ async def test_uniprot_mapping_jobs_e2e( # Add an accession to the target gene's post mapped metadata target_gene = sample_score_set.target_genes[0] - target_gene.post_mapped_metadata = {"protein": {"sequence_accessions": [TEST_REFSEQ_IDENTIFIER]}} + target_gene.post_mapped_metadata = {"protein": {"sequence_accessions": [VERSIONED_REFSEQ_ACCESSION]}} session.commit() await arq_redis.enqueue_job( @@ -45,12 +47,12 @@ async def test_uniprot_mapping_jobs_e2e( submitted_jobs = sample_submit_uniprot_mapping_jobs_run_in_pipeline.metadata_["submitted_jobs"] assert "1" in submitted_jobs assert submitted_jobs["1"]["job_id"] is not None - assert submitted_jobs["1"]["accession"] == TEST_REFSEQ_IDENTIFIER + assert submitted_jobs["1"]["accession"] == VERSIONED_REFSEQ_ACCESSION # Verify that polling job params have been updated correctly session.refresh(sample_polling_job_for_submission_run_in_pipeline) assert sample_polling_job_for_submission_run_in_pipeline.job_params["mapping_jobs"] == { - "1": {"job_id": submitted_jobs["1"]["job_id"], "accession": TEST_REFSEQ_IDENTIFIER} + "1": {"job_id": submitted_jobs["1"]["job_id"], "accession": VERSIONED_REFSEQ_ACCESSION} } # Verify that the submission job was completed successfully From e96248f052528ddf4b24f3e20cb0aef4d1098c83 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 29 Sep 2026 13:05:14 -0700 Subject: [PATCH 175/252] perf(db): index variants by score set and variant number Score set CSV queries order a score set's variants by the number after '#' in their URN. With no index on that expression, every request sorted the whole score set, so a page deep into a large one (e.g. offset 600,000 of urn:mavedb:00000053-a-1) took minutes. - Add Variant.variant_number as the single definition of that expression and order the CSV fetch by it. - Index (scoreset_id, variant_number, id), built concurrently. - The index rejects a variant URN that does not end in '#', so the mapping tests now use realistic URNs. --- ...71d14d_index_variants_by_variant_number.py | 40 +++++++++++++ src/mavedb/lib/csv/fetch.py | 6 +- src/mavedb/models/variant.py | 20 ++++++- tests/lib/csv/test_fetch.py | 57 +++++++++++++++++++ tests/models/__init__.py | 0 tests/models/test_variant.py | 12 ++++ .../jobs/variant_processing/test_mapping.py | 10 ++-- 7 files changed, 134 insertions(+), 11 deletions(-) create mode 100644 alembic/versions/f6752571d14d_index_variants_by_variant_number.py create mode 100644 tests/lib/csv/test_fetch.py create mode 100644 tests/models/__init__.py create mode 100644 tests/models/test_variant.py diff --git a/alembic/versions/f6752571d14d_index_variants_by_variant_number.py b/alembic/versions/f6752571d14d_index_variants_by_variant_number.py new file mode 100644 index 000000000..04b064f1b --- /dev/null +++ b/alembic/versions/f6752571d14d_index_variants_by_variant_number.py @@ -0,0 +1,40 @@ +"""Index score set variants by variant number + +Score set CSV queries order rows by the number after '#' in the variant URN (``Variant.variant_number``). Without an +index on that expression, every request sorts the whole score set, and a page deep into a large score set takes +minutes. + +The index evaluates the expression for every row, so every non-null variant URN must end in '#'; an insert +that breaks that fails. No existing URN broke it when this migration was written. + +Revision ID: f6752571d14d +Revises: c4b18d0f7a92 +Create Date: 2026-09-29 00:00:00.000000 + +""" + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "f6752571d14d" +down_revision = "c4b18d0f7a92" +branch_labels = None +depends_on = None + +INDEX_NAME = "ix_variants_scoreset_id_variant_number" + + +def upgrade(): + # Built concurrently so variant writes continue during the build, which must run outside a transaction. A failed + # concurrent build leaves an invalid index behind, so drop any leftover before building. + with op.get_context().autocommit_block(): + op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {INDEX_NAME}") + op.execute( + f"CREATE INDEX CONCURRENTLY {INDEX_NAME} " + "ON variants (scoreset_id, (CAST(split_part(urn, '#', 2) AS INTEGER)), id)" + ) + + +def downgrade(): + with op.get_context().autocommit_block(): + op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {INDEX_NAME}") diff --git a/src/mavedb/lib/csv/fetch.py b/src/mavedb/lib/csv/fetch.py index 9244db15c..f9b9434ae 100644 --- a/src/mavedb/lib/csv/fetch.py +++ b/src/mavedb/lib/csv/fetch.py @@ -7,7 +7,7 @@ from dataclasses import dataclass from typing import Any, Optional, Sequence -from sqlalchemy import Integer, and_, cast, func, select +from sqlalchemy import and_, select from sqlalchemy.orm import Session, aliased, selectinload from mavedb.lib.csv.namespaces import CLINVAR_DB_NAME @@ -73,9 +73,7 @@ def fetch_variant_csv_data( query = select(*select_columns) if score_set is not None: - query = query.where(Variant.score_set_id == score_set.id).order_by( - cast(func.split_part(Variant.urn, "#", 2), Integer) - ) + query = query.where(Variant.score_set_id == score_set.id).order_by(Variant.variant_number) else: query = query.where(Variant.id.in_(variant_ids or [])) diff --git a/src/mavedb/models/variant.py b/src/mavedb/models/variant.py index 59b6e729c..dd2c7c8be 100644 --- a/src/mavedb/models/variant.py +++ b/src/mavedb/models/variant.py @@ -1,8 +1,9 @@ from datetime import date -from typing import TYPE_CHECKING, List +from typing import TYPE_CHECKING, List, Optional -from sqlalchemy import Column, Date, ForeignKey, Integer, String +from sqlalchemy import Column, ColumnElement, Date, ForeignKey, Index, Integer, String, cast, func from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.orm import Mapped, relationship from mavedb.db.base import Base @@ -37,3 +38,18 @@ class Variant(Base): # Bidirectional relationship with ScoreCalibrationFunctionalClassification is left # purposefully undefined for performance reasons. + + @hybrid_property + def variant_number(self) -> Optional[int]: + """The integer after '#' in the URN, which orders a score set's variants.""" + return int(self.urn.split("#")[1]) if self.urn else None + + @variant_number.inplace.expression + @classmethod + def _variant_number_expression(cls) -> ColumnElement[int]: + # Order by this expression rather than restating it: the index below serves only an identical one. + return cast(func.split_part(cls.urn, "#", 2), Integer) + + +# Serves ordering a score set's variants by number, so deep pages of a large score set skip the full sort. +Index("ix_variants_scoreset_id_variant_number", Variant.score_set_id, Variant.variant_number, Variant.id) diff --git a/tests/lib/csv/test_fetch.py b/tests/lib/csv/test_fetch.py new file mode 100644 index 000000000..9cd089ec5 --- /dev/null +++ b/tests/lib/csv/test_fetch.py @@ -0,0 +1,57 @@ +# ruff: noqa: E402 + +from types import SimpleNamespace + +import pytest + +pytest.importorskip("psycopg2") + +from sqlalchemy import event, text + +from mavedb.lib.csv.columns import plan_csv_columns +from mavedb.lib.csv.fetch import fetch_variant_csv_data + +VARIANT_NUMBER_INDEX = "ix_variants_scoreset_id_variant_number" + + +def _captured_variant_query(session, namespaces, start=None, limit=None): + """Run the score set CSV fetch and return the statement and parameters it sent for the variant rows.""" + captured = [] + engine = session.get_bind() + + def capture(_conn, _cursor, statement, parameters, _context, _executemany): + if "FROM variants" in statement and not captured: + captured.append((statement, parameters)) + + event.listen(engine, "before_cursor_execute", capture) + try: + plan = plan_csv_columns({"score_columns": ["score"]}, namespaces) + fetch_variant_csv_data( + session, + plan.namespaced_columns, + plan.clinvar_namespaces, + score_set=SimpleNamespace(id=1), + start=start, + limit=limit, + ) + finally: + event.remove(engine, "before_cursor_execute", capture) + + return captured[0] + + +@pytest.mark.unit +class TestVariantNumberOrdering: + @pytest.mark.parametrize("namespaces", [["scores"], ["scores", "mavedb"]], ids=["scores", "with_mappings"]) + def test_score_set_query_is_served_by_the_variant_number_index(self, session, namespaces): + statement, parameters = _captured_variant_query(session, namespaces, start=250000, limit=50000) + + # Rule out the alternatives so the plan shows whether the index can serve the ORDER BY at all. + session.execute(text("SET LOCAL enable_seqscan = off")) + session.execute(text("SET LOCAL enable_sort = off")) + cursor = session.connection().connection.cursor() + cursor.execute("EXPLAIN " + statement, parameters) + plan = "\n".join(row[0] for row in cursor.fetchall()) + + assert VARIANT_NUMBER_INDEX in plan + assert "Sort" not in plan diff --git a/tests/models/__init__.py b/tests/models/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/models/test_variant.py b/tests/models/test_variant.py new file mode 100644 index 000000000..81da770e5 --- /dev/null +++ b/tests/models/test_variant.py @@ -0,0 +1,12 @@ +import pytest + +from mavedb.models.variant import Variant + + +@pytest.mark.unit +@pytest.mark.parametrize( + "urn, expected", + [("urn:mavedb:00000053-a-1#648022", 648022), ("tmp:0c8d1a2e-5f3b-4c6d-9e7f-1a2b3c4d5e6f#1", 1), (None, None)], +) +def test_variant_number_is_the_integer_after_the_hash(urn, expected): + assert Variant(urn=urn).variant_number == expected diff --git a/tests/worker/jobs/variant_processing/test_mapping.py b/tests/worker/jobs/variant_processing/test_mapping.py index 5c513d7e8..725c7f5cb 100644 --- a/tests/worker/jobs/variant_processing/test_mapping.py +++ b/tests/worker/jobs/variant_processing/test_mapping.py @@ -663,14 +663,14 @@ async def dummy_mapping_job(): hgvs_nt="NM_000000.1:c.1A>G", hgvs_pro="NP_000000.1:p.Met1Val", data={}, - urn="variant:1", + urn="urn:mavedb:00000001-a-1#1", ) variant2 = Variant( score_set_id=sample_score_set.id, hgvs_nt="NM_000000.1:c.2G>T", hgvs_pro="NP_000000.1:p.Val2Leu", data={}, - urn="variant:2", + urn="urn:mavedb:00000001-a-1#2", ) session.add_all([variant1, variant2]) session.commit() @@ -759,14 +759,14 @@ async def dummy_mapping_job(): hgvs_nt="NM_000000.1:c.1A>G", hgvs_pro="NP_000000.1:p.Met1Val", data={}, - urn="variant:1", + urn="urn:mavedb:00000001-a-1#1", ) variant2 = Variant( score_set_id=sample_score_set.id, hgvs_nt="NM_000000.1:c.2G>T", hgvs_pro="NP_000000.1:p.Val2Leu", data={}, - urn="variant:2", + urn="urn:mavedb:00000001-a-1#2", ) session.add_all([variant1, variant2]) session.commit() @@ -796,7 +796,7 @@ async def dummy_mapping_job(): # Verify that both variants have post-mapped data. I'm comfortable assuming the # data is correct given our layer permutation tests above. - for urn in ["variant:1", "variant:2"]: + for urn in ["urn:mavedb:00000001-a-1#1", "urn:mavedb:00000001-a-1#2"]: mapped_variant = session.query(MappedVariant).filter(MappedVariant.variant.has(urn=urn)).one_or_none() assert mapped_variant is not None assert mapped_variant.post_mapped != {} From a31816460f1a360aa796da6a5bde2288a6b2f41b Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 29 Sep 2026 14:25:57 -0700 Subject: [PATCH 176/252] fix(db): chain the variant-number index after #874 and follow the migration rules - Chain after 15c40c367731 (#874), which also follows c4b18d0f7a92, so release-2026.3.0 keeps a single migration head. - Create the index with a plain CREATE INDEX IF NOT EXISTS, per #877's rules. The docstring gives the CONCURRENTLY build to run by hand before deploying. - Name it ix_variants_scoreset_number, as #770 does. --- ...71d14d_index_variants_by_variant_number.py | 27 +++++++++---------- src/mavedb/models/variant.py | 2 +- tests/lib/csv/test_fetch.py | 2 +- 3 files changed, 15 insertions(+), 16 deletions(-) diff --git a/alembic/versions/f6752571d14d_index_variants_by_variant_number.py b/alembic/versions/f6752571d14d_index_variants_by_variant_number.py index 04b064f1b..bc8aba1b2 100644 --- a/alembic/versions/f6752571d14d_index_variants_by_variant_number.py +++ b/alembic/versions/f6752571d14d_index_variants_by_variant_number.py @@ -7,8 +7,14 @@ The index evaluates the expression for every row, so every non-null variant URN must end in '#'; an insert that breaks that fails. No existing URN broke it when this migration was written. +Building it takes a lock that blocks writes to ``variants``, so in a populated database build it by hand first, which +makes this migration a no-op: + + CREATE INDEX CONCURRENTLY IF NOT EXISTS ix_variants_scoreset_number + ON variants (scoreset_id, (CAST(split_part(urn, '#', 2) AS INTEGER)), id); + Revision ID: f6752571d14d -Revises: c4b18d0f7a92 +Revises: 15c40c367731 Create Date: 2026-09-29 00:00:00.000000 """ @@ -17,24 +23,17 @@ # revision identifiers, used by Alembic. revision = "f6752571d14d" -down_revision = "c4b18d0f7a92" +down_revision = "15c40c367731" branch_labels = None depends_on = None -INDEX_NAME = "ix_variants_scoreset_id_variant_number" - def upgrade(): - # Built concurrently so variant writes continue during the build, which must run outside a transaction. A failed - # concurrent build leaves an invalid index behind, so drop any leftover before building. - with op.get_context().autocommit_block(): - op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {INDEX_NAME}") - op.execute( - f"CREATE INDEX CONCURRENTLY {INDEX_NAME} " - "ON variants (scoreset_id, (CAST(split_part(urn, '#', 2) AS INTEGER)), id)" - ) + op.execute( + "CREATE INDEX IF NOT EXISTS ix_variants_scoreset_number " + "ON variants (scoreset_id, (CAST(split_part(urn, '#', 2) AS INTEGER)), id)" + ) def downgrade(): - with op.get_context().autocommit_block(): - op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {INDEX_NAME}") + op.execute("DROP INDEX IF EXISTS ix_variants_scoreset_number") diff --git a/src/mavedb/models/variant.py b/src/mavedb/models/variant.py index dd2c7c8be..7df2e54b6 100644 --- a/src/mavedb/models/variant.py +++ b/src/mavedb/models/variant.py @@ -52,4 +52,4 @@ def _variant_number_expression(cls) -> ColumnElement[int]: # Serves ordering a score set's variants by number, so deep pages of a large score set skip the full sort. -Index("ix_variants_scoreset_id_variant_number", Variant.score_set_id, Variant.variant_number, Variant.id) +Index("ix_variants_scoreset_number", Variant.score_set_id, Variant.variant_number, Variant.id) diff --git a/tests/lib/csv/test_fetch.py b/tests/lib/csv/test_fetch.py index 9cd089ec5..84b8a1a84 100644 --- a/tests/lib/csv/test_fetch.py +++ b/tests/lib/csv/test_fetch.py @@ -11,7 +11,7 @@ from mavedb.lib.csv.columns import plan_csv_columns from mavedb.lib.csv.fetch import fetch_variant_csv_data -VARIANT_NUMBER_INDEX = "ix_variants_scoreset_id_variant_number" +VARIANT_NUMBER_INDEX = "ix_variants_scoreset_number" def _captured_variant_query(session, namespaces, start=None, limit=None): From f9b0a2780f6fb350d1174399a6085679d3c435d6 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 29 Sep 2026 15:37:52 -0700 Subject: [PATCH 177/252] fix(db): let the variant-number index tolerate URNs without a number The index evaluates its expression for every row, so the strict cast failed any insert whose URN has no '#', which many test fixtures use. NULLIF gives those variants a null number, so they sort last instead. --- .../f6752571d14d_index_variants_by_variant_number.py | 8 ++++---- src/mavedb/models/variant.py | 10 ++++++---- tests/models/test_variant.py | 9 +++++++-- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/alembic/versions/f6752571d14d_index_variants_by_variant_number.py b/alembic/versions/f6752571d14d_index_variants_by_variant_number.py index bc8aba1b2..83f6ef609 100644 --- a/alembic/versions/f6752571d14d_index_variants_by_variant_number.py +++ b/alembic/versions/f6752571d14d_index_variants_by_variant_number.py @@ -4,14 +4,14 @@ index on that expression, every request sorts the whole score set, and a page deep into a large score set takes minutes. -The index evaluates the expression for every row, so every non-null variant URN must end in '#'; an insert -that breaks that fails. No existing URN broke it when this migration was written. +The index evaluates the expression for every row. A URN with no '#' gets a null number; one whose text after '#' isn't +an integer fails the insert. No existing URN did when this migration was written. Building it takes a lock that blocks writes to ``variants``, so in a populated database build it by hand first, which makes this migration a no-op: CREATE INDEX CONCURRENTLY IF NOT EXISTS ix_variants_scoreset_number - ON variants (scoreset_id, (CAST(split_part(urn, '#', 2) AS INTEGER)), id); + ON variants (scoreset_id, (CAST(NULLIF(split_part(urn, '#', 2), '') AS INTEGER)), id); Revision ID: f6752571d14d Revises: 15c40c367731 @@ -31,7 +31,7 @@ def upgrade(): op.execute( "CREATE INDEX IF NOT EXISTS ix_variants_scoreset_number " - "ON variants (scoreset_id, (CAST(split_part(urn, '#', 2) AS INTEGER)), id)" + "ON variants (scoreset_id, (CAST(NULLIF(split_part(urn, '#', 2), '') AS INTEGER)), id)" ) diff --git a/src/mavedb/models/variant.py b/src/mavedb/models/variant.py index 7df2e54b6..5b0e6848d 100644 --- a/src/mavedb/models/variant.py +++ b/src/mavedb/models/variant.py @@ -41,14 +41,16 @@ class Variant(Base): @hybrid_property def variant_number(self) -> Optional[int]: - """The integer after '#' in the URN, which orders a score set's variants.""" - return int(self.urn.split("#")[1]) if self.urn else None + """The integer after '#' in the URN, which orders a score set's variants. None without a '#'.""" + parts = (self.urn or "").split("#") + return int(parts[1]) if len(parts) > 1 and parts[1] else None @variant_number.inplace.expression @classmethod def _variant_number_expression(cls) -> ColumnElement[int]: - # Order by this expression rather than restating it: the index below serves only an identical one. - return cast(func.split_part(cls.urn, "#", 2), Integer) + # Order by this expression rather than restating it: the index below serves only an identical one. NULLIF + # keeps a URN with no '#' from failing the cast, so such variants sort last instead of breaking inserts. + return cast(func.nullif(func.split_part(cls.urn, "#", 2), ""), Integer) # Serves ordering a score set's variants by number, so deep pages of a large score set skip the full sort. diff --git a/tests/models/test_variant.py b/tests/models/test_variant.py index 81da770e5..ed318d7c2 100644 --- a/tests/models/test_variant.py +++ b/tests/models/test_variant.py @@ -6,7 +6,12 @@ @pytest.mark.unit @pytest.mark.parametrize( "urn, expected", - [("urn:mavedb:00000053-a-1#648022", 648022), ("tmp:0c8d1a2e-5f3b-4c6d-9e7f-1a2b3c4d5e6f#1", 1), (None, None)], + [ + ("urn:mavedb:00000053-a-1#648022", 648022), + ("tmp:0c8d1a2e-5f3b-4c6d-9e7f-1a2b3c4d5e6f#1", 1), + ("urn:mavedb:variant-1", None), + (None, None), + ], ) -def test_variant_number_is_the_integer_after_the_hash(urn, expected): +def test_variant_number_is_the_integer_after_the_hash_or_none(urn, expected): assert Variant(urn=urn).variant_number == expected From 4bb07de39b95ca9cc8e3003eb334a523f830a24b Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 29 Sep 2026 15:46:53 -0700 Subject: [PATCH 178/252] Revert "fix(db): let the variant-number index tolerate URNs without a number" This reverts commit f9b0a2780f6fb350d1174399a6085679d3c435d6. --- .../f6752571d14d_index_variants_by_variant_number.py | 8 ++++---- src/mavedb/models/variant.py | 10 ++++------ tests/models/test_variant.py | 9 ++------- 3 files changed, 10 insertions(+), 17 deletions(-) diff --git a/alembic/versions/f6752571d14d_index_variants_by_variant_number.py b/alembic/versions/f6752571d14d_index_variants_by_variant_number.py index 83f6ef609..bc8aba1b2 100644 --- a/alembic/versions/f6752571d14d_index_variants_by_variant_number.py +++ b/alembic/versions/f6752571d14d_index_variants_by_variant_number.py @@ -4,14 +4,14 @@ index on that expression, every request sorts the whole score set, and a page deep into a large score set takes minutes. -The index evaluates the expression for every row. A URN with no '#' gets a null number; one whose text after '#' isn't -an integer fails the insert. No existing URN did when this migration was written. +The index evaluates the expression for every row, so every non-null variant URN must end in '#'; an insert +that breaks that fails. No existing URN broke it when this migration was written. Building it takes a lock that blocks writes to ``variants``, so in a populated database build it by hand first, which makes this migration a no-op: CREATE INDEX CONCURRENTLY IF NOT EXISTS ix_variants_scoreset_number - ON variants (scoreset_id, (CAST(NULLIF(split_part(urn, '#', 2), '') AS INTEGER)), id); + ON variants (scoreset_id, (CAST(split_part(urn, '#', 2) AS INTEGER)), id); Revision ID: f6752571d14d Revises: 15c40c367731 @@ -31,7 +31,7 @@ def upgrade(): op.execute( "CREATE INDEX IF NOT EXISTS ix_variants_scoreset_number " - "ON variants (scoreset_id, (CAST(NULLIF(split_part(urn, '#', 2), '') AS INTEGER)), id)" + "ON variants (scoreset_id, (CAST(split_part(urn, '#', 2) AS INTEGER)), id)" ) diff --git a/src/mavedb/models/variant.py b/src/mavedb/models/variant.py index 5b0e6848d..7df2e54b6 100644 --- a/src/mavedb/models/variant.py +++ b/src/mavedb/models/variant.py @@ -41,16 +41,14 @@ class Variant(Base): @hybrid_property def variant_number(self) -> Optional[int]: - """The integer after '#' in the URN, which orders a score set's variants. None without a '#'.""" - parts = (self.urn or "").split("#") - return int(parts[1]) if len(parts) > 1 and parts[1] else None + """The integer after '#' in the URN, which orders a score set's variants.""" + return int(self.urn.split("#")[1]) if self.urn else None @variant_number.inplace.expression @classmethod def _variant_number_expression(cls) -> ColumnElement[int]: - # Order by this expression rather than restating it: the index below serves only an identical one. NULLIF - # keeps a URN with no '#' from failing the cast, so such variants sort last instead of breaking inserts. - return cast(func.nullif(func.split_part(cls.urn, "#", 2), ""), Integer) + # Order by this expression rather than restating it: the index below serves only an identical one. + return cast(func.split_part(cls.urn, "#", 2), Integer) # Serves ordering a score set's variants by number, so deep pages of a large score set skip the full sort. diff --git a/tests/models/test_variant.py b/tests/models/test_variant.py index ed318d7c2..81da770e5 100644 --- a/tests/models/test_variant.py +++ b/tests/models/test_variant.py @@ -6,12 +6,7 @@ @pytest.mark.unit @pytest.mark.parametrize( "urn, expected", - [ - ("urn:mavedb:00000053-a-1#648022", 648022), - ("tmp:0c8d1a2e-5f3b-4c6d-9e7f-1a2b3c4d5e6f#1", 1), - ("urn:mavedb:variant-1", None), - (None, None), - ], + [("urn:mavedb:00000053-a-1#648022", 648022), ("tmp:0c8d1a2e-5f3b-4c6d-9e7f-1a2b3c4d5e6f#1", 1), (None, None)], ) -def test_variant_number_is_the_integer_after_the_hash_or_none(urn, expected): +def test_variant_number_is_the_integer_after_the_hash(urn, expected): assert Variant(urn=urn).variant_number == expected From bfea1dec08ff5fa41822dd7d08fb2ff3580dfecb Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 29 Sep 2026 15:57:30 -0700 Subject: [PATCH 179/252] test: build fixture variant URNs from their score set Worker and calibration fixtures created variants with invented URNs such as urn:variant:test-variant-with-caid, which the variant-number index rejects. They now use #, as production does. --- tests/lib/test_score_calibrations.py | 68 +++++++++---------- tests/worker/jobs/conftest.py | 10 +-- .../external_services/test_clingen_cache.py | 8 +-- .../jobs/external_services/test_clinvar.py | 22 +++--- .../test_variant_translation.py | 3 +- .../worker/jobs/external_services/test_vep.py | 2 +- 6 files changed, 57 insertions(+), 56 deletions(-) diff --git a/tests/lib/test_score_calibrations.py b/tests/lib/test_score_calibrations.py index 3b080e90d..b941cc089 100644 --- a/tests/lib/test_score_calibrations.py +++ b/tests/lib/test_score_calibrations.py @@ -1820,9 +1820,9 @@ def test_variants_for_functional_classification_raises_error_when_index_column_n variant_classes = pd.DataFrame( { "some_other_column": [ - "urn:mavedb:variant-1", - "urn:mavedb:variant-2", - "urn:mavedb:variant-3", + f"{VALID_SCORE_SET_URN}#1", + f"{VALID_SCORE_SET_URN}#2", + f"{VALID_SCORE_SET_URN}#3", ], calibration_class_column_name: [ "pathogenic", @@ -1851,9 +1851,9 @@ def test_variants_for_functional_classification_raises_error_when_index_column_n pd.DataFrame( { calibration_variant_column_name: [ - "urn:mavedb:variant-1", - "urn:mavedb:variant-2", - "urn:mavedb:variant-3", + f"{VALID_SCORE_SET_URN}#1", + f"{VALID_SCORE_SET_URN}#2", + f"{VALID_SCORE_SET_URN}#3", ], calibration_class_column_name: [ "pathogenic", @@ -1910,21 +1910,21 @@ def test_variants_for_functional_classification_python_filtering_with_valid_vari variant_1 = Variant( data={"score_data": {"score": 0.5}}, score_set_id=setup_lib_db_with_score_set.id, - urn="urn:mavedb:variant-1", + urn=f"{VALID_SCORE_SET_URN}#1", hgvs_nt="NC_000001.11:g.1000A>T", hgvs_pro="NP_000000.1:p.Lys100Asn", ) variant_2 = Variant( data={"score_data": {"score": 1.5}}, score_set_id=setup_lib_db_with_score_set.id, - urn="urn:mavedb:variant-2", + urn=f"{VALID_SCORE_SET_URN}#2", hgvs_nt="NC_000001.11:g.1001G>C", hgvs_pro="NP_000000.1:p.Gly101Arg", ) variant_3 = Variant( data={"score_data": {"score": 2.5}}, score_set_id=setup_lib_db_with_score_set.id, - urn="urn:mavedb:variant-3", + urn=f"{VALID_SCORE_SET_URN}#3", hgvs_nt="NC_000001.11:g.1002T>A", hgvs_pro="NP_000000.1:p.Ser102Thr", ) @@ -1967,14 +1967,14 @@ def test_variants_for_functional_classification_python_filtering_skips_variants_ variant_without_score_data = Variant( data={"other_data": {"value": 1.0}}, score_set_id=setup_lib_db_with_score_set.id, - urn="urn:mavedb:variant-1", + urn=f"{VALID_SCORE_SET_URN}#1", ) # Create variant with valid score variant_with_score = Variant( data={"score_data": {"score": 1.5}}, score_set_id=setup_lib_db_with_score_set.id, - urn="urn:mavedb:variant-2", + urn=f"{VALID_SCORE_SET_URN}#2", ) session.add_all([variant_without_score_data, variant_with_score]) @@ -2013,14 +2013,14 @@ def test_variants_for_functional_classification_python_filtering_skips_variants_ variant_invalid_score_data = Variant( data={"score_data": "not_a_dict"}, score_set_id=setup_lib_db_with_score_set.id, - urn="urn:mavedb:variant-1", + urn=f"{VALID_SCORE_SET_URN}#1", ) # Create variant with valid score variant_with_score = Variant( data={"score_data": {"score": 1.5}}, score_set_id=setup_lib_db_with_score_set.id, - urn="urn:mavedb:variant-2", + urn=f"{VALID_SCORE_SET_URN}#2", ) session.add_all([variant_invalid_score_data, variant_with_score]) @@ -2058,14 +2058,14 @@ def test_variants_for_functional_classification_python_filtering_skips_variants_ variant_none_score = Variant( data={"score_data": {"score": None}}, score_set_id=setup_lib_db_with_score_set.id, - urn="urn:mavedb:variant-1", + urn=f"{VALID_SCORE_SET_URN}#1", ) # Create variant with valid score variant_with_score = Variant( data={"score_data": {"score": 1.5}}, score_set_id=setup_lib_db_with_score_set.id, - urn="urn:mavedb:variant-2", + urn=f"{VALID_SCORE_SET_URN}#2", ) session.add_all([variant_none_score, variant_with_score]) @@ -2103,14 +2103,14 @@ def test_variants_for_functional_classification_python_filtering_skips_variants_ variant_string_score = Variant( data={"score_data": {"score": "not_a_number"}}, score_set_id=setup_lib_db_with_score_set.id, - urn="urn:mavedb:variant-1", + urn=f"{VALID_SCORE_SET_URN}#1", ) # Create variant with valid score variant_with_score = Variant( data={"score_data": {"score": 1.5}}, score_set_id=setup_lib_db_with_score_set.id, - urn="urn:mavedb:variant-2", + urn=f"{VALID_SCORE_SET_URN}#2", ) session.add_all([variant_string_score, variant_with_score]) @@ -2146,14 +2146,14 @@ def test_variants_for_functional_classification_python_filtering_skips_variants_ ): # Create variant with non-dict data variant_invalid_data = Variant( - data="not_a_dict", score_set_id=setup_lib_db_with_score_set.id, urn="urn:mavedb:variant-1" + data="not_a_dict", score_set_id=setup_lib_db_with_score_set.id, urn=f"{VALID_SCORE_SET_URN}#1" ) # Create variant with valid score variant_with_score = Variant( data={"score_data": {"score": 1.5}}, score_set_id=setup_lib_db_with_score_set.id, - urn="urn:mavedb:variant-2", + urn=f"{VALID_SCORE_SET_URN}#2", ) session.add_all([variant_invalid_data, variant_with_score]) @@ -2191,11 +2191,11 @@ def test_variants_for_functional_classification_python_filtering_skips_variants_ pd.DataFrame( { calibration_variant_column_name: [ - "urn:mavedb:variant-1", - "urn:mavedb:variant-2", - "urn:mavedb:variant-3", - "urn:mavedb:variant-4", - "urn:mavedb:variant-5", + f"{VALID_SCORE_SET_URN}#1", + f"{VALID_SCORE_SET_URN}#2", + f"{VALID_SCORE_SET_URN}#3", + f"{VALID_SCORE_SET_URN}#4", + f"{VALID_SCORE_SET_URN}#5", ], calibration_class_column_name: [ "pathogenic", @@ -2280,7 +2280,7 @@ def test_variants_for_functional_classification_filters_by_conditions( variant = Variant( data={"score_data": {"score": score}}, score_set_id=setup_lib_db_with_score_set.id, - urn=f"urn:mavedb:variant-{i}", + urn=f"{VALID_SCORE_SET_URN}#{i}", hgvs_nt=hgvs_nts[i - 1], hgvs_pro=hgvs_pros[i - 1], ) @@ -2325,9 +2325,9 @@ def test_variants_for_functional_classification_filters_by_conditions( pd.DataFrame( { calibration_variant_column_name: [ - "urn:mavedb:variant-1", - "urn:mavedb:variant-2", - "urn:mavedb:variant-3", + f"{VALID_SCORE_SET_URN}#1", + f"{VALID_SCORE_SET_URN}#2", + f"{VALID_SCORE_SET_URN}#3", ], calibration_class_column_name: [ "benign", @@ -2385,7 +2385,7 @@ def test_variants_for_functional_classification_sql_fallback_on_exception( variant = Variant( data={"score_data": {"score": 1.5}}, score_set_id=setup_lib_db_with_score_set.id, - urn="urn:mavedb:variant-1", + urn=f"{VALID_SCORE_SET_URN}#1", hgvs_nt="NC_000001.11:g.1000A>T", hgvs_pro="NP_000000.1:p.Lys100Asn", ) @@ -2441,7 +2441,7 @@ def test_variants_for_functional_classification_sql_with_infinite_bound( variant = Variant( data={"score_data": {"score": score}}, score_set_id=setup_lib_db_with_score_set.id, - urn=f"urn:mavedb:variant-{i}", + urn=f"{VALID_SCORE_SET_URN}#{i}", ) variants.append(variant) @@ -2495,7 +2495,7 @@ def test_variants_for_functional_classification_sql_with_exclusive_bounds( variant = Variant( data={"score_data": {"score": score}}, score_set_id=setup_lib_db_with_score_set.id, - urn=f"urn:mavedb:variant-{i}", + urn=f"{VALID_SCORE_SET_URN}#{i}", ) variants.append(variant) @@ -2555,10 +2555,10 @@ def test_variants_for_functional_classification_only_returns_variants_from_corre variant_in_target_set = Variant( data={"score_data": {"score": 1.5}}, score_set_id=setup_lib_db_with_score_set.id, - urn="urn:mavedb:variant-target", + urn=f"{VALID_SCORE_SET_URN}#1", ) variant_in_other_set = Variant( - data={"score_data": {"score": 1.5}}, score_set_id=other_score_set.id, urn="urn:mavedb:variant-other" + data={"score_data": {"score": 1.5}}, score_set_id=other_score_set.id, urn=f"{other_score_set.urn}#1" ) session.add_all([variant_in_target_set, variant_in_other_set]) @@ -2581,7 +2581,7 @@ def test_variants_for_functional_classification_only_returns_variants_from_corre # Should only return variant from the target score set assert len(result) == 1 assert result[0].score_set_id == setup_lib_db_with_score_set.id - assert result[0].urn == "urn:mavedb:variant-target" + assert result[0].urn == f"{VALID_SCORE_SET_URN}#1" ################################################################################ diff --git a/tests/worker/jobs/conftest.py b/tests/worker/jobs/conftest.py index de835a56d..a88152109 100644 --- a/tests/worker/jobs/conftest.py +++ b/tests/worker/jobs/conftest.py @@ -229,7 +229,7 @@ def setup_sample_variants_with_caid( # Add a variant and mapped variant to the database with a CAID variant = Variant( - urn="urn:variant:test-variant-with-caid", + urn=f"{score_set.urn}#1", score_set_id=score_set.id, hgvs_nt="NM_000000.1:c.1A>G", hgvs_pro="NP_000000.1:p.Met1Val", @@ -970,7 +970,7 @@ def setup_sample_variants_with_caid_for_hgvs( score_set = session.get(ScoreSet, sample_populate_hgvs_run.job_params["score_set_id"]) variant = Variant( - urn="urn:variant:test-variant-with-caid-hgvs", + urn=f"{score_set.urn}#2", score_set_id=score_set.id, hgvs_nt="NM_000000.1:c.1A>G", hgvs_pro="NP_000000.1:p.Met1Val", @@ -1066,7 +1066,7 @@ def setup_sample_variants_with_caid_for_translation( score_set = session.get(ScoreSet, sample_populate_variant_translations_run.job_params["score_set_id"]) variant = Variant( - urn="urn:variant:test-variant-with-caid-translation", + urn=f"{score_set.urn}#3", score_set_id=score_set.id, hgvs_nt="NM_000000.1:c.1A>G", hgvs_pro="NP_000000.1:p.Met1Val", @@ -1228,7 +1228,7 @@ def setup_sample_variants_for_vep(session, with_populated_domain_data, mock_work score_set = session.get(ScoreSet, sample_populate_vep_run.job_params["score_set_id"]) variant = Variant( - urn="urn:variant:test-variant-for-vep", + urn=f"{score_set.urn}#4", score_set_id=score_set.id, hgvs_nt="NM_007294.4:c.5A>G", hgvs_pro="NP_009225.1:p.Cys2Tyr", @@ -1260,7 +1260,7 @@ def setup_sample_protein_variant_for_vep(session, with_populated_domain_data, mo score_set = session.get(ScoreSet, sample_populate_vep_run.job_params["score_set_id"]) variant = Variant( - urn="urn:variant:test-protein-variant-for-vep", + urn=f"{score_set.urn}#5", score_set_id=score_set.id, hgvs_pro="NP_009225.1:p.Val1696His", data={"hgvs_p": "NP_009225.1:p.Val1696His"}, diff --git a/tests/worker/jobs/external_services/test_clingen_cache.py b/tests/worker/jobs/external_services/test_clingen_cache.py index a55eb6b88..fa28a7259 100644 --- a/tests/worker/jobs/external_services/test_clingen_cache.py +++ b/tests/worker/jobs/external_services/test_clingen_cache.py @@ -52,7 +52,7 @@ async def test_warms_cache_for_variants_with_caids( # Create two variants with the same CAID — should only warm once (distinct) for i, caid in enumerate(["CA111111", "CA222222", "CA111111"]): variant = Variant( - urn=f"urn:variant:warm-test-{i}", + urn=f"{score_set.urn}#{100 + i}", score_set_id=score_set.id, hgvs_nt=f"NM_000000.1:c.{i + 1}A>G", hgvs_pro=f"NP_000000.1:p.Met{i + 1}Val", @@ -101,7 +101,7 @@ async def test_skips_null_and_multi_variant_caids( caids = ["CA333333", None, "CA-MULTI-001,CA-MULTI-002"] for i, caid in enumerate(caids): variant = Variant( - urn=f"urn:variant:warm-filter-{i}", + urn=f"{score_set.urn}#{200 + i}", score_set_id=score_set.id, hgvs_nt=f"NM_000000.1:c.{i + 10}A>G", hgvs_pro=f"NP_000000.1:p.Met{i + 10}Val", @@ -148,7 +148,7 @@ async def test_continues_on_individual_fetch_failure( for i, caid in enumerate(["CA444444", "CA555555"]): variant = Variant( - urn=f"urn:variant:warm-fail-{i}", + urn=f"{score_set.urn}#{300 + i}", score_set_id=score_set.id, hgvs_nt=f"NM_000000.1:c.{i + 20}A>G", hgvs_pro=f"NP_000000.1:p.Met{i + 20}Val", @@ -196,7 +196,7 @@ async def test_only_warms_current_mapped_variants( score_set = session.get(ScoreSet, sample_warm_clingen_cache_job_run.job_params["score_set_id"]) variant = Variant( - urn="urn:variant:warm-current-test", + urn=f"{score_set.urn}#400", score_set_id=score_set.id, hgvs_nt="NM_000000.1:c.30A>G", hgvs_pro="NP_000000.1:p.Met30Val", diff --git a/tests/worker/jobs/external_services/test_clinvar.py b/tests/worker/jobs/external_services/test_clinvar.py index 1ac616ffc..344029717 100644 --- a/tests/worker/jobs/external_services/test_clinvar.py +++ b/tests/worker/jobs/external_services/test_clinvar.py @@ -103,7 +103,7 @@ async def test_refresh_clinvar_controls_no_variants_have_caids( # Add a variant without a CAID score_set = session.get(ScoreSet, sample_refresh_clinvar_controls_job_run.job_params["score_set_id"]) variant = Variant( - urn="urn:variant:test-variant-no-caid", + urn=f"{score_set.urn}#11", score_set_id=score_set.id, hgvs_nt="NM_000000.1:c.2G>A", hgvs_pro="NP_000000.1:p.Val2Ile", @@ -366,7 +366,7 @@ async def test_refresh_clinvar_controls_successful_annotation_new_control( # Add a variant and mapped variant to the database with a CAID score_set = session.get(ScoreSet, sample_refresh_clinvar_controls_job_run.job_params["score_set_id"]) variant = Variant( - urn="urn:variant:test-variant-with-caid-2", + urn=f"{score_set.urn}#12", score_set_id=score_set.id, hgvs_nt="NM_000000.1:c.3C>T", hgvs_pro="NP_000000.1:p.Ala3Val", @@ -484,7 +484,7 @@ async def test_refresh_clinvar_controls_partial_failure( # Add an additional mapped variant to the database with a CAID score_set = session.get(ScoreSet, sample_refresh_clinvar_controls_job_run.job_params["score_set_id"]) variant2 = Variant( - urn="urn:variant:test-variant-with-caid-2", + urn=f"{score_set.urn}#12", score_set_id=score_set.id, hgvs_nt="NM_000000.1:c.4G>C", hgvs_pro="NP_000000.1:p.Gly4Ala", @@ -690,7 +690,7 @@ async def test_refresh_clinvar_controls_no_variants_with_caid( # Add a variant without a CAID score_set = session.get(ScoreSet, sample_refresh_clinvar_controls_job_run.job_params["score_set_id"]) variant = Variant( - urn="urn:variant:integration-test-variant-no-caid", + urn=f"{score_set.urn}#13", score_set_id=score_set.id, hgvs_nt="NM_000000.1:c.5T>A", hgvs_pro="NP_000000.1:p.Leu5Gln", @@ -746,7 +746,7 @@ async def test_refresh_clinvar_controlsvariants_are_multivariants( # Add a variant with a multi-variant CAID score_set = session.get(ScoreSet, sample_refresh_clinvar_controls_job_run.job_params["score_set_id"]) variant = Variant( - urn="urn:variant:integration-test-variant-multicid", + urn=f"{score_set.urn}#14", score_set_id=score_set.id, hgvs_nt="NM_000000.1:c.6A>G", hgvs_pro="NP_000000.1:p.Thr6Ala", @@ -808,7 +808,7 @@ async def test_refresh_clinvar_controls_no_associated_clinvar_allele_id( # Add a variant with a CAID score_set = session.get(ScoreSet, sample_refresh_clinvar_controls_job_run.job_params["score_set_id"]) variant = Variant( - urn="urn:variant:integration-test-variant-with-caid", + urn=f"{score_set.urn}#15", score_set_id=score_set.id, hgvs_nt="NM_000000.1:c.7C>A", hgvs_pro="NP_000000.1:p.Ser7Tyr", @@ -872,7 +872,7 @@ async def test_refresh_clinvar_controls_no_clinvar_data( # Add a variant with a CAID score_set = session.get(ScoreSet, sample_refresh_clinvar_controls_job_run.job_params["score_set_id"]) variant = Variant( - urn="urn:variant:integration-test-variant-with-caid", + urn=f"{score_set.urn}#15", score_set_id=score_set.id, hgvs_nt="NM_000000.1:c.8G>T", hgvs_pro="NP_000000.1:p.Val8Phe", @@ -936,7 +936,7 @@ async def test_refresh_clinvar_controls_successful_annotation_existing_control( # Add a variant with a CAID score_set = session.get(ScoreSet, sample_refresh_clinvar_controls_job_run.job_params["score_set_id"]) variant = Variant( - urn="urn:variant:integration-test-variant-with-caid", + urn=f"{score_set.urn}#15", score_set_id=score_set.id, hgvs_nt="NM_000000.1:c.9A>C", hgvs_pro="NP_000000.1:p.Lys9Thr", @@ -1015,7 +1015,7 @@ async def test_refresh_clinvar_controls_successful_annotation_new_control( # Add a variant with a CAID score_set = session.get(ScoreSet, sample_refresh_clinvar_controls_job_run.job_params["score_set_id"]) variant = Variant( - urn="urn:variant:integration-test-variant-with-caid", + urn=f"{score_set.urn}#15", score_set_id=score_set.id, hgvs_nt="NM_000000.1:c.10C>G", hgvs_pro="NP_000000.1:p.Pro10Arg", @@ -1082,7 +1082,7 @@ async def test_refresh_clinvar_controls_successful_annotation_pipeline_context( # Add a variant with a CAID score_set = session.get(ScoreSet, sample_refresh_clinvar_controls_job_in_pipeline.job_params["score_set_id"]) variant = Variant( - urn="urn:variant:integration-test-variant-with-caid", + urn=f"{score_set.urn}#15", score_set_id=score_set.id, hgvs_nt="NM_000000.1:c.12G>A", hgvs_pro="NP_000000.1:p.Met12Ile", @@ -1210,7 +1210,7 @@ async def test_refresh_clinvar_controls_partial_failure( # Add an additional mapped variant to the database with a CAID score_set = session.get(ScoreSet, sample_refresh_clinvar_controls_job_run.job_params["score_set_id"]) variant2 = Variant( - urn="urn:variant:integration-test-variant-with-caid-2", + urn=f"{score_set.urn}#16", score_set_id=score_set.id, hgvs_nt="NM_000000.1:c.11G>C", hgvs_pro="NP_000000.1:p.Gly11Ala", diff --git a/tests/worker/jobs/external_services/test_variant_translation.py b/tests/worker/jobs/external_services/test_variant_translation.py index 3e1f364ea..acb21f4bf 100644 --- a/tests/worker/jobs/external_services/test_variant_translation.py +++ b/tests/worker/jobs/external_services/test_variant_translation.py @@ -11,6 +11,7 @@ from mavedb.lib.types.workflow import JobExecutionOutcome from mavedb.models.enums.job_pipeline import FailureCategory, JobStatus, PipelineStatus from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.score_set import ScoreSet from mavedb.models.variant import Variant from mavedb.models.variant_annotation_status import VariantAnnotationStatus from mavedb.models.variant_translation import VariantTranslation @@ -370,7 +371,7 @@ async def test_multiple_alleles_sharing_pa_no_duplicate_error( score_set_id = sample_populate_variant_translations_run.job_params["score_set_id"] variant2 = Variant( - urn="urn:variant:test-second-ca-allele", + urn=f"{session.get(ScoreSet, score_set_id).urn}#21", score_set_id=score_set_id, hgvs_nt="NM_000000.1:c.2T>G", hgvs_pro="NP_000000.1:p.Val2Gly", diff --git a/tests/worker/jobs/external_services/test_vep.py b/tests/worker/jobs/external_services/test_vep.py index 611c33894..774c6cc4d 100644 --- a/tests/worker/jobs/external_services/test_vep.py +++ b/tests/worker/jobs/external_services/test_vep.py @@ -349,7 +349,7 @@ async def test_multiple_variants_sharing_hgvs_all_get_consequence( score_set = session.get(ScoreSet, sample_populate_vep_run.job_params["score_set_id"]) variant_2 = Variant( - urn="urn:variant:test-variant-for-vep-2", + urn=f"{score_set.urn}#22", score_set_id=score_set.id, hgvs_nt=hgvs, data={"hgvs_c": hgvs}, From 7270b8b59af371fddcc12cf84c063bdb096d9a2a Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 29 Sep 2026 16:02:37 -0700 Subject: [PATCH 180/252] fix(refget): return 416 for start > end on a linear sequence The refget spec requires 416 Range Not Satisfiable when start > end on a sequence that isn't circular, and 501 only for a circular one when circular sub-sequences aren't supported. The route returned 501 for every start > end. It now checks the resolved sequence against the known circular accessions (the human mitochondrial references), since SeqRepo records no topology. Also update the inverted Range header test for the 400 the route has returned since ab892504. --- src/mavedb/routers/refget.py | 29 +++++++++++++++++++---------- tests/routers/test_refget.py | 17 ++++++++++++++--- 2 files changed, 33 insertions(+), 13 deletions(-) diff --git a/src/mavedb/routers/refget.py b/src/mavedb/routers/refget.py index 9e406c97a..c19e3bbc1 100644 --- a/src/mavedb/routers/refget.py +++ b/src/mavedb/routers/refget.py @@ -33,6 +33,10 @@ logger = logging.getLogger(__name__) +# SeqRepo records no sequence topology, so the human mitochondrial references are the only sequences treated as +# circular. Refget needs the distinction: start > end is 416 on a linear sequence and 501 on a circular one. +CIRCULAR_SEQUENCE_ACCESSIONS = frozenset({"NC_012920.1", "NC_001807.4"}) + router = APIRouter( prefix=f"{ROUTER_BASE_PREFIX}/refget", tags=[TAG_NAME], @@ -166,16 +170,6 @@ def get_sequence( raise HTTPException(status_code=400, detail="Invalid range header format") save_to_logging_context({"requested_refget_start": start, "requested_refget_end": end}) - if start is not None and end is not None: - if start > end: - logger.error( - msg="Invalid coordinates: start is greater than end and circular chromosomes are not supported", - extra=logging_context(), - ) - raise HTTPException( - status_code=501, - detail="Invalid coordinates: start is greater than end and circular chromosomes are not supported", - ) seq_ids = get_sequence_ids(sr, alias) if not seq_ids: @@ -195,6 +189,21 @@ def get_sequence( seq_start = start if start is not None else 0 seq_end = end if end is not None else seq_len + # Refget spec: start > end asks for a circular sub-sequence across the origin. That's unsatisfiable on a linear + # sequence, and not implemented here for a circular one. + if start is not None and end is not None and start > end: + if not CIRCULAR_SEQUENCE_ACCESSIONS.isdisjoint(a["alias"] for a in sr.aliases.find_aliases(seq_id=seq_id)): + logger.error(msg="Circular sub-sequences are not supported", extra=logging_context()) + raise HTTPException( + status_code=501, + detail="Invalid coordinates: start is greater than end and circular chromosomes are not supported", + ) + raise HTTPException( + status_code=416, + detail="Invalid coordinates: start is greater than end on a linear sequence", + headers={"Content-Range": f"bytes */{seq_len}"}, + ) + # Refget spec: only a start "larger than" the sequence length is invalid (strict inequality). if start is not None and end is not None: if seq_start > seq_len: diff --git a/tests/routers/test_refget.py b/tests/routers/test_refget.py index c6afd8623..6b656e5ed 100644 --- a/tests/routers/test_refget.py +++ b/tests/routers/test_refget.py @@ -11,6 +11,7 @@ biocommons = pytest.importorskip("biocommons") bioutils = pytest.importorskip("bioutils") +from mavedb.routers import refget from tests.helpers.constants import TEST_SEQREPO_INITIAL_STATE, VALID_ENSEMBL_IDENTIFIER @@ -193,8 +194,8 @@ def test_get_sequence_multiple_ids(client): def test_get_sequence_invalid_header_range_coords_start_larger_than_end(client): resp = client.get(f"/api/v1/refget/sequence/{VALID_ENSEMBL_IDENTIFIER}", headers={"Range": "bytes=12-10"}) - assert resp.status_code == 501 - assert "Invalid coordinates" in resp.text + assert resp.status_code == 400 + assert "Invalid range header format" in resp.text def test_get_sequence_invalid_header_range_coords_start_too_large(client): @@ -213,8 +214,18 @@ def test_get_sequence_invalid_header_range_coords_end_too_large(client): def test_get_sequence_invalid_query_range_coords_start_larger_than_end(client): resp = client.get(f"/api/v1/refget/sequence/{VALID_ENSEMBL_IDENTIFIER}", params={"start": 12, "end": 10}) - assert resp.status_code == 501 + assert resp.status_code == 416 assert "Invalid coordinates" in resp.text + assert "Content-Range" in resp.headers + + +def test_get_sequence_query_range_start_larger_than_end_on_circular_sequence_is_not_implemented(client, monkeypatch): + monkeypatch.setattr(refget, "CIRCULAR_SEQUENCE_ACCESSIONS", frozenset({VALID_ENSEMBL_IDENTIFIER})) + + resp = client.get(f"/api/v1/refget/sequence/{VALID_ENSEMBL_IDENTIFIER}", params={"start": 12, "end": 10}) + + assert resp.status_code == 501 + assert "circular chromosomes are not supported" in resp.text def test_get_sequence_invalid_query_range_coords_start_too_large(client): From 21ae2546e433390b25f7f7a717125f6b7f9bc27f Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 29 Sep 2026 14:02:33 -0700 Subject: [PATCH 181/252] perf(csv): cap concurrent large CSV builds per process Building a large score set CSV is CPU-bound Python. A client pulling full CSVs for every score set (1,706 in two hours on 2026-09-28) kept several builds running at once and stalled the API, well under the WAF's per-IP request limit. - Each API process runs at most CSV_BUILD_SLOTS (2) builds of CSV_BUILD_MIN_ROWS (10,000) rows or more at once. Smaller builds, including the score set page previews, skip the cap. - A large build waits up to 10s for a slot, then returns 503 with Retry-After and a pointer to the bulk archive. - Add MAVEDB_BULK_DOWNLOAD_URL for the archive's concept DOI. A stopgap until full exports are cheap or cached (#845, #771). --- README.md | 6 ++- src/mavedb/constants.py | 2 + src/mavedb/lib/csv/build_limiter.py | 67 +++++++++++++++++++++++++++++ src/mavedb/routers/score_sets.py | 38 +++++++++------- tests/lib/csv/test_build_limiter.py | 58 +++++++++++++++++++++++++ tests/routers/test_score_set.py | 30 +++++++++++++ 6 files changed, 183 insertions(+), 18 deletions(-) create mode 100644 src/mavedb/lib/csv/build_limiter.py create mode 100644 tests/lib/csv/test_build_limiter.py diff --git a/README.md b/README.md index 147e922ea..20e7f10ec 100644 --- a/README.md +++ b/README.md @@ -88,8 +88,8 @@ after running `docker-compose down`. The database username and password should be edited for production deployments. `NCBI_API_KEY` will be removed in the future. **TODO** Move these to an .env file. -2. Optional database tuning variables. All values are in seconds, and `0` disables a timeout. The four timeouts apply - only to the API server; the worker, scripts and migrations run without them. +2. Optional database and CSV export tuning variables. Times are in seconds, and setting a timeout or `CSV_BUILD_SLOTS` + to `0` disables it. All of them apply only to the API server; the worker, scripts and migrations run without them. | Variable | Default | Purpose | |---|---|---| @@ -100,6 +100,8 @@ after running `docker-compose down`. | `DB_LOCK_TIMEOUT_SECONDS` | 5 | Longest wait for a row or table lock before the request returns 503 | | `DB_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS` | 60 | Longest idle gap inside an open transaction before Postgres ends the session | | `DB_SLOW_STATEMENT_SECONDS` | 5 | Statements slower than this are logged with their route | + | `CSV_BUILD_SLOTS` | 2 | Score set CSV builds of `CSV_BUILD_MIN_ROWS` or more rows each process runs at once | + | `CSV_BUILD_MIN_ROWS` | 10000 | Smaller builds skip the cap. A large build waits up to 10s for a slot, then returns 503 with `Retry-After` | ### Running the API server in Docker for development diff --git a/src/mavedb/constants.py b/src/mavedb/constants.py index 41f9d825b..0932f858a 100644 --- a/src/mavedb/constants.py +++ b/src/mavedb/constants.py @@ -2,3 +2,5 @@ MAVEDB_BASE_GIT = "https://github.com/VariantEffect/mavedb-api" MAVEDB_FRONTEND_URL = os.getenv("MAVE_FRONTEND_URL", "https://mavedb.org") +# Concept DOI of the public data archive on Zenodo; it always resolves to the latest release. +MAVEDB_BULK_DOWNLOAD_URL = "https://doi.org/10.5281/zenodo.11201736" diff --git a/src/mavedb/lib/csv/build_limiter.py b/src/mavedb/lib/csv/build_limiter.py new file mode 100644 index 000000000..a3f8b43cb --- /dev/null +++ b/src/mavedb/lib/csv/build_limiter.py @@ -0,0 +1,67 @@ +"""Load shedding for the score set CSV exports. + +Building a large CSV is CPU-bound Python, and Python runs one thread at a time per process. A few large builds +running at once in one API process starve everything else that process serves, so each process runs only a few at a +time and refuses the rest with a 503 the client can retry. + +TODO(#845, #771): A stopgap. Remove it once full exports are cheap or served from cache, and ``csv_build_refused`` +no longer appears in the request logs. +""" + +import logging +import os +import threading +from collections.abc import Iterator +from contextlib import contextmanager +from typing import Optional + +from fastapi import HTTPException + +from mavedb.constants import MAVEDB_BULK_DOWNLOAD_URL +from mavedb.lib.logging.context import logging_context, save_to_logging_context + +logger = logging.getLogger(__name__) + +# Large builds one API process runs at once; 0 disables the cap. +CSV_BUILD_SLOTS = int(os.getenv("CSV_BUILD_SLOTS", "2")) +# Builds of fewer rows skip the cap, so previews and small score sets are never refused. +CSV_BUILD_MIN_ROWS = int(os.getenv("CSV_BUILD_MIN_ROWS", "10000")) +# How long a request waits for a slot. Leaves most of the load balancer's 60s idle timeout for the build itself. +CSV_BUILD_WAIT_SECONDS = 10 +RETRY_AFTER_SECONDS = 30 + +BUSY_DETAIL = ( + "The server is busy building other large downloads. Retry after the number of seconds in the Retry-After header." + f" To download many datasets, consider using the bulk archive instead: {MAVEDB_BULK_DOWNLOAD_URL}" +) + +_slots = threading.BoundedSemaphore(CSV_BUILD_SLOTS) if CSV_BUILD_SLOTS else None + + +def rows_to_build(num_variants: Optional[int], start: Optional[int], limit: Optional[int]) -> int: + """The number of rows a request for ``start``/``limit`` of a score set with ``num_variants`` variants returns.""" + # TODO(#372): `num_variants` is NOT NULL but typed optional. + remaining = max((num_variants or 0) - (start or 0), 0) + return min(remaining, limit) if limit else remaining + + +@contextmanager +def csv_build_slot(rows: int) -> Iterator[None]: + """Hold one of this process's CSV build slots while building ``rows`` rows. + + Waits up to ``CSV_BUILD_WAIT_SECONDS`` for a slot, then raises a 503 with ``Retry-After``. Builds smaller than + ``CSV_BUILD_MIN_ROWS`` run without a slot. + """ + if _slots is None or rows < CSV_BUILD_MIN_ROWS: + yield + return + + if not _slots.acquire(timeout=CSV_BUILD_WAIT_SECONDS): + save_to_logging_context({"csv_build_refused": True, "csv_build_rows": rows}) + logger.info(msg="Refused a large CSV build; every build slot is in use.", extra=logging_context()) + raise HTTPException(status_code=503, detail=BUSY_DETAIL, headers={"Retry-After": str(RETRY_AFTER_SECONDS)}) + + try: + yield + finally: + _slots.release() diff --git a/src/mavedb/routers/score_sets.py b/src/mavedb/routers/score_sets.py index 9e10fcf43..de85be7f3 100644 --- a/src/mavedb/routers/score_sets.py +++ b/src/mavedb/routers/score_sets.py @@ -37,6 +37,7 @@ require_current_user_with_email, ) from mavedb.lib.contributors import find_or_create_contributor +from mavedb.lib.csv.build_limiter import csv_build_slot, rows_to_build from mavedb.lib.csv.columns import variants_to_csv_rows from mavedb.lib.csv.deprecated_params import ( DROP_NA_COLUMNS_DESCRIPTION, @@ -1084,18 +1085,19 @@ def get_score_set_variants_csv( # Full-table CSV builds legitimately outrun the API's default statement timeout. allow_long_statements(db) - csv_str = get_score_set_variants_as_csv( - db, - score_set, - namespaces, - True, - start, - limit, - drop_unused_hgvs_columns, - # Asked separately from the score set: a private calibration is readable only by its owner, - # investigator contributors, or an admin, whoever can read the score set. - viewer=principal.viewer_for(ScoreCalibrationViewer), - ) + with csv_build_slot(rows_to_build(score_set.num_variants, start, limit)): + csv_str = get_score_set_variants_as_csv( + db, + score_set, + namespaces, + True, + start, + limit, + drop_unused_hgvs_columns, + # Asked separately from the score set: a private calibration is readable only by its owner, + # investigator contributors, or an admin, whoever can read the score set. + viewer=principal.viewer_for(ScoreCalibrationViewer), + ) return StreamingResponse(iter([csv_str]), media_type="text/csv", headers=deprecated.response_headers) @@ -1164,9 +1166,10 @@ def get_score_set_scores_csv( # Both score namespaces: this endpoint has always returned every score column the investigator # uploaded, and `scores` alone is now just the required one. - csv_str = get_score_set_variants_as_csv( - db, score_set, ["scores", "scores_custom"], False, start, limit, drop_unused_hgvs_columns - ) + with csv_build_slot(rows_to_build(score_set.num_variants, start, limit)): + csv_str = get_score_set_variants_as_csv( + db, score_set, ["scores", "scores_custom"], False, start, limit, drop_unused_hgvs_columns + ) return StreamingResponse(iter([csv_str]), media_type="text/csv", headers=deprecated.response_headers) @@ -1233,7 +1236,10 @@ def get_score_set_counts_csv( # Full-table CSV builds legitimately outrun the API's default statement timeout. allow_long_statements(db) - csv_str = get_score_set_variants_as_csv(db, score_set, ["counts"], False, start, limit, drop_unused_hgvs_columns) + with csv_build_slot(rows_to_build(score_set.num_variants, start, limit)): + csv_str = get_score_set_variants_as_csv( + db, score_set, ["counts"], False, start, limit, drop_unused_hgvs_columns + ) return StreamingResponse(iter([csv_str]), media_type="text/csv", headers=deprecated.response_headers) diff --git a/tests/lib/csv/test_build_limiter.py b/tests/lib/csv/test_build_limiter.py new file mode 100644 index 000000000..a1792d8a5 --- /dev/null +++ b/tests/lib/csv/test_build_limiter.py @@ -0,0 +1,58 @@ +import threading + +import pytest +from fastapi import HTTPException + +from mavedb.lib.csv import build_limiter +from mavedb.lib.csv.build_limiter import csv_build_slot, rows_to_build + + +@pytest.fixture +def one_slot(monkeypatch): + """A single build slot, a 1000-row threshold and no waiting, so a held slot refuses immediately.""" + slots = threading.BoundedSemaphore(1) + monkeypatch.setattr(build_limiter, "_slots", slots) + monkeypatch.setattr(build_limiter, "CSV_BUILD_MIN_ROWS", 1000) + monkeypatch.setattr(build_limiter, "CSV_BUILD_WAIT_SECONDS", 0.01) + return slots + + +@pytest.mark.unit +@pytest.mark.parametrize( + "num_variants, start, limit, expected", + [(648022, None, None, 648022), (648022, 600000, 50000, 48022), (648022, None, 5, 5), (10, 20, None, 0)], +) +def test_rows_to_build(num_variants, start, limit, expected): + assert rows_to_build(num_variants, start, limit) == expected + + +@pytest.mark.unit +class TestCsvBuildSlot: + def test_refuses_a_large_build_when_every_slot_is_held(self, one_slot): + one_slot.acquire() + + with pytest.raises(HTTPException) as exc_info: + with csv_build_slot(5000): + pass + + assert exc_info.value.status_code == 503 + assert exc_info.value.headers == {"Retry-After": str(build_limiter.RETRY_AFTER_SECONDS)} + + def test_small_builds_skip_the_cap(self, one_slot): + one_slot.acquire() + + with csv_build_slot(999): + pass + + def test_releases_the_slot_even_when_the_build_fails(self, one_slot): + with pytest.raises(RuntimeError): + with csv_build_slot(5000): + raise RuntimeError("build failed") + + assert one_slot.acquire(blocking=False) + + def test_zero_slots_disables_the_cap(self, monkeypatch): + monkeypatch.setattr(build_limiter, "_slots", None) + + with csv_build_slot(10_000_000): + pass diff --git a/tests/routers/test_score_set.py b/tests/routers/test_score_set.py index 3b3bd33d9..9fd1b84fe 100644 --- a/tests/routers/test_score_set.py +++ b/tests/routers/test_score_set.py @@ -6,6 +6,7 @@ from copy import deepcopy from datetime import date from io import StringIO +import threading from unittest.mock import patch import jsonschema @@ -18,6 +19,7 @@ fastapi = pytest.importorskip("fastapi") from mavedb.lib.annotation.annotate import variant_study_result +from mavedb.lib.csv import build_limiter from mavedb.lib.annotation.exceptions import MappingDataDoesntExistException from mavedb.lib.exceptions import NonexistentOrcidUserError from mavedb.lib.validation.urn_re import MAVEDB_EXPERIMENT_URN_RE, MAVEDB_SCORE_SET_URN_RE, MAVEDB_TMP_URN_RE @@ -3851,6 +3853,34 @@ def test_csv_routes_raise_the_statement_timeout(session, data_provider, client, allow_long_statements.assert_called_once() +@pytest.mark.parametrize("path", ["scores", "counts", "variants/data"]) +def test_csv_routes_refuse_large_builds_when_every_slot_is_held( + session, data_provider, client, setup_router_db, data_files, monkeypatch, path +): + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion( + client, session, data_provider, score_set, data_files / "scores.csv", data_files / "counts.csv" + ) + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published_score_set = publish_score_set(client, score_set["urn"]) + + held = threading.BoundedSemaphore(1) + held.acquire() + monkeypatch.setattr(build_limiter, "_slots", held) + monkeypatch.setattr(build_limiter, "CSV_BUILD_MIN_ROWS", 1) + monkeypatch.setattr(build_limiter, "CSV_BUILD_WAIT_SECONDS", 0.01) + + refused = client.get(f"/api/v1/score-sets/{published_score_set['urn']}/{path}") + + assert refused.status_code == 503 + assert refused.headers["retry-after"] == str(build_limiter.RETRY_AFTER_SECONDS) + # A limit under the threshold is never refused, which keeps the score set page's previews working. + monkeypatch.setattr(build_limiter, "CSV_BUILD_MIN_ROWS", 2) + preview = client.get(f"/api/v1/score-sets/{published_score_set['urn']}/{path}?limit=1") + assert preview.status_code == 200 + + # Deprecated query-parameter aliases. Galaxy and other external tooling call these endpoints, so the old # names keep working for a release rather than being silently ignored. def test_deprecated_drop_na_columns_still_drops_unused_hgvs_columns( From 38dc07f977fd6e78575fb1d12aa3b6c9ca69cef9 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 29 Sep 2026 12:49:18 -0700 Subject: [PATCH 182/252] fix(db): bound API database waits and stop per-request login writes - Apply statement (30s), lock (5s) and idle-in-transaction (60s) timeouts to API connections only; the worker, scripts and migrations are unaffected. Each is env-configurable; 0 disables. - Raise the statement timeout to 45s on the score set CSV routes. - Return 503 without a Slack alert for statement, lock and connection-pool timeouts. - Wait at most 10s for a pooled connection (DB_POOL_TIMEOUT). - Log statements slower than 5s with their route. - Make the auth dependencies sync and write last_login at most every 15 minutes. - Make the counts CSV route sync. Refs #881 --- README.md | 13 +++ src/mavedb/db/session.py | 3 + src/mavedb/db/timeouts.py | 101 +++++++++++++++++++++++ src/mavedb/lib/authentication.py | 40 +++++++--- src/mavedb/lib/middleware/errors.py | 36 +++++++-- src/mavedb/routers/score_sets.py | 12 ++- src/mavedb/server_main.py | 5 ++ tests/db/__init__.py | 0 tests/db/test_timeouts.py | 105 ++++++++++++++++++++++++ tests/lib/middleware/test_errors.py | 65 +++++++++++++++ tests/lib/test_authentication.py | 119 ++++++++++++++++++---------- tests/routers/test_score_set.py | 17 ++++ 12 files changed, 456 insertions(+), 60 deletions(-) create mode 100644 src/mavedb/db/timeouts.py create mode 100644 tests/db/__init__.py create mode 100644 tests/db/test_timeouts.py diff --git a/README.md b/README.md index e59bbc13e..147e922ea 100644 --- a/README.md +++ b/README.md @@ -88,6 +88,19 @@ after running `docker-compose down`. The database username and password should be edited for production deployments. `NCBI_API_KEY` will be removed in the future. **TODO** Move these to an .env file. +2. Optional database tuning variables. All values are in seconds, and `0` disables a timeout. The four timeouts apply + only to the API server; the worker, scripts and migrations run without them. + + | Variable | Default | Purpose | + |---|---|---| + | `DB_POOL_SIZE` / `DB_MAX_OVERFLOW` | 5 / 10 | Persistent and burst connections per process | + | `DB_POOL_TIMEOUT` | 10 | Longest wait for a free pooled connection before the request returns 503 | + | `DB_STATEMENT_TIMEOUT_SECONDS` | 30 | Longest single statement before the request returns 503 | + | `DB_CSV_STATEMENT_TIMEOUT_SECONDS` | 45 | The same limit for the score set CSV routes | + | `DB_LOCK_TIMEOUT_SECONDS` | 5 | Longest wait for a row or table lock before the request returns 503 | + | `DB_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS` | 60 | Longest idle gap inside an open transaction before Postgres ends the session | + | `DB_SLOW_STATEMENT_SECONDS` | 5 | Statements slower than this are logged with their route | + ### Running the API server in Docker for development A similar procedure can be followed to run the API server in development mode on your local machine. There are a couple diff --git a/src/mavedb/db/session.py b/src/mavedb/db/session.py index 8c0127ac4..c667d7299 100644 --- a/src/mavedb/db/session.py +++ b/src/mavedb/db/session.py @@ -12,6 +12,8 @@ DB_PASSWORD = os.getenv("DB_PASSWORD") DB_POOL_SIZE = int(os.getenv("DB_POOL_SIZE", "5")) DB_MAX_OVERFLOW = int(os.getenv("DB_MAX_OVERFLOW", "10")) +# Kept short so a request waiting on the pool fails inside the load balancer's idle timeout. +DB_POOL_TIMEOUT = int(os.getenv("DB_POOL_TIMEOUT", "10")) DB_URL = f"postgresql://{DB_USERNAME}:{DB_PASSWORD}@{DB_HOST}:{DB_PORT}/{DB_DATABASE_NAME}" @@ -19,6 +21,7 @@ DB_URL, pool_size=DB_POOL_SIZE, max_overflow=DB_MAX_OVERFLOW, + pool_timeout=DB_POOL_TIMEOUT, pool_pre_ping=True, ) SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) diff --git a/src/mavedb/db/timeouts.py b/src/mavedb/db/timeouts.py new file mode 100644 index 000000000..540a2b812 --- /dev/null +++ b/src/mavedb/db/timeouts.py @@ -0,0 +1,101 @@ +"""Database timeouts for the API. + +The API sets its limits on each connection it opens, so the worker, scripts and migrations, which share the +same login and legitimately run long statements, are unaffected. Prod connects to RDS directly; behind RDS +Proxy these session-level ``SET``s would pin every connection and belong on the login instead. +""" + +import logging +import os +import time + +from sqlalchemy import event, text +from sqlalchemy.engine import Engine +from sqlalchemy.orm import Session + +from mavedb.lib.logging.context import logging_context + +logger = logging.getLogger(__name__) + +# Each limit is in seconds and can be overridden from the environment; 0 disables it, as in Postgres. +# The prod API load balancer's idle timeout is 60s. A statement's limit plus the 10s pool wait stays under it. +API_STATEMENT_TIMEOUT_SECONDS = int(os.getenv("DB_STATEMENT_TIMEOUT_SECONDS", "30")) +CSV_STATEMENT_TIMEOUT_SECONDS = int(os.getenv("DB_CSV_STATEMENT_TIMEOUT_SECONDS", "45")) +# Below the statement timeout, so API writes queued behind a worker's bulk write fail fast. +API_LOCK_TIMEOUT_SECONDS = int(os.getenv("DB_LOCK_TIMEOUT_SECONDS", "5")) +# Above the longest legitimate gap between statements inside one request's transaction. +API_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS = int(os.getenv("DB_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS", "60")) +# Not a limit: statements slower than this are logged so the limits above can be tuned from real data. +SLOW_STATEMENT_SECONDS = float(os.getenv("DB_SLOW_STATEMENT_SECONDS", "5")) + +_STARTED_AT_ATTR = "_mavedb_statement_started_at" +_SLOW_STATEMENT_LOG_LENGTH = 200 + + +def _set_api_timeouts(dbapi_connection, _connection_record) -> None: + cursor = dbapi_connection.cursor() + try: + cursor.execute(f"SET statement_timeout = '{API_STATEMENT_TIMEOUT_SECONDS}s'") + cursor.execute(f"SET lock_timeout = '{API_LOCK_TIMEOUT_SECONDS}s'") + cursor.execute(f"SET idle_in_transaction_session_timeout = '{API_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS}s'") + finally: + cursor.close() + + # psycopg2 opens a transaction for the first statement; commit so the SETs are not rolled back. + dbapi_connection.commit() + + +def apply_api_timeouts(engine: Engine) -> None: + """Apply the API's timeouts to every new connection of ``engine``. Safe to call repeatedly. + + Call this before the first connection is opened: connections already in the pool are not touched. + """ + if not event.contains(engine, "connect", _set_api_timeouts): + event.listen(engine, "connect", _set_api_timeouts) + + +def allow_long_statements(db: Session, seconds: int = CSV_STATEMENT_TIMEOUT_SECONDS) -> None: + """Raise ``statement_timeout`` for the rest of the current transaction. + + Transaction-scoped, so it does not outlive the request. A ``commit()`` or ``rollback()`` ends it, so call + this immediately before the long read. Does nothing while the API's statement timeout is disabled, so + disabling it disables it everywhere. + """ + if not API_STATEMENT_TIMEOUT_SECONDS: + return + + db.execute(text("SELECT set_config('statement_timeout', :ms, true)"), {"ms": str(seconds * 1000)}) + + +def log_slow_statements(engine: Engine, threshold_seconds: float) -> None: + """Log any statement on ``engine`` that runs longer than ``threshold_seconds``, including ones that error. + + A statement cancelled by ``statement_timeout`` is logged too, since those are the ones worth tuning. + Logs the duration and the start of the SQL text, never the bound parameters, which can hold user data. + """ + + def _log(statement: str, started_at: float) -> None: + elapsed = time.perf_counter() - started_at + if elapsed < threshold_seconds: + return + + text_start = " ".join(statement.split())[:_SLOW_STATEMENT_LOG_LENGTH] + logger.warning( + msg=f"Slow database statement ({elapsed:.1f}s): {text_start}", + extra={**logging_context(), "statement_seconds": round(elapsed, 3)}, + ) + + @event.listens_for(engine, "before_cursor_execute") + def _start_timer(_conn, _cursor, _statement, _parameters, context, _executemany): + setattr(context, _STARTED_AT_ATTR, time.perf_counter()) + + @event.listens_for(engine, "after_cursor_execute") + def _log_completed(_conn, _cursor, statement, _parameters, context, _executemany): + _log(statement, getattr(context, _STARTED_AT_ATTR)) + + @event.listens_for(engine, "handle_error") + def _log_failed(exception_context): + context = exception_context.execution_context + started_at = getattr(context, _STARTED_AT_ATTR, None) + if started_at is not None and exception_context.statement: + _log(exception_context.statement, started_at) diff --git a/src/mavedb/lib/authentication.py b/src/mavedb/lib/authentication.py index fed13ae1f..d10fad7ea 100644 --- a/src/mavedb/lib/authentication.py +++ b/src/mavedb/lib/authentication.py @@ -1,6 +1,6 @@ import logging import os -from datetime import datetime +from datetime import datetime, timedelta from enum import Enum from typing import Optional @@ -38,6 +38,10 @@ ACCESS_TOKEN_NAME = "X-API-key" +# How stale `last_login` may get before a request rewrites it. Nothing reads the value at request granularity, +# and writing it on every request costs a commit and a refresh on the hot path of every authenticated route. +LAST_LOGIN_REFRESH_INTERVAL = timedelta(minutes=15) + logger = logging.getLogger(__name__) @@ -121,7 +125,7 @@ async def get_access_token( return access_token_header or access_token_cookie -async def get_current_user_data_from_api_key( +def get_current_user_data_from_api_key( db: Session = Depends(deps.get_db), access_token: str = Depends(get_access_token) ) -> Optional[UserData]: user = None @@ -155,7 +159,7 @@ async def get_current_user_data_from_api_key( #################################################################################################### -async def get_current_user( +def get_current_user( api_key_user_data: Optional[UserData] = Depends(get_current_user_data_from_api_key), token_payload: Optional[dict] = Depends(JWTBearer()), db: Session = Depends(deps.get_db), @@ -205,6 +209,7 @@ async def get_current_user( email=email, is_first_login=True, ) + needs_write = True save_to_logging_context( { "user": user.id, @@ -220,8 +225,14 @@ async def get_current_user( logger.info(msg="Failed to authenticate user; User is inactive.", extra=logging_context()) return None else: - user.last_login = datetime.now() - user.is_first_login = False + needs_write = ( + user.is_first_login + or user.last_login is None + or datetime.now() - user.last_login >= LAST_LOGIN_REFRESH_INTERVAL + ) + if needs_write: + user.last_login = datetime.now() + user.is_first_login = False save_to_logging_context( { "user": user.id, @@ -230,17 +241,20 @@ async def get_current_user( } ) - db.add(user) - try: - db.commit() - except IntegrityError: + if needs_write: + db.add(user) + try: + db.commit() + # A concurrent request created this user between our initial query and this commit. # Roll back and re-fetch the existing record. - db.rollback() - user = db.query(User).filter(User.username == username).one() - logger.debug(msg="Concurrent first-login resolved; returning existing user.", extra=logging_context()) + except IntegrityError: + db.rollback() + user = db.query(User).filter(User.username == username).one() + logger.debug(msg="Concurrent first-login resolved; returning existing user.", extra=logging_context()) + + db.refresh(user) - db.refresh(user) logger.info(msg="Successfully authenticated user via JWT.", extra=logging_context()) # When no roles are requested, the user may act as any assigned role. diff --git a/src/mavedb/lib/middleware/errors.py b/src/mavedb/lib/middleware/errors.py index 871eac5d9..b78c59525 100644 --- a/src/mavedb/lib/middleware/errors.py +++ b/src/mavedb/lib/middleware/errors.py @@ -3,6 +3,8 @@ import logging import time +from sqlalchemy.exc import DBAPIError +from sqlalchemy.exc import TimeoutError as PoolTimeoutError from starlette.requests import Request from starlette.responses import JSONResponse from starlette.types import ASGIApp, Message, Receive, Scope, Send @@ -18,9 +20,25 @@ logger = logging.getLogger(__name__) +# SQLSTATEs Postgres raises when a query exceeds `statement_timeout` (query_canceled) or waits past +# `lock_timeout` (lock_not_available). +_DATABASE_TIMEOUT_SQLSTATES = frozenset({"57014", "55P03"}) +_BUSY_DETAIL = "The database is busy. Please try again shortly." + + +def _is_database_busy(err: BaseException) -> bool: + """Whether ``err`` means the database is too busy right now, rather than that the server is broken. + + Covers the server's statement and lock timeouts and a request that waited past ``pool_timeout`` for a + pooled connection. + """ + if isinstance(err, PoolTimeoutError): + return True + return isinstance(err, DBAPIError) and getattr(err.orig, "pgcode", None) in _DATABASE_TIMEOUT_SQLSTATES + class CatchAllErrorMiddleware: - """Turn an uncaught exception into a 500 that the browser is allowed to read. + """Turn an uncaught exception into a 500 (503 when the database is busy) that the browser is allowed to read. Starlette dispatches ``@app.exception_handler(Exception)`` from ``ServerErrorMiddleware``, which sits outside the user middleware stack. Its response therefore never passes through ``CORSMiddleware`` and @@ -61,14 +79,22 @@ async def send_wrapper(message: Message) -> None: save_to_logging_context(format_raised_exception_info_as_dict(err)) request = Request(scope, receive) + busy = _is_database_busy(err) response = JSONResponse( - status_code=500, - content={"detail": "Internal server error", "correlation_id": correlation_id_for_context()}, + status_code=503 if busy else 500, + content={ + "detail": _BUSY_DETAIL if busy else "Internal server error", + "correlation_id": correlation_id_for_context(), + }, ) try: - logger.error(msg="Uncaught exception.", extra=logging_context(), exc_info=err) - send_slack_error(err=err, request=request) + # No Slack alert: these arrive in bursts under load, and the request log counts them. + if busy: + logger.warning(msg="Database busy.", extra=logging_context(), exc_info=err) + else: + logger.error(msg="Uncaught exception.", extra=logging_context(), exc_info=err) + send_slack_error(err=err, request=request) finally: log_request(request, response, time.time_ns()) diff --git a/src/mavedb/routers/score_sets.py b/src/mavedb/routers/score_sets.py index 915f82dda..9e10fcf43 100644 --- a/src/mavedb/routers/score_sets.py +++ b/src/mavedb/routers/score_sets.py @@ -23,6 +23,7 @@ from mavedb import deps from mavedb.data_providers.services import CSV_UPLOAD_S3_BUCKET_NAME, s3_client +from mavedb.db.timeouts import allow_long_statements from mavedb.lib.annotation.annotate import ( variant_functional_impact_statement, variant_pathogenicity_statement, @@ -1080,6 +1081,9 @@ def get_score_set_variants_csv( assert_permission(user_data, score_set, Action.READ) + # Full-table CSV builds legitimately outrun the API's default statement timeout. + allow_long_statements(db) + csv_str = get_score_set_variants_as_csv( db, score_set, @@ -1155,6 +1159,9 @@ def get_score_set_scores_csv( assert_permission(user_data, score_set, Action.READ) + # Full-table CSV builds legitimately outrun the API's default statement timeout. + allow_long_statements(db) + # Both score namespaces: this endpoint has always returned every score column the investigator # uploaded, and `scores` alone is now just the required one. csv_str = get_score_set_variants_as_csv( @@ -1177,7 +1184,7 @@ def get_score_set_scores_csv( }, summary="Get score set counts in CSV format", ) -async def get_score_set_counts_csv( +def get_score_set_counts_csv( *, urn: str, start: int = Query(default=None, description="Start index for pagination"), @@ -1223,6 +1230,9 @@ async def get_score_set_counts_csv( assert_permission(user_data, score_set, Action.READ) + # Full-table CSV builds legitimately outrun the API's default statement timeout. + allow_long_statements(db) + csv_str = get_score_set_variants_as_csv(db, score_set, ["counts"], False, start, limit, drop_unused_hgvs_columns) return StreamingResponse(iter([csv_str]), media_type="text/csv", headers=deprecated.response_headers) diff --git a/src/mavedb/server_main.py b/src/mavedb/server_main.py index 7f2233e28..5000b8a88 100644 --- a/src/mavedb/server_main.py +++ b/src/mavedb/server_main.py @@ -20,6 +20,8 @@ ) from mavedb import __version__ +from mavedb.db.session import engine +from mavedb.db.timeouts import SLOW_STATEMENT_SECONDS, apply_api_timeouts, log_slow_statements from mavedb.lib.exceptions import ( AmbiguousIdentifierError, HGNCGeneNotFoundError, @@ -76,6 +78,9 @@ # an instance of the related class has been created. configure_mappers() +apply_api_timeouts(engine) +log_slow_statements(engine, SLOW_STATEMENT_SECONDS) + # forward_retired_urns is applied to every route, so that one implementation forwards a read of a URN # publication has retired wherever that URN points: a record, or any of its sub-resources. app = FastAPI(dependencies=[Depends(forward_retired_urns)]) diff --git a/tests/db/__init__.py b/tests/db/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/db/test_timeouts.py b/tests/db/test_timeouts.py new file mode 100644 index 000000000..fb82e5663 --- /dev/null +++ b/tests/db/test_timeouts.py @@ -0,0 +1,105 @@ +# ruff: noqa: E402 + +import logging + +import pytest + +pytest.importorskip("psycopg2") + +from sqlalchemy import create_engine, text +from sqlalchemy.exc import OperationalError +from sqlalchemy.orm import Session +from sqlalchemy.pool import NullPool + +from mavedb.db import timeouts +from mavedb.db.timeouts import ( + CSV_STATEMENT_TIMEOUT_SECONDS, + allow_long_statements, + apply_api_timeouts, + log_slow_statements, +) + + +def _show(connection_or_session, setting: str) -> str: + return connection_or_session.execute(text(f"SHOW {setting}")).scalar() + + +@pytest.fixture +def engine(session): + return session.get_bind() + + +@pytest.mark.unit +class TestAllowLongStatements: + def test_raises_statement_timeout_for_the_transaction(self, session): + allow_long_statements(session) + + assert _show(session, "statement_timeout") == f"{CSV_STATEMENT_TIMEOUT_SECONDS}s" + + +@pytest.mark.unit +class TestApplyApiTimeouts: + @pytest.fixture + def api_engine(self, engine): + api_engine = create_engine(engine.url, poolclass=NullPool) + apply_api_timeouts(api_engine) + apply_api_timeouts(api_engine) # idempotent + return api_engine + + def test_new_connections_carry_the_api_limits(self, api_engine): + with api_engine.connect() as conn: + assert _show(conn, "statement_timeout") == "30s" + assert _show(conn, "lock_timeout") == "5s" + assert _show(conn, "idle_in_transaction_session_timeout") == "1min" + + def test_csv_override_raises_and_then_restores_the_limit(self, api_engine): + with Session(api_engine) as session: + allow_long_statements(session) + assert _show(session, "statement_timeout") == f"{CSV_STATEMENT_TIMEOUT_SECONDS}s" + session.rollback() + assert _show(session, "statement_timeout") == "30s" + + def test_zero_disables_the_statement_timeout_everywhere(self, engine, monkeypatch): + monkeypatch.setattr(timeouts, "API_STATEMENT_TIMEOUT_SECONDS", 0) + disabled_engine = create_engine(engine.url, poolclass=NullPool) + apply_api_timeouts(disabled_engine) + + with Session(disabled_engine) as session: + assert _show(session, "statement_timeout") == "0" + allow_long_statements(session) + assert _show(session, "statement_timeout") == "0" + + def test_other_engines_are_unbounded(self, engine): + with engine.connect() as conn: + assert _show(conn, "statement_timeout") == "0" + + +@pytest.mark.unit +class TestLogSlowStatements: + def test_logs_a_slow_statement_without_its_parameters(self, session, engine, caplog): + log_slow_statements(engine, threshold_seconds=0.05) + + with caplog.at_level(logging.WARNING, logger="mavedb.db.timeouts"): + session.execute(text("SELECT pg_sleep(0.1), :secret"), {"secret": "hunter2"}) + + messages = [record.message for record in caplog.records] + assert any("Slow database statement" in message and "pg_sleep" in message for message in messages) + assert not any("hunter2" in message for message in messages) + + def test_does_not_log_a_fast_statement(self, session, engine, caplog): + log_slow_statements(engine, threshold_seconds=5) + + with caplog.at_level(logging.WARNING, logger="mavedb.db.timeouts"): + session.execute(text("SELECT 1")) + + assert not caplog.records + + def test_logs_a_statement_cancelled_by_the_timeout(self, session, engine, caplog): + log_slow_statements(engine, threshold_seconds=0.05) + session.execute(text("SET LOCAL statement_timeout = '100ms'")) + + with caplog.at_level(logging.WARNING, logger="mavedb.db.timeouts"): + with pytest.raises(OperationalError): + session.execute(text("SELECT pg_sleep(1)")) + + assert any("Slow database statement" in record.message for record in caplog.records) diff --git a/tests/lib/middleware/test_errors.py b/tests/lib/middleware/test_errors.py index 76a14b0e2..698b114b8 100644 --- a/tests/lib/middleware/test_errors.py +++ b/tests/lib/middleware/test_errors.py @@ -15,6 +15,8 @@ pytest.importorskip("fastapi") from fastapi.testclient import TestClient +from sqlalchemy.exc import OperationalError +from sqlalchemy.exc import TimeoutError as PoolTimeoutError from mavedb.server_main import app @@ -41,6 +43,34 @@ def boom(): app.router.routes.remove(route) +class FakePostgresError(Exception): + """Stands in for a psycopg2 error, which carries the server's SQLSTATE as ``pgcode``.""" + + def __init__(self, pgcode): + super().__init__(pgcode) + self.pgcode = pgcode + + +def postgres_error(pgcode): + return OperationalError("SELECT 1", {}, FakePostgresError(pgcode)) + + +@pytest.fixture +def database_error_route(): + """Register a route that raises whichever error the test sets on ``raised["error"]``.""" + raised = {} + + @app.get(BOOM_PATH) + def boom(): + raise raised["error"] + + route = app.router.routes[-1] + try: + yield raised + finally: + app.router.routes.remove(route) + + @pytest.fixture def slack_error(): with patch("mavedb.lib.middleware.errors.send_slack_error") as mocked: @@ -85,3 +115,38 @@ def test_successful_request_is_untouched(self, slack_error): assert response.status_code == 200 assert response.headers.get("access-control-allow-origin") in ("*", TEST_ORIGIN) slack_error.assert_not_called() + + +@pytest.mark.unit +class TestDatabaseTimeouts: + @pytest.mark.parametrize( + "error", + [postgres_error("57014"), postgres_error("55P03"), PoolTimeoutError("QueuePool limit reached")], + ids=["statement_timeout", "lock_timeout", "pool_timeout"], + ) + def test_timeout_returns_503_with_cors_headers(self, database_error_route, slack_error, error): + database_error_route["error"] = error + + with TestClient(app) as tc: + response = tc.get(BOOM_PATH, headers={"Origin": TEST_ORIGIN}) + + assert response.status_code == 503 + assert response.headers.get("access-control-allow-origin") in ("*", TEST_ORIGIN) + assert response.json()["correlation_id"] + + def test_timeout_does_not_alert_slack(self, database_error_route, slack_error): + database_error_route["error"] = postgres_error("57014") + + with TestClient(app) as tc: + tc.get(BOOM_PATH) + + slack_error.assert_not_called() + + def test_other_database_errors_are_still_500(self, database_error_route, slack_error): + database_error_route["error"] = postgres_error("23505") + + with TestClient(app) as tc: + response = tc.get(BOOM_PATH) + + assert response.status_code == 500 + slack_error.assert_called_once() diff --git a/tests/lib/test_authentication.py b/tests/lib/test_authentication.py index 80ee503cb..248e34642 100644 --- a/tests/lib/test_authentication.py +++ b/tests/lib/test_authentication.py @@ -1,5 +1,6 @@ # ruff: noqa: E402 +from datetime import datetime, timedelta from unittest.mock import patch import pytest @@ -9,7 +10,11 @@ cdot = pytest.importorskip("cdot") fastapi = pytest.importorskip("fastapi") -from mavedb.lib.authentication import get_current_user, get_current_user_data_from_api_key +from mavedb.lib.authentication import ( + LAST_LOGIN_REFRESH_INTERVAL, + get_current_user, + get_current_user_data_from_api_key, +) from mavedb.models.enums.user_role import UserRole from mavedb.models.user import User from tests.helpers.constants import ADMIN_USER, ADMIN_USER_DECODED_JWT, TEST_USER, TEST_USER_DECODED_JWT @@ -17,61 +22,53 @@ from tests.helpers.util.user import mark_user_inactive -@pytest.mark.asyncio -async def test_get_current_user_data_from_key_valid_token(session, setup_lib_db): +def test_get_current_user_data_from_key_valid_token(session, setup_lib_db): access_key = create_api_key_for_user(session, TEST_USER["username"]) - user_data = await get_current_user_data_from_api_key(session, access_key) + user_data = get_current_user_data_from_api_key(session, access_key) assert user_data.user.username == TEST_USER["username"] -@pytest.mark.asyncio -async def test_get_current_user_data_from_key_invalid_token(session, setup_lib_db): +def test_get_current_user_data_from_key_invalid_token(session, setup_lib_db): access_key = create_api_key_for_user(session, TEST_USER["username"]) - user_data = await get_current_user_data_from_api_key(session, f"invalid_{access_key}") + user_data = get_current_user_data_from_api_key(session, f"invalid_{access_key}") assert user_data is None -@pytest.mark.asyncio -async def test_get_current_user_data_from_key_nonetype_token(session, setup_lib_db): +def test_get_current_user_data_from_key_nonetype_token(session, setup_lib_db): create_api_key_for_user(session, TEST_USER["username"]) - user_data = await get_current_user_data_from_api_key(session, None) + user_data = get_current_user_data_from_api_key(session, None) assert user_data is None -@pytest.mark.asyncio -async def test_get_current_user_via_api_key(session, setup_lib_db): +def test_get_current_user_via_api_key(session, setup_lib_db): access_key = create_api_key_for_user(session, TEST_USER["username"]) - user_data = await get_current_user_data_from_api_key(session, access_key) + user_data = get_current_user_data_from_api_key(session, access_key) - user_data = await get_current_user(user_data, None, session, None) + user_data = get_current_user(user_data, None, session, None) assert user_data.user.username == TEST_USER["username"] -@pytest.mark.asyncio -async def test_get_current_user_via_token_payload(session, setup_lib_db): - user_data = await get_current_user(None, TEST_USER_DECODED_JWT, session, None) +def test_get_current_user_via_token_payload(session, setup_lib_db): + user_data = get_current_user(None, TEST_USER_DECODED_JWT, session, None) assert user_data.user.username == TEST_USER["username"] -@pytest.mark.asyncio -async def test_get_current_user_no_api_no_jwt(session, setup_lib_db): - user_data = await get_current_user(None, None, session, None) +def test_get_current_user_no_api_no_jwt(session, setup_lib_db): + user_data = get_current_user(None, None, session, None) assert user_data is None -@pytest.mark.asyncio -async def test_get_current_user_no_username(session, setup_lib_db): +def test_get_current_user_no_username(session, setup_lib_db): # Remove the username key from the JWT jwt_without_sub = TEST_USER_DECODED_JWT.copy() jwt_without_sub.pop("sub") - user_data = await get_current_user(None, jwt_without_sub, session, None) + user_data = get_current_user(None, jwt_without_sub, session, None) assert user_data is None -@pytest.mark.asyncio @pytest.mark.parametrize("with_email", [True, False]) -async def test_get_current_user_nonexistent_user(session, setup_lib_db, with_email): +def test_get_current_user_nonexistent_user(session, setup_lib_db, with_email): new_user_jwt = { "sub": "5555-5555-5555-5555", "given_name": "Temporary", @@ -81,7 +78,7 @@ async def test_get_current_user_nonexistent_user(session, setup_lib_db, with_ema email = "tempemail@test.com" if with_email else None with patch("mavedb.lib.authentication.fetch_orcid_user_email") as orc_email: orc_email.return_value = email - user_data = await get_current_user(None, new_user_jwt, session, None) + user_data = get_current_user(None, new_user_jwt, session, None) orc_email.assert_called_once() assert user_data.user.username == new_user_jwt["sub"] @@ -93,39 +90,34 @@ async def test_get_current_user_nonexistent_user(session, setup_lib_db, with_ema session.query(User).filter(User.username == new_user_jwt["sub"]).one() -@pytest.mark.asyncio -async def test_get_current_user_user_is_inactive(session, setup_lib_db): +def test_get_current_user_user_is_inactive(session, setup_lib_db): mark_user_inactive(session, TEST_USER["username"]) - user_data = await get_current_user(None, TEST_USER_DECODED_JWT, session, None) + user_data = get_current_user(None, TEST_USER_DECODED_JWT, session, None) assert user_data is None -@pytest.mark.asyncio -async def test_get_current_user_set_active_roles(session, setup_lib_db): - user_data = await get_current_user(None, ADMIN_USER_DECODED_JWT, session, "admin") +def test_get_current_user_set_active_roles(session, setup_lib_db): + user_data = get_current_user(None, ADMIN_USER_DECODED_JWT, session, "admin") assert user_data.user.username == ADMIN_USER["username"] assert UserRole.admin in user_data.active_roles -@pytest.mark.asyncio -async def test_get_current_user_user_with_invalid_role_membership(session, setup_lib_db): +def test_get_current_user_user_with_invalid_role_membership(session, setup_lib_db): with pytest.raises(Exception) as exc_info: - await get_current_user(None, TEST_USER_DECODED_JWT, session, "admin") + get_current_user(None, TEST_USER_DECODED_JWT, session, "admin") assert "This user is not a member of the requested acting role." in str(exc_info.value.detail) -@pytest.mark.asyncio -async def test_get_current_user_user_extraneous_roles(session, setup_lib_db): - user_data = await get_current_user(None, TEST_USER_DECODED_JWT, session, "extra_role") +def test_get_current_user_user_extraneous_roles(session, setup_lib_db): + user_data = get_current_user(None, TEST_USER_DECODED_JWT, session, "extra_role") assert user_data.user.username == TEST_USER["username"] assert user_data.active_roles == [] -@pytest.mark.asyncio -async def test_get_current_user_concurrent_first_login_integrity_error_returns_existing_user(session, setup_lib_db): +def test_get_current_user_concurrent_first_login_integrity_error_returns_existing_user(session, setup_lib_db): """ Simulate two servers racing on first login: the commit raises IntegrityError because a concurrent request already inserted the row. The handler should roll back and return the @@ -162,7 +154,7 @@ def fake_commit(): session.commit = fake_commit with patch("mavedb.lib.authentication.fetch_orcid_user_email", return_value=None): - user_data = await get_current_user(None, new_user_jwt, session, None) + user_data = get_current_user(None, new_user_jwt, session, None) assert user_data is not None assert user_data.user.username == new_user_jwt["sub"] @@ -170,3 +162,48 @@ def fake_commit(): # Only one user record should exist in the database. users = session.query(User).filter(User.username == new_user_jwt["sub"]).all() assert len(users) == 1 + + +def _set_login_state(session, username, *, last_login, is_first_login=False): + user = session.query(User).filter(User.username == username).one() + user.last_login = last_login + user.is_first_login = is_first_login + session.commit() + return user + + +def test_get_current_user_skips_write_when_last_login_is_recent(session, setup_lib_db): + recent = datetime.now() - timedelta(minutes=1) + _set_login_state(session, TEST_USER["username"], last_login=recent) + + with patch.object(session, "commit", wraps=session.commit) as commit: + user_data = get_current_user(None, TEST_USER_DECODED_JWT, session, None) + + assert user_data.user.username == TEST_USER["username"] + commit.assert_not_called() + assert user_data.user.last_login == recent + + +def test_get_current_user_refreshes_stale_last_login(session, setup_lib_db): + stale = datetime.now() - LAST_LOGIN_REFRESH_INTERVAL - timedelta(minutes=1) + _set_login_state(session, TEST_USER["username"], last_login=stale) + + user_data = get_current_user(None, TEST_USER_DECODED_JWT, session, None) + + assert user_data.user.last_login > stale + + +def test_get_current_user_writes_when_last_login_is_missing(session, setup_lib_db): + _set_login_state(session, TEST_USER["username"], last_login=None) + + user_data = get_current_user(None, TEST_USER_DECODED_JWT, session, None) + + assert user_data.user.last_login is not None + + +def test_get_current_user_clears_first_login_flag_even_when_last_login_is_recent(session, setup_lib_db): + _set_login_state(session, TEST_USER["username"], last_login=datetime.now(), is_first_login=True) + + user_data = get_current_user(None, TEST_USER_DECODED_JWT, session, None) + + assert user_data.user.is_first_login is False diff --git a/tests/routers/test_score_set.py b/tests/routers/test_score_set.py index 6f575267d..3b3bd33d9 100644 --- a/tests/routers/test_score_set.py +++ b/tests/routers/test_score_set.py @@ -3834,6 +3834,23 @@ def test_download_counts_file(session, data_provider, client, setup_router_db, d assert "hgvs_splice" not in columns +@pytest.mark.parametrize("path", ["scores", "counts", "variants/data"]) +def test_csv_routes_raise_the_statement_timeout(session, data_provider, client, setup_router_db, data_files, path): + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion( + client, session, data_provider, score_set, data_files / "scores.csv", data_files / "counts.csv" + ) + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published_score_set = publish_score_set(client, score_set["urn"]) + + with patch("mavedb.routers.score_sets.allow_long_statements") as allow_long_statements: + response = client.get(f"/api/v1/score-sets/{published_score_set['urn']}/{path}") + + assert response.status_code == 200 + allow_long_statements.assert_called_once() + + # Deprecated query-parameter aliases. Galaxy and other external tooling call these endpoints, so the old # names keep working for a release rather than being silently ignored. def test_deprecated_drop_na_columns_still_drops_unused_hgvs_columns( From d49743123d98d9d4d7fedc89e8b53a01fe4b8370 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 29 Sep 2026 15:30:37 -0700 Subject: [PATCH 183/252] test(csv): skip the build limiter tests without FastAPI The core-dependency CI job doesn't install FastAPI, so importing it at module level failed collection. --- tests/lib/csv/test_build_limiter.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/lib/csv/test_build_limiter.py b/tests/lib/csv/test_build_limiter.py index a1792d8a5..8c69ddae5 100644 --- a/tests/lib/csv/test_build_limiter.py +++ b/tests/lib/csv/test_build_limiter.py @@ -1,6 +1,11 @@ +# ruff: noqa: E402 + import threading import pytest + +pytest.importorskip("fastapi") + from fastapi import HTTPException from mavedb.lib.csv import build_limiter From 72c3e8d46a38966d85260cc2139197628cfb1721 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 29 Sep 2026 20:28:59 -0700 Subject: [PATCH 184/252] fix(migrations): build functional classification variant index without CONCURRENTLY Follows the migration rules in #877: no CONCURRENTLY and no autocommit_block(). The migration now uses a plain CREATE INDEX IF NOT EXISTS, and the docstring gives the by-hand CONCURRENTLY statement to run before deploying to a populated database. --- ...7731_index_functional_classification_variant_.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/alembic/versions/15c40c367731_index_functional_classification_variant_.py b/alembic/versions/15c40c367731_index_functional_classification_variant_.py index 773a5ed9c..5140e3b7a 100644 --- a/alembic/versions/15c40c367731_index_functional_classification_variant_.py +++ b/alembic/versions/15c40c367731_index_functional_classification_variant_.py @@ -7,8 +7,11 @@ with a nontrivial number of variants. See mavedb-api#677 ("unpublished score set deletion often fails"). -Built CONCURRENTLY, outside a transaction, so this doesn't hold an exclusive lock on the table for -the build's duration. +Building it takes a lock that blocks writes to the table, so in a populated database build it by hand +first, which makes this migration a no-op: + + CREATE INDEX CONCURRENTLY IF NOT EXISTS ix_score_calib_func_classification_variants_variant_id + ON score_calibration_functional_classification_variants (variant_id); Revision ID: 15c40c367731 Revises: c4b18d0f7a92 @@ -29,10 +32,8 @@ def upgrade(): - with op.get_context().autocommit_block(): - op.execute(f"CREATE INDEX CONCURRENTLY IF NOT EXISTS {INDEX_NAME} ON {TABLE_NAME} (variant_id)") + op.execute(f"CREATE INDEX IF NOT EXISTS {INDEX_NAME} ON {TABLE_NAME} (variant_id)") def downgrade(): - with op.get_context().autocommit_block(): - op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {INDEX_NAME}") + op.execute(f"DROP INDEX IF EXISTS {INDEX_NAME}") From 0e55109daa2083d6484ae54861e200621a5a8764 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 29 Sep 2026 20:47:10 -0700 Subject: [PATCH 185/252] chore(migrations): chain calibration controls migration after the variant number index The 2026.3.0 migrations run as one linear chain, so 024370c4bca7 now follows #899's f6752571d14d instead of forking from a7f3c2e9b104. Until #899 merges, this branch's migration history has a missing parent. --- .../024370c4bca7_add_calibration_controls_table_and_phi_.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/alembic/versions/024370c4bca7_add_calibration_controls_table_and_phi_.py b/alembic/versions/024370c4bca7_add_calibration_controls_table_and_phi_.py index b0999f4ef..996badf85 100644 --- a/alembic/versions/024370c4bca7_add_calibration_controls_table_and_phi_.py +++ b/alembic/versions/024370c4bca7_add_calibration_controls_table_and_phi_.py @@ -1,7 +1,7 @@ """add calibration controls, mondo terms, and phi/disease columns Revision ID: 024370c4bca7 -Revises: a7f3c2e9b104 +Revises: f6752571d14d Create Date: 2026-09-15 16:01:06.013964 """ @@ -12,7 +12,7 @@ # revision identifiers, used by Alembic. revision = "024370c4bca7" -down_revision = "a7f3c2e9b104" +down_revision = "f6752571d14d" branch_labels = None depends_on = None From 05d864f832288c4988d2eb6f342912b0db5c6a33 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 29 Sep 2026 20:51:43 -0700 Subject: [PATCH 186/252] fix(variants): cascade mapping records and annotation events when a variant is deleted Deleting a mapped score set returned a 500, and re-uploading its variants failed, because the foreign keys from mapping_records and annotation_event to variants didn't cascade. Both now do, as variant_annotation_status did. Allele-subject events keep RESTRICT. 45e8ec1db15e hasn't run anywhere yet, so it's edited in place. The annotation_event key was created by a migration staging has already run, so it changes in the new migration 393764b9e833. --- ...cascade_annotation_event_variant_delete.py | 43 ++++++++++ ...5e_carry_score_set_id_on_mapping_tables.py | 4 +- src/mavedb/models/annotation_event.py | 2 +- src/mavedb/models/mapping_record.py | 1 + tests/routers/test_score_set.py | 36 +++++++++ .../jobs/variant_processing/test_creation.py | 78 +++++++++++++++++++ 6 files changed, 162 insertions(+), 2 deletions(-) create mode 100644 alembic/versions/393764b9e833_cascade_annotation_event_variant_delete.py diff --git a/alembic/versions/393764b9e833_cascade_annotation_event_variant_delete.py b/alembic/versions/393764b9e833_cascade_annotation_event_variant_delete.py new file mode 100644 index 000000000..95450a795 --- /dev/null +++ b/alembic/versions/393764b9e833_cascade_annotation_event_variant_delete.py @@ -0,0 +1,43 @@ +"""cascade annotation events when their variant is deleted + +Revision ID: 393764b9e833 +Revises: 45e8ec1db15e +Create Date: 2026-09-29 + +The mapping job records a variant-subject ``annotation_event`` for every variant it maps. With the foreign key +on ``RESTRICT``, deleting a mapped score set or re-uploading its variants failed on that key. Variant-subject +events now go with their variant, as ``variant_annotation_status`` rows do. Allele-subject events keep +``RESTRICT``: alleles are shared across score sets and are never deleted with one. +""" + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "393764b9e833" +down_revision = "45e8ec1db15e" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.drop_constraint("annotation_event_variant_id_fkey", "annotation_event", type_="foreignkey") + op.create_foreign_key( + "annotation_event_variant_id_fkey", + "annotation_event", + "variants", + ["variant_id"], + ["id"], + ondelete="CASCADE", + ) + + +def downgrade() -> None: + op.drop_constraint("annotation_event_variant_id_fkey", "annotation_event", type_="foreignkey") + op.create_foreign_key( + "annotation_event_variant_id_fkey", + "annotation_event", + "variants", + ["variant_id"], + ["id"], + ondelete="RESTRICT", + ) diff --git a/alembic/versions/45e8ec1db15e_carry_score_set_id_on_mapping_tables.py b/alembic/versions/45e8ec1db15e_carry_score_set_id_on_mapping_tables.py index b0a68c059..92d88dd81 100644 --- a/alembic/versions/45e8ec1db15e_carry_score_set_id_on_mapping_tables.py +++ b/alembic/versions/45e8ec1db15e_carry_score_set_id_on_mapping_tables.py @@ -10,7 +10,8 @@ Each column sits under a composite foreign key to its parent's id and score set, so it can't disagree with the variant it belongs to. The composite keys replace the single-column ``variant_id`` and -``mapping_record_id`` foreign keys. +``mapping_record_id`` foreign keys. Both cascade on delete, so deleting a variant (on re-upload or score set +deletion) removes its mapping records and their allele links. **Before deploying to production**, build the ``variants`` index by hand, since a migration can't build it concurrently (#877). The ``IF NOT EXISTS`` below then does nothing there, and builds the index on an empty @@ -45,6 +46,7 @@ def upgrade() -> None: "variants", ["variant_id", "score_set_id"], ["id", "scoreset_id"], + ondelete="CASCADE", ) op.execute("ALTER TABLE mapping_record_alleles ADD COLUMN score_set_id integer") diff --git a/src/mavedb/models/annotation_event.py b/src/mavedb/models/annotation_event.py index 7b5f69b7f..5eba5ea4a 100644 --- a/src/mavedb/models/annotation_event.py +++ b/src/mavedb/models/annotation_event.py @@ -76,7 +76,7 @@ class AnnotationEvent(Base): # Exactly one is set, per ck_annotation_event_subject. variant_id: Mapped[Optional[int]] = mapped_column( - Integer, ForeignKey("variants.id", ondelete="RESTRICT"), nullable=True + Integer, ForeignKey("variants.id", ondelete="CASCADE"), nullable=True ) allele_id: Mapped[Optional[int]] = mapped_column( Integer, ForeignKey("alleles.id", ondelete="RESTRICT"), nullable=True diff --git a/src/mavedb/models/mapping_record.py b/src/mavedb/models/mapping_record.py index ad7d0151c..e0d952788 100644 --- a/src/mavedb/models/mapping_record.py +++ b/src/mavedb/models/mapping_record.py @@ -107,6 +107,7 @@ def _transcript_expression(cls): ["variants.id", "variants.scoreset_id"], name="fk_mapping_records_variant_score_set", link_to_name=True, + ondelete="CASCADE", ), # Target of mapping_record_alleles' composite foreign key. Index("uq_mapping_records_id_score_set", id, score_set_id, unique=True), diff --git a/tests/routers/test_score_set.py b/tests/routers/test_score_set.py index e421018d1..3b5d65d72 100644 --- a/tests/routers/test_score_set.py +++ b/tests/routers/test_score_set.py @@ -22,11 +22,15 @@ from mavedb.lib.annotation.exceptions import MappingDataDoesntExistException from mavedb.lib.exceptions import NonexistentOrcidUserError from mavedb.lib.validation.urn_re import MAVEDB_EXPERIMENT_URN_RE, MAVEDB_SCORE_SET_URN_RE, MAVEDB_TMP_URN_RE +from mavedb.models.annotation_event import AnnotationEvent from mavedb.models.clinvar_allele_link import ClinvarAlleleLink +from mavedb.models.enums.annotation_type import AnnotationType +from mavedb.models.enums.disposition import Disposition from mavedb.models.enums.processing_state import ProcessingState from mavedb.models.enums.target_category import TargetCategory from mavedb.models.experiment import Experiment as ExperimentDbModel from mavedb.models.job_run import JobRun +from mavedb.models.mapping_record import MappingRecord from mavedb.models.pipeline import Pipeline from mavedb.models.gnomad_variant import GnomADVariant as GnomADVariantDbModel from mavedb.models.score_set import ScoreSet as ScoreSetDbModel @@ -3213,6 +3217,38 @@ def test_can_delete_own_private_scoreset(session, data_provider, client, setup_r assert response.status_code == 200 +def test_can_delete_own_private_scoreset_with_mapped_variants( + session, data_provider, client, setup_router_db, data_files +): + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") + variant = session.scalars( + select(VariantDbModel).join(ScoreSetDbModel).where(ScoreSetDbModel.urn == score_set["urn"]) + ).first() + variant_ids = [variant.id] + session.add( + MappingRecord( + variant_id=variant.id, score_set_id=variant.score_set_id, assay_level="cdna", mapping_api_version="test.0.0" + ) + ) + session.add( + AnnotationEvent( + annotation_type=AnnotationType.VRS_MAPPING, + variant_id=variant.id, + disposition=Disposition.PRESENT, + reason="mapped", + ) + ) + session.commit() + + response = client.delete(f"/api/v1/score-sets/{score_set['urn']}") + + assert response.status_code == 200 + assert not session.scalars(select(MappingRecord).where(MappingRecord.variant_id.in_(variant_ids))).first() + assert not session.scalars(select(AnnotationEvent).where(AnnotationEvent.variant_id.in_(variant_ids))).first() + + def test_cannot_delete_own_published_scoreset(session, data_provider, client, setup_router_db, data_files): experiment = create_experiment(client) score_set = create_seq_score_set(client, experiment["urn"]) diff --git a/tests/worker/jobs/variant_processing/test_creation.py b/tests/worker/jobs/variant_processing/test_creation.py index d0d37562a..f7f2d7836 100644 --- a/tests/worker/jobs/variant_processing/test_creation.py +++ b/tests/worker/jobs/variant_processing/test_creation.py @@ -7,10 +7,14 @@ import math from unittest.mock import ANY, MagicMock, call, patch +from mavedb.models.annotation_event import AnnotationEvent +from mavedb.models.enums.annotation_type import AnnotationType +from mavedb.models.enums.disposition import Disposition from mavedb.models.enums.job_pipeline import JobStatus, PipelineStatus from mavedb.models.enums.mapping_state import MappingState from mavedb.models.enums.processing_state import ProcessingState from mavedb.models.job_run import JobRun +from mavedb.models.mapping_record import MappingRecord from mavedb.models.pipeline import Pipeline from mavedb.models.variant import Variant from mavedb.worker.jobs.variant_processing.creation import create_variants_for_score_set @@ -286,6 +290,80 @@ async def test_create_variants_for_score_set_removes_existing_variants_before_cr session.refresh(sample_score_set) assert sample_score_set.num_variants == 0 # Updated after creation + async def test_create_variants_for_score_set_removes_existing_mapped_variants_before_creation( + self, + session, + with_independent_processing_runs, + with_populated_domain_data, + mock_worker_ctx, + mock_s3_client, + create_variants_sample_params, + sample_score_dataframe, + sample_count_dataframe, + sample_score_set, + sample_independent_variant_creation_run, + ): + # An existing variant that has been mapped: its mapping record and annotation event must not + # block its removal. + sample_score_set.num_variants = 1 + variant = Variant(data={}, score_set_id=sample_score_set.id) + session.add(variant) + session.commit() + session.add( + MappingRecord( + variant_id=variant.id, + score_set_id=sample_score_set.id, + assay_level="cdna", + mapping_api_version="test.0.0", + ) + ) + session.add( + AnnotationEvent( + annotation_type=AnnotationType.VRS_MAPPING, + variant_id=variant.id, + disposition=Disposition.PRESENT, + reason="mapped", + ) + ) + session.commit() + variant_id = variant.id + + with ( + patch.object(mock_s3_client, "download_fileobj", return_value=None), + # Mock pd.read_csv to return sample dataframes + patch( + "mavedb.worker.jobs.variant_processing.creation.pd.read_csv", + side_effect=[sample_score_dataframe, sample_count_dataframe], + ), + patch( + "mavedb.worker.jobs.variant_processing.creation.validate_and_standardize_dataframe_pair", + return_value=( + sample_score_dataframe, + sample_count_dataframe, + create_variants_sample_params["score_columns_metadata"], + create_variants_sample_params["count_columns_metadata"], + ), + ), + patch( + "mavedb.worker.jobs.variant_processing.creation.create_variants_data", + return_value=[MagicMock(spec=Variant)], + ), + patch("mavedb.worker.jobs.variant_processing.creation.create_variants", return_value=None), + ): + await create_variants_for_score_set( + mock_worker_ctx, + sample_independent_variant_creation_run.id, + JobManager(session, mock_worker_ctx["redis"], sample_independent_variant_creation_run.id), + ) + + # Verify that existing variants have been removed + remaining_variants = session.query(Variant).filter(Variant.score_set_id == sample_score_set.id).all() + assert len(remaining_variants) == 0 + assert session.query(MappingRecord).filter(MappingRecord.score_set_id == sample_score_set.id).count() == 0 + assert session.query(AnnotationEvent).filter(AnnotationEvent.variant_id == variant_id).count() == 0 + session.refresh(sample_score_set) + assert sample_score_set.num_variants == 0 # Updated after creation + async def test_create_variants_for_score_set_updates_processing_state( self, session, From 33456618a9e512abb400792ee7aa5db3569895f4 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 29 Sep 2026 21:13:43 -0700 Subject: [PATCH 187/252] test(worker): build variant URNs as # in RT fixtures #899's index on variants requires every variant URN to end in '#'. The RT worker fixtures used URNs like 'variant:1' and 'urn:variant:test-...', which fail against that index once the branches merge. Numbering follows #899: #1-#5 in conftest, #11 and up in tests, #100 and up in loops. --- tests/worker/jobs/conftest.py | 8 +- .../external_services/test_clingen_cache.py | 18 +++-- .../jobs/variant_processing/test_mapping.py | 14 ++-- .../test_reverse_translation.py | 76 +++++++++++-------- 4 files changed, 64 insertions(+), 52 deletions(-) diff --git a/tests/worker/jobs/conftest.py b/tests/worker/jobs/conftest.py index 6f90825d6..ac1cbe605 100644 --- a/tests/worker/jobs/conftest.py +++ b/tests/worker/jobs/conftest.py @@ -265,7 +265,7 @@ def setup_sample_variants_with_caid( # Add a variant and mapped variant to the database with a CAID variant = Variant( - urn="urn:variant:test-variant-with-caid", + urn=f"{score_set.urn}#1", score_set_id=score_set.id, hgvs_nt="NM_000000.1:c.1A>G", hgvs_pro="NP_000000.1:p.Met1Val", @@ -294,7 +294,7 @@ def setup_sample_alleles_with_caid(session, with_populated_domain_data, sample_l score_set = session.get(ScoreSet, sample_link_gnomad_variants_run.job_params["score_set_id"]) variant = Variant( - urn="urn:variant:test-variant-with-allele-caid", + urn=f"{score_set.urn}#2", score_set_id=score_set.id, hgvs_nt="NM_000000.1:c.1A>G", hgvs_pro="NP_000000.1:p.Met1Val", @@ -1162,7 +1162,7 @@ def setup_sample_alleles_for_vep(session, with_populated_domain_data, mock_worke score_set = session.get(ScoreSet, sample_populate_vep_run.job_params["score_set_id"]) variant = Variant( - urn="urn:variant:test-variant-for-vep", + urn=f"{score_set.urn}#4", score_set_id=score_set.id, hgvs_nt="NM_007294.4:c.5A>G", hgvs_pro="NP_009225.1:p.Cys2Tyr", @@ -1210,7 +1210,7 @@ def setup_sample_protein_allele_for_vep(session, with_populated_domain_data, moc score_set = session.get(ScoreSet, sample_populate_vep_run.job_params["score_set_id"]) variant = Variant( - urn="urn:variant:test-protein-variant-for-vep", + urn=f"{score_set.urn}#5", score_set_id=score_set.id, hgvs_pro="NP_009225.1:p.Val1696His", data={"hgvs_p": "NP_009225.1:p.Val1696His"}, diff --git a/tests/worker/jobs/external_services/test_clingen_cache.py b/tests/worker/jobs/external_services/test_clingen_cache.py index 625bb0c90..6bff657b9 100644 --- a/tests/worker/jobs/external_services/test_clingen_cache.py +++ b/tests/worker/jobs/external_services/test_clingen_cache.py @@ -20,14 +20,16 @@ pytestmark = pytest.mark.usefixtures("patch_db_session_ctxmgr") -def _make_allele_with_caid(session, score_set_id: int, urn_suffix: str, caid: str | None, vrs_digest: str) -> Allele: +def _make_allele_with_caid( + session, score_set_id: int, variant_number: int, caid: str | None, vrs_digest: str +) -> Allele: """Create a Variant → MappingRecord → Allele chain and return the Allele. get_alleles_for_score_set joins: Allele ← MappingRecordAllele (current) ← MappingRecord (current) ← Variant. Leaving valid_to=NULL on MappingRecord and MappingRecordAllele makes them current. """ variant = Variant( - urn=f"urn:variant:{urn_suffix}", + urn=f"{session.get(ScoreSet, score_set_id).urn}#{variant_number}", score_set_id=score_set_id, hgvs_nt="NM_000000.1:c.1A>G", hgvs_pro="NP_000000.1:p.Met1Val", @@ -101,12 +103,12 @@ async def test_warms_cache_for_variants_with_caids( # Three variants, two sharing the same CAID — should only warm 2 distinct IDs. # Two separate allele rows are needed (different VRS digests) to get 2 distinct CAIDs. - _make_allele_with_caid(session, score_set.id, "warm-test-0", "CA111111", "digest-warm-0") - _make_allele_with_caid(session, score_set.id, "warm-test-1", "CA222222", "digest-warm-1") + _make_allele_with_caid(session, score_set.id, 11, "CA111111", "digest-warm-0") + _make_allele_with_caid(session, score_set.id, 12, "CA222222", "digest-warm-1") # Third variant points to same allele as first (same CAID CA111111 via a fresh allele row # with the same digest — but since get_alleles_for_score_set returns per-variant rows and # caid dedup happens in the warmer, we can share the digest). - _make_allele_with_caid(session, score_set.id, "warm-test-2", "CA111111", "digest-warm-2") + _make_allele_with_caid(session, score_set.id, 13, "CA111111", "digest-warm-2") session.commit() mock_get_allele_data = AsyncMock(return_value={"some": "data"}) @@ -139,7 +141,7 @@ async def test_skips_null_and_multi_variant_caids( caids = ["CA333333", None, "CA-MULTI-001,CA-MULTI-002"] for i, caid in enumerate(caids): - _make_allele_with_caid(session, score_set.id, f"warm-filter-{i}", caid, f"digest-filter-{i}") + _make_allele_with_caid(session, score_set.id, 100 + i, caid, f"digest-filter-{i}") session.commit() mock_get_allele_data = AsyncMock(return_value={"some": "data"}) @@ -170,7 +172,7 @@ async def test_continues_on_individual_fetch_failure( score_set = session.get(ScoreSet, sample_warm_clingen_cache_job_run.job_params["score_set_id"]) for i, caid in enumerate(["CA444444", "CA555555"]): - _make_allele_with_caid(session, score_set.id, f"warm-fail-{i}", caid, f"digest-fail-{i}") + _make_allele_with_caid(session, score_set.id, 100 + i, caid, f"digest-fail-{i}") session.commit() # First call raises, second succeeds @@ -203,7 +205,7 @@ async def test_only_warms_current_mapped_variants( score_set = session.get(ScoreSet, sample_warm_clingen_cache_job_run.job_params["score_set_id"]) variant = Variant( - urn="urn:variant:warm-current-test", + urn=f"{score_set.urn}#11", score_set_id=score_set.id, hgvs_nt="NM_000000.1:c.30A>G", hgvs_pro="NP_000000.1:p.Met30Val", diff --git a/tests/worker/jobs/variant_processing/test_mapping.py b/tests/worker/jobs/variant_processing/test_mapping.py index 88d4af978..ca13d8505 100644 --- a/tests/worker/jobs/variant_processing/test_mapping.py +++ b/tests/worker/jobs/variant_processing/test_mapping.py @@ -744,14 +744,14 @@ async def dummy_mapping_job(): hgvs_nt="NM_000000.1:c.1A>G", hgvs_pro="NP_000000.1:p.Met1Val", data={}, - urn="variant:1", + urn=f"{sample_score_set.urn}#11", ) variant2 = Variant( score_set_id=sample_score_set.id, hgvs_nt="NM_000000.1:c.2G>T", hgvs_pro="NP_000000.1:p.Val2Leu", data={}, - urn="variant:2", + urn=f"{sample_score_set.urn}#12", ) session.add_all([variant1, variant2]) session.commit() @@ -843,14 +843,14 @@ async def dummy_mapping_job(): hgvs_nt="NM_000000.1:c.1A>G", hgvs_pro="NP_000000.1:p.Met1Val", data={}, - urn="variant:1", + urn=f"{sample_score_set.urn}#11", ) variant2 = Variant( score_set_id=sample_score_set.id, hgvs_nt="NM_000000.1:c.2G>T", hgvs_pro="NP_000000.1:p.Val2Leu", data={}, - urn="variant:2", + urn=f"{sample_score_set.urn}#12", ) session.add_all([variant1, variant2]) session.commit() @@ -926,14 +926,14 @@ async def dummy_mapping_job(): hgvs_nt="NM_000000.1:c.1A>G", hgvs_pro="NP_000000.1:p.Met1Val", data={}, - urn="variant:1", + urn=f"{sample_score_set.urn}#11", ) variant2 = Variant( score_set_id=sample_score_set.id, hgvs_nt="NM_000000.1:c.2G>T", hgvs_pro="NP_000000.1:p.Val2Leu", data={}, - urn="variant:2", + urn=f"{sample_score_set.urn}#12", ) session.add_all([variant1, variant2]) session.commit() @@ -963,7 +963,7 @@ async def dummy_mapping_job(): # Verify that both variants have post-mapped data. I'm comfortable assuming the # data is correct given our layer permutation tests above. - for urn in ["variant:1", "variant:2"]: + for urn in [f"{sample_score_set.urn}#11", f"{sample_score_set.urn}#12"]: mapping_record = session.query(MappingRecord).filter(MappingRecord.variant.has(urn=urn)).one_or_none() assert mapping_record is not None assert authoritative_allele_for(session, mapping_record) is not None diff --git a/tests/worker/jobs/variant_processing/test_reverse_translation.py b/tests/worker/jobs/variant_processing/test_reverse_translation.py index 0cb425563..d6bb54955 100644 --- a/tests/worker/jobs/variant_processing/test_reverse_translation.py +++ b/tests/worker/jobs/variant_processing/test_reverse_translation.py @@ -325,7 +325,7 @@ async def test_same_day_remap_without_a_cdna_row_skips_rather_than_binding_a_sta earlier same-day run's cdna transcript, which the day-granular key alone could not tell apart.""" variant = Variant( score_set_id=sample_score_set.id, - urn="variant:1", + urn=f"{sample_score_set.urn}#11", hgvs_nt="NM_000000.1:c.1A>G", hgvs_pro="NP_000000.1:p.Met1Val", data={}, @@ -380,7 +380,7 @@ async def test_cis_phased_assay_hgvs_is_skipped_before_the_engine( engine (which would otherwise count it as a failure the drop gate reads as a regression).""" variant = Variant( score_set_id=sample_score_set.id, - urn="variant:1", + urn=f"{sample_score_set.urn}#11", hgvs_nt="NM_000000.1:c.1A>G", hgvs_pro="NP_000000.1:p.Met1Val", data={}, @@ -429,7 +429,7 @@ async def test_falls_back_to_mapped_date_when_the_record_has_no_job_run_id( via the day-granular (target_gene_id, mapped_date) fallback rather than skipping.""" variant = Variant( score_set_id=sample_score_set.id, - urn="variant:1", + urn=f"{sample_score_set.urn}#11", hgvs_nt="NM_000000.1:c.1A>G", hgvs_pro="NP_000000.1:p.Met1Val", data={}, @@ -476,7 +476,7 @@ async def test_creates_genomic_and_coding_candidate_alleles( projection pair (coding + genomic projection) share one projection_group.""" variant = Variant( score_set_id=sample_score_set.id, - urn="variant:1", + urn=f"{sample_score_set.urn}#11", hgvs_nt="NM_000000.1:c.1A>G", hgvs_pro="NP_000000.1:p.Met1Val", data={}, @@ -547,7 +547,7 @@ async def test_transcript_is_queryable_via_derived_expression( the accession of the populated HGVS column, with no stored transcript column.""" variant = Variant( score_set_id=sample_score_set.id, - urn="variant:1", + urn=f"{sample_score_set.urn}#11", hgvs_nt="NM_000000.1:c.1A>G", hgvs_pro="NP_000000.1:p.Met1Val", data={}, @@ -590,7 +590,7 @@ async def test_protein_consequence_is_persisted_as_a_protein_allele( belongs to no projection pair, so its link's projection_group is NULL.""" variant = Variant( score_set_id=sample_score_set.id, - urn="variant:1", + urn=f"{sample_score_set.urn}#11", hgvs_nt="NM_000000.1:c.1A>G", hgvs_pro="NP_000000.1:p.Met1Val", data={}, @@ -647,7 +647,7 @@ async def test_cis_phased_multivariant_candidate_is_persisted_as_a_block( grouped with its coding partner in the projection pair.""" variant = Variant( score_set_id=sample_score_set.id, - urn="variant:1", + urn=f"{sample_score_set.urn}#11", hgvs_nt="NM_000000.1:c.1A>G", hgvs_pro="NP_000000.1:p.Met1Val", data={}, @@ -699,7 +699,7 @@ async def test_duplicate_candidate_digests_are_deduped_per_record( The surviving link still carries the pair's group.""" variant = Variant( score_set_id=sample_score_set.id, - urn="variant:1", + urn=f"{sample_score_set.urn}#11", hgvs_nt="NM_000000.1:c.1A>G", hgvs_pro="NP_000000.1:p.Met1Val", data={}, @@ -745,7 +745,7 @@ async def test_authoritative_allele_is_folded_into_its_projection_group( authoritative allele plus its sibling projection derived in the same projection pair.""" variant = Variant( score_set_id=sample_score_set.id, - urn="variant:1", + urn=f"{sample_score_set.urn}#11", hgvs_nt="NM_000000.1:c.1A>G", hgvs_pro="NP_000000.1:p.Met1Val", data={}, @@ -809,7 +809,7 @@ async def test_authoritative_fold_in_when_measured_at_genomic_level( the coding sibling becomes the derived member of the same group.""" variant = Variant( score_set_id=sample_score_set.id, - urn="variant:1", + urn=f"{sample_score_set.urn}#11", hgvs_nt="NC_000001.11:g.1000A>G", data={}, ) @@ -860,7 +860,7 @@ async def test_projection_failed_candidate_is_a_one_member_group( link. This is the case the old two-list shape silently dropped.""" variant = Variant( score_set_id=sample_score_set.id, - urn="variant:1", + urn=f"{sample_score_set.urn}#11", hgvs_nt="NM_000000.1:c.1A>G", hgvs_pro="NP_000000.1:p.Met1Val", data={}, @@ -917,7 +917,7 @@ async def test_protein_assay_leaves_authoritative_group_null_over_a_fan_out( ambiguous fan-out becomes N distinct groups, each a coding/genomic pair.""" variant = Variant( score_set_id=sample_score_set.id, - urn="variant:1", + urn=f"{sample_score_set.urn}#11", hgvs_pro="NP_000000.1:p.Met1Val", data={}, ) @@ -981,7 +981,7 @@ async def test_independent_rerun_retires_prior_links_and_regroups( identity across re-maps.""" variant = Variant( score_set_id=sample_score_set.id, - urn="variant:1", + urn=f"{sample_score_set.urn}#11", hgvs_nt="NM_000000.1:c.1A>G", hgvs_pro="NP_000000.1:p.Met1Val", data={}, @@ -1067,7 +1067,7 @@ async def test_all_failures_fail_the_job( """If every variant's translation errors, the job fails and records FAILED annotations.""" variant = Variant( score_set_id=sample_score_set.id, - urn="variant:1", + urn=f"{sample_score_set.urn}#11", hgvs_nt="NM_000000.1:c.1A>G", hgvs_pro="NP_000000.1:p.Met1Val", data={}, @@ -1109,14 +1109,14 @@ async def test_partial_success_succeeds_with_mixed_annotations( success and a failure annotation.""" variant_ok = Variant( score_set_id=sample_score_set.id, - urn="variant:1", + urn=f"{sample_score_set.urn}#11", hgvs_nt="NM_000000.1:c.1A>G", hgvs_pro="NP_000000.1:p.Met1Val", data={}, ) variant_err = Variant( score_set_id=sample_score_set.id, - urn="variant:2", + urn=f"{sample_score_set.urn}#12", hgvs_nt="NM_000000.1:c.2G>T", hgvs_pro="NP_000000.1:p.Val2Leu", data={}, @@ -1161,7 +1161,7 @@ async def test_non_translatable_edit_is_skipped_not_failed( #767 fix: such variants must not pollute FAILED for the retroactive backfill.""" variant = Variant( score_set_id=sample_score_set.id, - urn="variant:1", + urn=f"{sample_score_set.urn}#11", hgvs_nt="NM_000000.1:c.1A>G", hgvs_pro="NP_000000.1:p.Met1Val", data={}, @@ -1211,9 +1211,15 @@ async def test_non_translatable_skip_coexists_with_success_and_failure( """Across three variants — one translatable, one non-translatable, one genuinely erroring — each lands in its own bucket: translated / skipped / failed. The genuine error still records FAILED (translation_error); the non-translatable one is a not_translatable skip.""" - variant_ok = Variant(score_set_id=sample_score_set.id, urn="variant:1", hgvs_nt="NM_000000.1:c.1A>G", data={}) - variant_skip = Variant(score_set_id=sample_score_set.id, urn="variant:2", hgvs_nt="NM_000000.1:c.2G>T", data={}) - variant_fail = Variant(score_set_id=sample_score_set.id, urn="variant:3", hgvs_nt="NM_000000.1:c.3T>A", data={}) + variant_ok = Variant( + score_set_id=sample_score_set.id, urn=f"{sample_score_set.urn}#11", hgvs_nt="NM_000000.1:c.1A>G", data={} + ) + variant_skip = Variant( + score_set_id=sample_score_set.id, urn=f"{sample_score_set.urn}#12", hgvs_nt="NM_000000.1:c.2G>T", data={} + ) + variant_fail = Variant( + score_set_id=sample_score_set.id, urn=f"{sample_score_set.urn}#13", hgvs_nt="NM_000000.1:c.3T>A", data={} + ) session.add_all([variant_ok, variant_skip, variant_fail]) session.commit() await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) @@ -1258,8 +1264,12 @@ async def test_upstream_outage_is_failed_as_api_error_not_translation_error( ): """A UTA outage that outlasts the library's retries is FAILED with reason api_error, so it stays distinguishable from a genuine translation_error when auditing coverage gaps.""" - variant_ok = Variant(score_set_id=sample_score_set.id, urn="variant:1", hgvs_nt="NM_000000.1:c.1A>G", data={}) - variant_down = Variant(score_set_id=sample_score_set.id, urn="variant:2", hgvs_nt="NM_000000.1:c.2G>T", data={}) + variant_ok = Variant( + score_set_id=sample_score_set.id, urn=f"{sample_score_set.urn}#11", hgvs_nt="NM_000000.1:c.1A>G", data={} + ) + variant_down = Variant( + score_set_id=sample_score_set.id, urn=f"{sample_score_set.urn}#12", hgvs_nt="NM_000000.1:c.2G>T", data={} + ) session.add_all([variant_ok, variant_down]) session.commit() await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) @@ -1300,7 +1310,7 @@ async def test_partial_candidate_translation_failure_keeps_success_with_metadata variant a SUCCESS (one allele created) while retaining the dropped candidate in metadata.""" variant = Variant( score_set_id=sample_score_set.id, - urn="variant:1", + urn=f"{sample_score_set.urn}#11", hgvs_nt="NM_000000.1:c.1A>G", hgvs_pro="NP_000000.1:p.Met1Val", data={}, @@ -1347,7 +1357,7 @@ async def test_all_candidates_failing_translation_marks_variant_failed( created, and the per-candidate errors are retained in metadata.""" variant = Variant( score_set_id=sample_score_set.id, - urn="variant:1", + urn=f"{sample_score_set.urn}#11", hgvs_nt="NM_000000.1:c.1A>G", hgvs_pro="NP_000000.1:p.Met1Val", data={}, @@ -1397,7 +1407,7 @@ async def test_unresolved_transcript_is_tallied_as_skip_but_recorded_as_failed( (no_coding_transcript → `absent`).""" variant = Variant( score_set_id=sample_score_set.id, - urn="variant:1", + urn=f"{sample_score_set.urn}#11", hgvs_nt="NC_000001.11:g.1000A>G", data={}, ) @@ -1442,7 +1452,7 @@ async def test_genomic_accession_coding_target_is_reverse_translated( TargetGeneMapping, whose reference_accession anchors the projection.""" variant = Variant( score_set_id=sample_score_set.id, - urn="variant:1", + urn=f"{sample_score_set.urn}#11", hgvs_nt="NC_000001.11:g.1000A>G", data={}, ) @@ -1503,7 +1513,7 @@ async def test_uses_latest_cdna_row_within_the_run( RT reverse-translates against the newest (highest id), not an arbitrary one.""" variant = Variant( score_set_id=sample_score_set.id, - urn="variant:1", + urn=f"{sample_score_set.urn}#11", hgvs_nt="NC_000001.11:g.1000A>G", data={}, ) @@ -1569,7 +1579,7 @@ async def test_ignores_stale_cdna_row_from_a_different_run( run's job_run_id, so a row carrying no such anchor never leaks in (mavedb-api#763).""" variant = Variant( score_set_id=sample_score_set.id, - urn="variant:1", + urn=f"{sample_score_set.urn}#11", hgvs_nt="NC_000001.11:g.1000A>G", data={}, ) @@ -1621,7 +1631,7 @@ async def test_coding_target_skip_is_classified_recoverable( assert sample_score_set.target_genes[0].category == TargetCategory.protein_coding variant = Variant( score_set_id=sample_score_set.id, - urn="variant:1", + urn=f"{sample_score_set.urn}#11", hgvs_nt="NC_000001.11:g.1000A>G", data={}, ) @@ -1662,7 +1672,7 @@ async def test_regulatory_target_skip_is_classified_correct( variant = Variant( score_set_id=sample_score_set.id, - urn="variant:1", + urn=f"{sample_score_set.urn}#11", hgvs_nt="NC_000001.11:g.1000A>G", data={}, ) @@ -1697,7 +1707,7 @@ async def test_translation_config_param_overrides_job_defaults( resulting config is passed through to construct_equivalent_variants.""" variant = Variant( score_set_id=sample_score_set.id, - urn="variant:1", + urn=f"{sample_score_set.urn}#11", hgvs_nt="NM_000000.1:c.1A>G", hgvs_pro="NP_000000.1:p.Met1Val", data={}, @@ -1750,7 +1760,7 @@ async def test_translation_config_defaults_when_param_absent( (full codon equivalence class, indels included).""" variant = Variant( score_set_id=sample_score_set.id, - urn="variant:1", + urn=f"{sample_score_set.urn}#11", hgvs_nt="NM_000000.1:c.1A>G", hgvs_pro="NP_000000.1:p.Met1Val", data={}, @@ -1796,7 +1806,7 @@ async def test_protein_level_transcript_resolved_via_uta( is resolved NP_→NM_ via UTA and supplied as the VariantInput hint.""" variant = Variant( score_set_id=sample_score_set.id, - urn="variant:1", + urn=f"{sample_score_set.urn}#11", hgvs_pro="NP_000000.1:p.Met1Val", data={}, ) From c63923539b8f1e015c79385183ab0662575ae13f Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 29 Sep 2026 21:14:31 -0700 Subject: [PATCH 188/252] test(variants): publish the score set before its calibration in VA-Spec tests release-2026.3.0 refuses to publish a calibration on a private score set, so these two tests fail once RT merges with it. Publishing the score set first works with or without that rule. --- tests/routers/test_variant_annotation.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/routers/test_variant_annotation.py b/tests/routers/test_variant_annotation.py index 54d7fadf8..7a764ec3f 100644 --- a/tests/routers/test_variant_annotation.py +++ b/tests/routers/test_variant_annotation.py @@ -9,6 +9,7 @@ import json from datetime import datetime, timezone +from unittest.mock import patch import pytest @@ -41,6 +42,7 @@ from tests.helpers.util.score_set import ( create_seq_score_set_with_mapped_variants, create_seq_score_set_with_variants, + publish_score_set, seed_annotation_substrate, ) @@ -106,6 +108,9 @@ def test_variant_functional_impact_statement( score_set = create_seq_score_set_with_mapped_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) + # A calibration can be published only on a published score set. + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + score_set = publish_score_set(client, score_set["urn"]) create_publish_and_promote_score_calibration( client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) ) @@ -150,6 +155,9 @@ def test_variant_pathogenicity_statement( score_set = create_seq_score_set_with_mapped_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) + # A calibration can be published only on a published score set. + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + score_set = publish_score_set(client, score_set["urn"]) create_publish_and_promote_score_calibration( client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) ) From 232f2e11c32dfd71d4e2579c04e613e393dafbc7 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 29 Sep 2026 22:35:06 -0700 Subject: [PATCH 189/252] fix(calibrations): return 422 for an unknown MONDO disease code An unknown MONDO code raised a ValidationError that no route caught, so creating or modifying a calibration with one returned a 500 and alerted Slack. The calibration create and modify routes, and score set creation with calibrations, now return 422 with the field's location. --- src/mavedb/routers/score_calibrations.py | 24 ++++++-- src/mavedb/routers/score_sets.py | 23 ++++++-- tests/routers/test_score_calibrations.py | 74 ++++++++++++++++++++++++ tests/routers/test_score_set.py | 31 ++++++++++ 4 files changed, 142 insertions(+), 10 deletions(-) diff --git a/src/mavedb/routers/score_calibrations.py b/src/mavedb/routers/score_calibrations.py index 2c61d06dc..193fd0a9a 100644 --- a/src/mavedb/routers/score_calibrations.py +++ b/src/mavedb/routers/score_calibrations.py @@ -394,9 +394,15 @@ async def create_score_calibration_route( detail=[{"loc": [e.custom_loc or "controlsFile"], "msg": str(e), "type": "value_error"}], ) - created_calibration = await create_score_calibration_in_score_set( - db, calibration, user_data.user, variant_classes if classes_file else None - ) + try: + created_calibration = await create_score_calibration_in_score_set( + db, calibration, user_data.user, variant_classes if classes_file else None + ) + except ValidationError as e: + raise HTTPException( + status_code=422, + detail=[{"loc": e.custom_loc or ["body"], "msg": str(e), "type": "value_error"}], + ) db.commit() db.refresh(created_calibration) @@ -604,9 +610,15 @@ async def modify_score_calibration_route( detail=[{"loc": [e.custom_loc or "controlsFile"], "msg": str(e), "type": "value_error"}], ) - updated_calibration = await modify_score_calibration( - db, item, calibration_update, user_data.user, variant_classes if classes_file else None - ) + try: + updated_calibration = await modify_score_calibration( + db, item, calibration_update, user_data.user, variant_classes if classes_file else None + ) + except ValidationError as e: + raise HTTPException( + status_code=422, + detail=[{"loc": e.custom_loc or ["body"], "msg": str(e), "type": "value_error"}], + ) # A public calibration may not carry unacknowledged controls, on this route the same as at # publish time. controls_not_phi is tristate: only True clears the gate — None (unaddressed) and diff --git a/src/mavedb/routers/score_sets.py b/src/mavedb/routers/score_sets.py index 45d4525b1..d6b619c7a 100644 --- a/src/mavedb/routers/score_sets.py +++ b/src/mavedb/routers/score_sets.py @@ -78,6 +78,7 @@ from mavedb.lib.taxonomies import find_or_create_taxonomy from mavedb.lib.types.authentication import UserData from mavedb.lib.urn_redirects import record_urn_redirect +from mavedb.lib.validation.exceptions import ValidationError as MaveDBValidationError from mavedb.lib.urns import ( generate_experiment_set_urn, generate_experiment_urn, @@ -1927,7 +1928,7 @@ async def create_score_set( score_calibrations: list[ScoreCalibration] = [] if item_create.score_calibrations: - for calibration_create in item_create.score_calibrations: + for calibration_index, calibration_create in enumerate(item_create.score_calibrations): # TODO#592: Support for class-based calibrations on score set creation if calibration_create.class_based: logger.info( @@ -1950,9 +1951,23 @@ async def create_score_set( detail="Calibration controls are not supported on score set creation. Please add controls after creating the score set.", ) - created_calibration_item = await create_score_calibration( - db, calibration_create, user_data.user, variant_classes=None - ) + try: + created_calibration_item = await create_score_calibration( + db, calibration_create, user_data.user, variant_classes=None + ) + except MaveDBValidationError as e: + # custom_loc is relative to the calibration body; nest it under this calibration. + field_loc = (e.custom_loc or ["body"])[1:] + raise HTTPException( + status_code=422, + detail=[ + { + "loc": ["body", "scoreCalibrations", calibration_index, *field_loc], + "msg": str(e), + "type": "value_error", + } + ], + ) created_calibration_item.investigator_provided = True # necessarily true on score set creation score_calibrations.append(created_calibration_item) diff --git a/tests/routers/test_score_calibrations.py b/tests/routers/test_score_calibrations.py index dc8c6c137..dbd6fc3d0 100644 --- a/tests/routers/test_score_calibrations.py +++ b/tests/routers/test_score_calibrations.py @@ -216,6 +216,80 @@ async def fake_fetch(code): assert stored.label == "Brugada syndrome" +@pytest.mark.parametrize( + "mock_publication_fetch", + [ + [ + {"dbName": "PubMed", "identifier": TEST_PUBMED_IDENTIFIER}, + {"dbName": "bioRxiv", "identifier": TEST_BIORXIV_IDENTIFIER}, + ] + ], + indirect=["mock_publication_fetch"], +) +def test_cannot_create_score_calibration_with_unknown_disease( + client, setup_router_db, mock_publication_fetch, session, data_provider, data_files, monkeypatch +): + async def fake_fetch(code): + return None + + monkeypatch.setattr("mavedb.lib.mondo_ols.fetch_mondo_term", fake_fetch) + + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + payload = { + **deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED), + "disease": "MONDO:9999999", + "scoreSetUrn": score_set["urn"], + } + + response = client.post("/api/v1/score-calibrations/", json=payload) + + assert response.status_code == 422 + assert response.json()["detail"][0]["loc"] == ["body", "disease"] + assert "MONDO:9999999" in response.json()["detail"][0]["msg"] + + +@pytest.mark.parametrize( + "mock_publication_fetch", + [ + [ + {"dbName": "PubMed", "identifier": TEST_PUBMED_IDENTIFIER}, + {"dbName": "bioRxiv", "identifier": TEST_BIORXIV_IDENTIFIER}, + ] + ], + indirect=["mock_publication_fetch"], +) +def test_cannot_modify_score_calibration_with_unknown_disease( + client, setup_router_db, mock_publication_fetch, session, data_provider, data_files, monkeypatch +): + async def fake_fetch(code): + return None + + monkeypatch.setattr("mavedb.lib.mondo_ols.fetch_mondo_term", fake_fetch) + + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + calibration = create_test_score_calibration_in_score_set_via_client( + client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) + ) + payload = { + **deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED), + "disease": "MONDO:9999999", + "scoreSetUrn": score_set["urn"], + } + + response = client.put(f"/api/v1/score-calibrations/{calibration['urn']}", json=payload) + + assert response.status_code == 422 + assert response.json()["detail"][0]["loc"] == ["body", "disease"] + stored = client.get(f"/api/v1/score-calibrations/{calibration['urn']}").json() + assert stored["disease"]["primaryCoding"]["code"] == "MONDO:0000001" + + @pytest.mark.parametrize( "mock_publication_fetch", [ diff --git a/tests/routers/test_score_set.py b/tests/routers/test_score_set.py index d2529f7e3..670689013 100644 --- a/tests/routers/test_score_set.py +++ b/tests/routers/test_score_set.py @@ -204,6 +204,37 @@ def test_create_score_set_with_contributor(client, mock_publication_fetch, setup assert response.status_code == 200 +@pytest.mark.parametrize( + "mock_publication_fetch", + [ + [ + {"dbName": "PubMed", "identifier": f"{TEST_PUBMED_IDENTIFIER}"}, + {"dbName": "bioRxiv", "identifier": f"{TEST_BIORXIV_IDENTIFIER}"}, + ] + ], + indirect=["mock_publication_fetch"], +) +def test_cannot_create_score_set_with_score_calibration_with_unknown_disease( + client, mock_publication_fetch, setup_router_db, monkeypatch +): + async def fake_fetch(code): + return None + + monkeypatch.setattr("mavedb.lib.mondo_ols.fetch_mondo_term", fake_fetch) + + experiment = create_experiment(client) + score_set = deepcopy(TEST_MINIMAL_SEQ_SCORESET) + score_set["experimentUrn"] = experiment["urn"] + score_set["scoreCalibrations"] = [ + {**deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED), "disease": "MONDO:9999999"} + ] + + response = client.post("/api/v1/score-sets/", json=score_set) + + assert response.status_code == 422 + assert response.json()["detail"][0]["loc"] == ["body", "scoreCalibrations", 0, "disease"] + + @pytest.mark.parametrize( "mock_publication_fetch", [ From 49ab85254c0ce1f01964d1c0ae229495182f6211 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 29 Sep 2026 22:26:59 -0700 Subject: [PATCH 190/252] fix(worker): keep derived alleles through a UTA outage on a reverse translation rerun A rerun retired the live derived links for every current record in the score set, so a UTA outage on a rerun wiped good links, even when every variant failed and the job returned FAILED. Records whose error this run is UPSTREAM_UNAVAILABLE now keep their links: their inputs are unchanged, so the links are still the last run's answer. Any other outcome, including NOT_TRANSLATABLE, a translation error or a VRS translation failure, is this run's answer and still replaces the prior links. --- .../variant_processing/reverse_translation.py | 11 +++ .../test_reverse_translation.py | 87 +++++++++++++++++++ 2 files changed, 98 insertions(+) diff --git a/src/mavedb/worker/jobs/variant_processing/reverse_translation.py b/src/mavedb/worker/jobs/variant_processing/reverse_translation.py index 7ee7fe7c0..6fe00fa1c 100644 --- a/src/mavedb/worker/jobs/variant_processing/reverse_translation.py +++ b/src/mavedb/worker/jobs/variant_processing/reverse_translation.py @@ -534,6 +534,16 @@ async def reverse_translate_variants_for_score_set( metadata=annotation_metadata, ) + # A record the library couldn't ask about because UTA was unavailable keeps its derived links: + # its inputs are unchanged since the run that produced them, so they are still that run's answer. + # Every other outcome, including a skip or a failed translation, is this run's answer and replaces + # the prior set. + unavailable_record_ids = [ + variant_input_map[id(error.input)][0].id + for error in errors + if error.reason is TranslationErrorReason.UPSTREAM_UNAVAILABLE + ] + # Supersede prior live derived links atomically. # TODO#765: re-runs retire and recreate the whole derived set because re-mapping re-mints # records; idempotent records would allow unchanged links to stay live. @@ -542,6 +552,7 @@ async def reverse_translate_variants_for_score_set( new_links, MappingRecordAllele.is_authoritative.is_(False), MappingRecordAllele.mapping_record_id.in_(current_record_ids), + MappingRecordAllele.mapping_record_id.not_in(unavailable_record_ids), ) # The library types each error's reason: NOT_TRANSLATABLE is a benign structural gap (the diff --git a/tests/worker/jobs/variant_processing/test_reverse_translation.py b/tests/worker/jobs/variant_processing/test_reverse_translation.py index d6bb54955..e8f9a0060 100644 --- a/tests/worker/jobs/variant_processing/test_reverse_translation.py +++ b/tests/worker/jobs/variant_processing/test_reverse_translation.py @@ -1054,6 +1054,93 @@ async def test_independent_rerun_retires_prior_links_and_regroups( assert len(events) == 2 assert max(events, key=lambda e: e.id).disposition == "present" + @pytest.mark.parametrize( + "rerun_outcome, keeps_prior_link", + [ + ("upstream_unavailable", True), + ("translation_error", False), + ("not_translatable", False), + ("vrs_translation_failure", False), + ], + ) + async def test_rerun_keeps_prior_links_only_when_upstream_unavailable( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + rerun_outcome, + keeps_prior_link, + ): + """A variant that fails a rerun because UTA was unavailable keeps the derived links from its last + successful run, since its inputs are unchanged. Any other failure is the rerun's answer for that + variant, so its prior links are retired.""" + variant_ok = Variant( + score_set_id=sample_score_set.id, urn=f"{sample_score_set.urn}#11", hgvs_nt="NM_000000.1:c.1A>G", data={} + ) + variant_down = Variant( + score_set_id=sample_score_set.id, urn=f"{sample_score_set.urn}#12", hgvs_nt="NM_000000.1:c.2G>T", data={} + ) + session.add_all([variant_ok, variant_down]) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + ok_candidate = "NM_000001.1:c.5A>G" + down_candidate = "NM_000001.1:c.6G>T" + ids = {ok_candidate: "ga4gh:VA.ok", down_candidate: "ga4gh:VA.down"} + first_construct = fake_construct( + {"NM_000000.1:c.1A>G": [(ok_candidate, None)], "NM_000000.1:c.2G>T": [(down_candidate, None)]} + ) + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", first_construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", fake_translate(ids)), + ): + first = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + assert first.status == JobStatus.SUCCEEDED + down_link = next( + link for link in _non_authoritative_links(session) if link.allele.vrs_digest == "ga4gh:VA.down" + ) + + rerun = JobRun( + urn="test:reverse_translate_variants_for_score_set:rerun", + job_type="reverse_translate_variants_for_score_set", + job_function="reverse_translate_variants_for_score_set", + max_retries=3, + retry_count=0, + job_params=dict(sample_independent_reverse_translation_run.job_params), + ) + session.add(rerun) + session.commit() + + ok_result = {"NM_000000.1:c.1A>G": [(ok_candidate, None)]} + down_message = {"NM_000000.1:c.2G>T": "reverse translation failed"} + second_translate = fake_translate(ids) + if rerun_outcome == "upstream_unavailable": + second_construct = fake_construct(ok_result, upstream_unavailable_by_hgvs=down_message) + elif rerun_outcome == "translation_error": + second_construct = fake_construct(ok_result, errors_by_hgvs=down_message) + elif rerun_outcome == "not_translatable": + second_construct = fake_construct(ok_result, not_translatable_by_hgvs=down_message) + else: + second_construct = first_construct + second_translate = fake_translate(ids, errors_by_hgvs={down_candidate: "cannot translate to VRS"}) + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", second_construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", second_translate), + ): + second = await _reverse_translate(session, mock_worker_ctx, rerun) + assert second.status == JobStatus.SUCCEEDED + + session.refresh(down_link) + assert (down_link.valid_to is None) is keeps_prior_link + live_digests = sorted( + link.allele.vrs_digest for link in _non_authoritative_links(session) if link.valid_to is None + ) + assert live_digests == (["ga4gh:VA.down", "ga4gh:VA.ok"] if keeps_prior_link else ["ga4gh:VA.ok"]) + async def test_all_failures_fail_the_job( self, session, From 7ca9135d656975b125862272dd7256f2d5de588d Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 30 Sep 2026 14:05:37 -0700 Subject: [PATCH 191/252] feat(alleles): verify allele refgets against SeqRepo at insert and pin the accession contract An allele digest covers the refget of the sequence it sits on, and alleles deduplicate on that digest. The mapper and reverse translation each build alleles, so when they resolved one accession to different sequences (NM_007294.3 and NP_001346945.1 on staging) they minted different digests for the same variant, and the copies never merged. The existing digest-consistency check could not see it, because both digests were internally consistent. - resolve_refget() returns the single refget SeqRepo holds for an accession and raises on none or more than one. - verify_allele_refget() compares an allele's refget with SeqRepo's for its HGVS accession and raises AlleleRefgetMismatchError on a mismatch. It runs in the mapping job before the authoritative allele is created and in reverse translation before each candidate allele. An accession SeqRepo cannot resolve is logged at error level without raising. - tests/fixtures/canonical_accession_sequences.json is a byte-identical copy of the dcd_mapping fixture of NCBI records and their refgets; the contract tests assert the API side against it, including that the seqfetcher chain reads SeqRepo ahead of the FASTA fallback. - The worker test contexts now carry ctx["seqrepo"], as lifecycle.on_job_start does. --- src/mavedb/lib/seqrepo.py | 31 ++++ src/mavedb/lib/vrs_utils.py | 64 +++++++- .../worker/jobs/variant_processing/mapping.py | 8 +- .../variant_processing/reverse_translation.py | 8 +- tests/conftest_optional.py | 5 +- .../canonical_accession_sequences.json | 21 +++ tests/lib/test_seqrepo_contract.py | 143 ++++++++++++++++++ 7 files changed, 276 insertions(+), 4 deletions(-) create mode 100644 tests/fixtures/canonical_accession_sequences.json create mode 100644 tests/lib/test_seqrepo_contract.py diff --git a/src/mavedb/lib/seqrepo.py b/src/mavedb/lib/seqrepo.py index 708d08e2a..11d53e347 100644 --- a/src/mavedb/lib/seqrepo.py +++ b/src/mavedb/lib/seqrepo.py @@ -20,6 +20,14 @@ DEFAULT_CHUNK_SIZE = 8192 +class SequenceNotFoundError(LookupError): + """Raised when SeqRepo holds no sequence for an accession.""" + + +class AmbiguousSequenceError(LookupError): + """Raised when SeqRepo holds more than one sequence for an accession.""" + + def base64url_to_hex(s: str) -> str: return hexlify(urlsafe_b64decode(s)).decode("ascii") @@ -49,6 +57,29 @@ def get_sequence_ids(sr: SeqRepo, query: str) -> list[str]: return seq_ids +def resolve_refget(sr: SeqRepo, accession: str) -> str: + """Return the single GA4GH refget (``SQ.…``) SeqRepo holds for an accession. + + SeqRepo's ``seq_id`` is the sequence's sha512t24u digest, so the refget is ``SQ.`` plus it. The + shared SeqRepo is the only authority for what an accession's sequence is, and every writer of VRS + alleles must agree with it for allele digests to deduplicate. More than one sequence for an accession + is an error rather than a pick. + + The same contract is held by ``dcd_mapping.lookup.resolve_refget``; both are pinned to + ``tests/fixtures/canonical_accession_sequences.json``. + + :raise SequenceNotFoundError: if SeqRepo has no sequence for the accession + :raise AmbiguousSequenceError: if it has more than one + """ + seq_ids = get_sequence_ids(sr, accession) + if not seq_ids: + raise SequenceNotFoundError(f"SeqRepo has no sequence for accession {accession}.") + if len(seq_ids) > 1: + raise AmbiguousSequenceError(f"SeqRepo holds {len(seq_ids)} sequences for accession {accession}.") + + return f"SQ.{seq_ids[0]}" + + def _generate_nsa_options(query: str) -> Union[list[tuple[str, ...]], list[tuple[None, str]]]: """ >>> _generate_nsa_options("NM_000551.3") diff --git a/src/mavedb/lib/vrs_utils.py b/src/mavedb/lib/vrs_utils.py index af4455140..97b6d7de1 100644 --- a/src/mavedb/lib/vrs_utils.py +++ b/src/mavedb/lib/vrs_utils.py @@ -28,8 +28,11 @@ ) from ga4gh.vrs.normalize import normalize -from mavedb.lib.hgvs import split_cis_phased_hgvs +from biocommons.seqrepo import SeqRepo + +from mavedb.lib.hgvs import extract_accession, split_cis_phased_hgvs from mavedb.lib.logging.context import logging_context +from mavedb.lib.seqrepo import AmbiguousSequenceError, SequenceNotFoundError, resolve_refget from mavedb.lib.vrs import vrs_object_from_mapped_variant logger = logging.getLogger(__name__) @@ -44,6 +47,14 @@ "r": Syntax.HGVS_R, } +# Accessions whose sequence SeqRepo is the authority for. Anything else (a target-sequence label, a +# ga4gh: digest) has no accession-to-sequence mapping to check an allele against. +CANONICAL_ACCESSION_PREFIXES = ("NC_", "NM_", "NR_", "NP_", "XM_", "XP_", "ENST", "ENSP") + + +class AlleleRefgetMismatchError(ValueError): + """Raised when an allele sits on a different sequence than SeqRepo holds for its accession.""" + def _hgvs_syntax(hgvs: str) -> Syntax: """Map an HGVS string to its VRS Expression syntax via the type letter after the accession.""" @@ -279,3 +290,54 @@ def canonical_variation_document(document: Mapping[str, Any], *, subject: str) - ) return variation.model_dump(mode="json", exclude_none=True), identifier + + +def _location_refgets(variation: Mapping[str, Any]) -> list[str]: + """Collect the refgetAccession of the variation's location, or of each member of a block.""" + members = variation.get("members") or [variation] + refgets = [] + for member in members: + reference = (member.get("location") or {}).get("sequenceReference") or {} + if reference.get("refgetAccession"): + refgets.append(reference["refgetAccession"]) + + return refgets + + +def verify_allele_refget(variation: Mapping[str, Any], hgvs: str, sr: SeqRepo, *, subject: str) -> None: + """Fail unless a variation sits on the sequence SeqRepo holds for its HGVS accession. + + An allele digest covers the refget of its sequence, and alleles deduplicate on that digest. Two + writers that resolve one accession to different sequences mint different digests for the same + variant, and the copies never merge (this happened with NM_007294.3 and NP_001346945.1). The + digest-consistency check in :func:`canonical_variation_document` cannot see it, because both + digests are internally consistent. Comparing against SeqRepo at insert time can. + + A mismatch raises. An accession SeqRepo cannot resolve, or resolves ambiguously, is logged at error + level without raising: the allele could not have been built on a sequence SeqRepo lacks, so it points + at a SeqRepo problem to alert on, not at this allele. + + :raise AlleleRefgetMismatchError: if the allele's refget differs from the accession's + """ + accession = extract_accession(hgvs) + if not accession.startswith(CANONICAL_ACCESSION_PREFIXES): + return + + try: + expected = resolve_refget(sr, accession) + except (SequenceNotFoundError, AmbiguousSequenceError) as e: + logger.error( + msg=f"Cannot verify the refget of {subject}: {e}", + extra=logging_context(), + ) + return + + for refget in _location_refgets(variation): + if refget != expected: + msg = ( + f"Allele for {subject} sits on {refget}, but SeqRepo holds {expected} for {accession}. " + "The writer and SeqRepo disagree about this accession's sequence, so the allele would not " + "deduplicate against the same variant written elsewhere." + ) + logger.error(msg=msg, extra=logging_context()) + raise AlleleRefgetMismatchError(msg) diff --git a/src/mavedb/worker/jobs/variant_processing/mapping.py b/src/mavedb/worker/jobs/variant_processing/mapping.py index 7f50ecad7..dec29a56a 100644 --- a/src/mavedb/worker/jobs/variant_processing/mapping.py +++ b/src/mavedb/worker/jobs/variant_processing/mapping.py @@ -29,7 +29,7 @@ from mavedb.lib.types.workflow import JobExecutionOutcome from mavedb.lib.variant_translations import get_or_create_allele from mavedb.lib.variants import get_hgvs_from_post_mapped -from mavedb.lib.vrs_utils import canonical_variation_document +from mavedb.lib.vrs_utils import canonical_variation_document, verify_allele_refget from mavedb.models.allele import Allele as AlleleDbModel from mavedb.models.enums.annotation_type import AnnotationType from mavedb.models.enums.disposition import Disposition @@ -451,6 +451,12 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan canonical_post_mapped, allele_digest = canonical_variation_document( post_mapped_allele, subject=f"variant {variant.urn}" ) + # Verify that the post-mapped allele's sequence matches what SeqRepo holds for its accession. + if (seqrepo := getattr(ctx["seqrepo"], "sr", None)) is not None: + verify_allele_refget( + canonical_post_mapped, assay_level_hgvs or "", seqrepo, subject=f"variant {variant.urn}" + ) + allele_draft = AlleleDbModel( vrs_digest=allele_digest, level=sequence_level, diff --git a/src/mavedb/worker/jobs/variant_processing/reverse_translation.py b/src/mavedb/worker/jobs/variant_processing/reverse_translation.py index 6fe00fa1c..ab20f3291 100644 --- a/src/mavedb/worker/jobs/variant_processing/reverse_translation.py +++ b/src/mavedb/worker/jobs/variant_processing/reverse_translation.py @@ -41,7 +41,7 @@ from mavedb.lib.hgvs import extract_accession, is_cis_phased_hgvs, strip_protein_prediction_parens from mavedb.lib.types.workflow import JobExecutionOutcome from mavedb.lib.variant_translations import get_or_create_allele -from mavedb.lib.vrs_utils import translate_hgvs_to_variation +from mavedb.lib.vrs_utils import translate_hgvs_to_variation, verify_allele_refget from mavedb.models.allele import Allele as AlleleDbModel from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.models.enums.annotation_type import AnnotationType @@ -466,6 +466,12 @@ async def reverse_translate_variants_for_score_set( failed_candidates.append({"hgvs": hgvs, "level": level.value, "error": str(e)}) continue + # Verify that the post-mapped allele's sequence matches what SeqRepo holds for its accession. + if (seqrepo := getattr(ctx["seqrepo"], "sr", None)) is not None: + verify_allele_refget( + variation.model_dump(exclude_none=True), hgvs, seqrepo, subject=f"variant record {rec.id}" + ) + if variation.id in seen_digests: continue diff --git a/tests/conftest_optional.py b/tests/conftest_optional.py index 7e3f28b2f..d61b07c52 100644 --- a/tests/conftest_optional.py +++ b/tests/conftest_optional.py @@ -4,7 +4,7 @@ from concurrent import futures from inspect import getsourcefile from posixpath import abspath -from unittest.mock import patch +from unittest.mock import Mock, patch import cdot.hgvs.dataproviders import pytest @@ -12,6 +12,7 @@ from arq.worker import Worker from biocommons.seqrepo import SeqRepo from fastapi.testclient import TestClient +from ga4gh.vrs.dataproxy import SeqRepoDataProxy from httpx import AsyncClient from sqlalchemy import Column, Float, Integer, MetaData, String, Table @@ -106,6 +107,7 @@ async def on_startup(ctx): async def on_job(ctx): ctx["db"] = session ctx["hdp"] = data_provider + ctx["seqrepo"] = Mock(spec=SeqRepoDataProxy) # mirrors lifecycle.on_job_start ctx["state"] = {} ctx["pool"] = futures.ProcessPoolExecutor() @@ -131,6 +133,7 @@ async def on_job(ctx): def standalone_worker_context(data_provider, arq_redis): yield { "hdp": data_provider, + "seqrepo": Mock(spec=SeqRepoDataProxy), # mirrors lifecycle.on_job_start "state": {}, "job_id": "test_job", "redis": arq_redis, diff --git a/tests/fixtures/canonical_accession_sequences.json b/tests/fixtures/canonical_accession_sequences.json new file mode 100644 index 000000000..b2166a015 --- /dev/null +++ b/tests/fixtures/canonical_accession_sequences.json @@ -0,0 +1,21 @@ +{ + "_comment": "Canonical accession sequences (NCBI records) and their GA4GH refgets, used to assert that every writer of VRS alleles resolves an accession to the same refget. expected_refget is sha512t24u(sequence). non_canonical_refgets are values that have been minted in error and must never be produced. Keep identical to the copy in the API repo (tests/fixtures/canonical_accession_sequences.json).", + "accessions": { + "NM_007294.3": { + "sequence": "GTACCTTGATTTCGTATTCTGAGAGGCTGCTGCTTAGCGGTAGCCCCTTGGTTTCCGTGGCAACGGAAAAGCGCGGGAATTACAGATAAATTAAAACTGCGACTGCGCGGCGTGAGCTCGCTGAGACTTCCTGGACGGGGGACAGGCTGTGGGGTTTCTCAGATAACTGGGCCCCTGCGCTCAGGAGGCCTTCACCCTCTGCTCTGGGTAAAGTTCATTGGAACAGAAAGAAATGGATTTATCTGCTCTTCGCGTTGAAGAAGTACAAAATGTCATTAATGCTATGCAGAAAATCTTAGAGTGTCCCATCTGTCTGGAGTTGATCAAGGAACCTGTCTCCACAAAGTGTGACCACATATTTTGCAAATTTTGCATGCTGAAACTTCTCAACCAGAAGAAAGGGCCTTCACAGTGTCCTTTATGTAAGAATGATATAACCAAAAGGAGCCTACAAGAAAGTACGAGATTTAGTCAACTTGTTGAAGAGCTATTGAAAATCATTTGTGCTTTTCAGCTTGACACAGGTTTGGAGTATGCAAACAGCTATAATTTTGCAAAAAAGGAAAATAACTCTCCTGAACATCTAAAAGATGAAGTTTCTATCATCCAAAGTATGGGCTACAGAAACCGTGCCAAAAGACTTCTACAGAGTGAACCCGAAAATCCTTCCTTGCAGGAAACCAGTCTCAGTGTCCAACTCTCTAACCTTGGAACTGTGAGAACTCTGAGGACAAAGCAGCGGATACAACCTCAAAAGACGTCTGTCTACATTGAATTGGGATCTGATTCTTCTGAAGATACCGTTAATAAGGCAACTTATTGCAGTGTGGGAGATCAAGAATTGTTACAAATCACCCCTCAAGGAACCAGGGATGAAATCAGTTTGGATTCTGCAAAAAAGGCTGCTTGTGAATTTTCTGAGACGGATGTAACAAATACTGAACATCATCAACCCAGTAATAATGATTTGAACACCACTGAGAAGCGTGCAGCTGAGAGGCATCCAGAAAAGTATCAGGGTAGTTCTGTTTCAAACTTGCATGTGGAGCCATGTGGCACAAATACTCATGCCAGCTCATTACAGCATGAGAACAGCAGTTTATTACTCACTAAAGACAGAATGAATGTAGAAAAGGCTGAATTCTGTAATAAAAGCAAACAGCCTGGCTTAGCAAGGAGCCAACATAACAGATGGGCTGGAAGTAAGGAAACATGTAATGATAGGCGGACTCCCAGCACAGAAAAAAAGGTAGATCTGAATGCTGATCCCCTGTGTGAGAGAAAAGAATGGAATAAGCAGAAACTGCCATGCTCAGAGAATCCTAGAGATACTGAAGATGTTCCTTGGATAACACTAAATAGCAGCATTCAGAAAGTTAATGAGTGGTTTTCCAGAAGTGATGAACTGTTAGGTTCTGATGACTCACATGATGGGGAGTCTGAATCAAATGCCAAAGTAGCTGATGTATTGGACGTTCTAAATGAGGTAGATGAATATTCTGGTTCTTCAGAGAAAATAGACTTACTGGCCAGTGATCCTCATGAGGCTTTAATATGTAAAAGTGAAAGAGTTCACTCCAAATCAGTAGAGAGTAATATTGAAGACAAAATATTTGGGAAAACCTATCGGAAGAAGGCAAGCCTCCCCAACTTAAGCCATGTAACTGAAAATCTAATTATAGGAGCATTTGTTACTGAGCCACAGATAATACAAGAGCGTCCCCTCACAAATAAATTAAAGCGTAAAAGGAGACCTACATCAGGCCTTCATCCTGAGGATTTTATCAAGAAAGCAGATTTGGCAGTTCAAAAGACTCCTGAAATGATAAATCAGGGAACTAACCAAACGGAGCAGAATGGTCAAGTGATGAATATTACTAATAGTGGTCATGAGAATAAAACAAAAGGTGATTCTATTCAGAATGAGAAAAATCCTAACCCAATAGAATCACTCGAAAAAGAATCTGCTTTCAAAACGAAAGCTGAACCTATAAGCAGCAGTATAAGCAATATGGAACTCGAATTAAATATCCACAATTCAAAAGCACCTAAAAAGAATAGGCTGAGGAGGAAGTCTTCTACCAGGCATATTCATGCGCTTGAACTAGTAGTCAGTAGAAATCTAAGCCCACCTAATTGTACTGAATTGCAAATTGATAGTTGTTCTAGCAGTGAAGAGATAAAGAAAAAAAAGTACAACCAAATGCCAGTCAGGCACAGCAGAAACCTACAACTCATGGAAGGTAAAGAACCTGCAACTGGAGCCAAGAAGAGTAACAAGCCAAATGAACAGACAAGTAAAAGACATGACAGCGATACTTTCCCAGAGCTGAAGTTAACAAATGCACCTGGTTCTTTTACTAAGTGTTCAAATACCAGTGAACTTAAAGAATTTGTCAATCCTAGCCTTCCAAGAGAAGAAAAAGAAGAGAAACTAGAAACAGTTAAAGTGTCTAATAATGCTGAAGACCCCAAAGATCTCATGTTAAGTGGAGAAAGGGTTTTGCAAACTGAAAGATCTGTAGAGAGTAGCAGTATTTCATTGGTACCTGGTACTGATTATGGCACTCAGGAAAGTATCTCGTTACTGGAAGTTAGCACTCTAGGGAAGGCAAAAACAGAACCAAATAAATGTGTGAGTCAGTGTGCAGCATTTGAAAACCCCAAGGGACTAATTCATGGTTGTTCCAAAGATAATAGAAATGACACAGAAGGCTTTAAGTATCCATTGGGACATGAAGTTAACCACAGTCGGGAAACAAGCATAGAAATGGAAGAAAGTGAACTTGATGCTCAGTATTTGCAGAATACATTCAAGGTTTCAAAGCGCCAGTCATTTGCTCCGTTTTCAAATCCAGGAAATGCAGAAGAGGAATGTGCAACATTCTCTGCCCACTCTGGGTCCTTAAAGAAACAAAGTCCAAAAGTCACTTTTGAATGTGAACAAAAGGAAGAAAATCAAGGAAAGAATGAGTCTAATATCAAGCCTGTACAGACAGTTAATATCACTGCAGGCTTTCCTGTGGTTGGTCAGAAAGATAAGCCAGTTGATAATGCCAAATGTAGTATCAAAGGAGGCTCTAGGTTTTGTCTATCATCTCAGTTCAGAGGCAACGAAACTGGACTCATTACTCCAAATAAACATGGACTTTTACAAAACCCATATCGTATACCACCACTTTTTCCCATCAAGTCATTTGTTAAAACTAAATGTAAGAAAAATCTGCTAGAGGAAAACTTTGAGGAACATTCAATGTCACCTGAAAGAGAAATGGGAAATGAGAACATTCCAAGTACAGTGAGCACAATTAGCCGTAATAACATTAGAGAAAATGTTTTTAAAGAAGCCAGCTCAAGCAATATTAATGAAGTAGGTTCCAGTACTAATGAAGTGGGCTCCAGTATTAATGAAATAGGTTCCAGTGATGAAAACATTCAAGCAGAACTAGGTAGAAACAGAGGGCCAAAATTGAATGCTATGCTTAGATTAGGGGTTTTGCAACCTGAGGTCTATAAACAAAGTCTTCCTGGAAGTAATTGTAAGCATCCTGAAATAAAAAAGCAAGAATATGAAGAAGTAGTTCAGACTGTTAATACAGATTTCTCTCCATATCTGATTTCAGATAACTTAGAACAGCCTATGGGAAGTAGTCATGCATCTCAGGTTTGTTCTGAGACACCTGATGACCTGTTAGATGATGGTGAAATAAAGGAAGATACTAGTTTTGCTGAAAATGACATTAAGGAAAGTTCTGCTGTTTTTAGCAAAAGCGTCCAGAAAGGAGAGCTTAGCAGGAGTCCTAGCCCTTTCACCCATACACATTTGGCTCAGGGTTACCGAAGAGGGGCCAAGAAATTAGAGTCCTCAGAAGAGAACTTATCTAGTGAGGATGAAGAGCTTCCCTGCTTCCAACACTTGTTATTTGGTAAAGTAAACAATATACCTTCTCAGTCTACTAGGCATAGCACCGTTGCTACCGAGTGTCTGTCTAAGAACACAGAGGAGAATTTATTATCATTGAAGAATAGCTTAAATGACTGCAGTAACCAGGTAATATTGGCAAAGGCATCTCAGGAACATCACCTTAGTGAGGAAACAAAATGTTCTGCTAGCTTGTTTTCTTCACAGTGCAGTGAATTGGAAGACTTGACTGCAAATACAAACACCCAGGATCCTTTCTTGATTGGTTCTTCCAAACAAATGAGGCATCAGTCTGAAAGCCAGGGAGTTGGTCTGAGTGACAAGGAATTGGTTTCAGATGATGAAGAAAGAGGAACGGGCTTGGAAGAAAATAATCAAGAAGAGCAAAGCATGGATTCAAACTTAGGTGAAGCAGCATCTGGGTGTGAGAGTGAAACAAGCGTCTCTGAAGACTGCTCAGGGCTATCCTCTCAGAGTGACATTTTAACCACTCAGCAGAGGGATACCATGCAACATAACCTGATAAAGCTCCAGCAGGAAATGGCTGAACTAGAAGCTGTGTTAGAACAGCATGGGAGCCAGCCTTCTAACAGCTACCCTTCCATCATAAGTGACTCTTCTGCCCTTGAGGACCTGCGAAATCCAGAACAAAGCACATCAGAAAAAGCAGTATTAACTTCACAGAAAAGTAGTGAATACCCTATAAGCCAGAATCCAGAAGGCCTTTCTGCTGACAAGTTTGAGGTGTCTGCAGATAGTTCTACCAGTAAAAATAAAGAACCAGGAGTGGAAAGGTCATCCCCTTCTAAATGCCCATCATTAGATGATAGGTGGTACATGCACAGTTGCTCTGGGAGTCTTCAGAATAGAAACTACCCATCTCAAGAGGAGCTCATTAAGGTTGTTGATGTGGAGGAGCAACAGCTGGAAGAGTCTGGGCCACACGATTTGACGGAAACATCTTACTTGCCAAGGCAAGATCTAGAGGGAACCCCTTACCTGGAATCTGGAATCAGCCTCTTCTCTGATGACCCTGAATCTGATCCTTCTGAAGACAGAGCCCCAGAGTCAGCTCGTGTTGGCAACATACCATCTTCAACCTCTGCATTGAAAGTTCCCCAATTGAAAGTTGCAGAATCTGCCCAGAGTCCAGCTGCTGCTCATACTACTGATACTGCTGGGTATAATGCAATGGAAGAAAGTGTGAGCAGGGAGAAGCCAGAATTGACAGCTTCAACAGAAAGGGTCAACAAAAGAATGTCCATGGTGGTGTCTGGCCTGACCCCAGAAGAATTTATGCTCGTGTACAAGTTTGCCAGAAAACACCACATCACTTTAACTAATCTAATTACTGAAGAGACTACTCATGTTGTTATGAAAACAGATGCTGAGTTTGTGTGTGAACGGACACTGAAATATTTTCTAGGAATTGCGGGAGGAAAATGGGTAGTTAGCTATTTCTGGGTGACCCAGTCTATTAAAGAAAGAAAAATGCTGAATGAGCATGATTTTGAAGTCAGAGGAGATGTGGTCAATGGAAGAAACCACCAAGGTCCAAAGCGAGCAAGAGAATCCCAGGACAGAAAGATCTTCAGGGGGCTAGAAATCTGTTGCTATGGGCCCTTCACCAACATGCCCACAGATCAACTGGAATGGATGGTACAGCTGTGTGGTGCTTCTGTGGTGAAGGAGCTTTCATCATTCACCCTTGGCACAGGTGTCCACCCAATTGTGGTTGTGCAGCCAGATGCCTGGACAGAGGACAATGGCTTCCATGCAATTGGGCAGATGTGTGAGGCACCTGTGGTGACCCGAGAGTGGGTGTTGGACAGTGTAGCACTCTACCAGTGCCAGGAGCTGGACACCTACCTGATACCCCAGATCCCCCACAGCCACTACTGACTGCAGCCAGCCACAGGTACAGAGCCACAGGACCCCAAGAATGAGCTTACAAAGTGGCCTTTCCAGGCCCTGGGAGCTCCTCTCACTCTTCAGTCCTTCTACTGTCCTGGCTACTAAATATTTTATGTACATCAGCCTGAAAAGGACTTCTGGCTATGCAAGGGTCCCTTAAAGATTTTCTGCTTGAAGTCTCCCTTGGAAATCTGCCATGAGCACAAAATTATGGTAATTTTTCACCTGAGAAGATTTTAAAACCATTTAAACGCCACCAATTGAGCAAGATGCTGATTCATTATTTATCAGCCCTATTCTTTCTATTCAGGCTGTTGTTGGCTTAGGGCTGGAAGCACAGAGTGGCTTGGCCTCAAGAGAATAGCTGGTTTCCCTAAGTTTACTTCTCTAAAACCCTGTGTTCACAAAGGCAGAGAGTCAGACCCTTCAATGGAAGGAGAGTGCTTGGGATCGATTATGTGACTTAAAGTCAGAATAGTCCTTGGGCAGTTCTCAAATGTTGGAGTGGAACATTGGGGAGGAAATTCTGAGGCAGGTATTAGAAATGAAAAGGAAACTTGAAACCTGGGCATGGTGGCTCACGCCTGTAATCCCAGCACTTTGGGAGGCCAAGGTGGGCAGATCACTGGAGGTCAGGAGTTCGAAACCAGCCTGGCCAACATGGTGAAACCCCATCTCTACTAAAAATACAGAAATTAGCCGGTCATGGTGGTGGACACCTGTAATCCCAGCTACTCAGGTGGCTAAGGCAGGAGAATCACTTCAGCCCGGGAGGTGGAGGTTGCAGTGAGCCAAGATCATACCACGGCACTCCAGCCTGGGTGACAGTGAGACTGTGGCTCAAAAAAAAAAAAAAAAAAAGGAAAATGAAACTAGAAGAGATTTCTAAAAGTCTGAGATATATTTGCTAGATTTCTAAAGAATGTGTTCTAAAACAGCAGAAGATTTTCAAGAACCGGTTTCCAAAGACAGTCTTCTAATTCCTCATTAGTAATAAGTAAAATGTTTATTGTTGTAGCTCTGGTATATAATCCATTCCTCTTAAAATATAAGACCTCTGGCATGAATATTTCATATCTATAAAATGACAGATCCCACCAGGAAGGAAGCTGTTGCTTTCTTTGAGGTGATTTTTTTCCTTTGCTCCCTGTTGCTGAAACCATACAGCTTCATAAATAATTTTGCTTGCTGAAGGAAGAAAAAGTGTTTTTCATAAACCCATTATCCAGGACTGTTTATAGCTGTTGGAAGGACTAGGTCTTCCCTAGCCCCCCCAGTGTGCAAGGGCAGTGAAGACTTGATTGTACAAAATACGTTTTGTAAATGTTGTGCTGTTAACACTGCAAATAAACTTGGTAGCAAACACTTCCAAAAAAAAAAAAAAAAAA", + "expected_refget": "SQ.jj1RAXMGdOU-D39IRYnDX_fXcM7LQvb6", + "non_canonical_refgets": [ + "SQ.bh0RAohFfTjG5VAsjisxENDHZvpHIO39" + ], + "note": "cdot's genome-assembled transcript (7207 nt) instead of NCBI's record (7224 nt)." + }, + "NP_001346945.1": { + "sequence": "MAEQVALSRTQVCGILREELFQGDAFHQSDTHIFIIMGASGDLAKKKIYPTIWWLFRDGLLPENTFIVGYARSRLTVADIRKQSEPFFKATPEEKLKLEDFFARNSYVAGQYDDAASYQRLNSHMNALHLGSQANRLFYLALPPTVYEAVTKNIHESCMSQIGWNRIIVEKPFGRDLQSSDRLSNHISSLFREDQIYRIDHYLGKEMVQNLMVLRFANRIFGPIWNRDNIACVILTFKEPFGTEGRGGYFDEFGIIRDVMQNHLLQMLCLVAMEKPASTNSDDVRDEKVKVLKCISEVQANNVVLGQYVGNPDGEGEATKGYLDDPTVPRGSTTATFAAVVLYVENERWDGVPFILRCGKALNERKAEVRLQFHDVAGDIFHQQCKRNELVIRVQPNEAVYTKMMTKKPGMFFNPEESELDLTYGNRYKNVKLPDAYERLILDVFCGSQMHFVRSDELREAWRIFTPLLHQIELEKPKPIPYIYGSRGPTEADELMKRVGFQYEGTYKWVNPHKL", + "expected_refget": "SQ.j1QRpZEz0P5Ke1zpojiPxviTGGIG4dhN", + "non_canonical_refgets": [ + "SQ._KAVUSriCpoIGu4Tu7eJLZZsEPHQ9dx3" + ], + "note": "Second accession observed with two refgets on staging (2026-09-30)." + } + } +} diff --git a/tests/lib/test_seqrepo_contract.py b/tests/lib/test_seqrepo_contract.py new file mode 100644 index 000000000..58e7383ab --- /dev/null +++ b/tests/lib/test_seqrepo_contract.py @@ -0,0 +1,143 @@ +"""Golden tests for the accession -> refget contract shared with dcd_mapping. + +A VRS allele digest covers the ``refgetAccession`` of the sequence its location sits on, and the allele +table deduplicates on that digest. The mapper and reverse translation each build alleles, so if they +resolve an accession to different sequences they mint different digests for the same variant and the +copies never merge. That happened: the mapper's fetcher chain lacked hgvs's SeqRepo-backed ``SeqFetcher``, +so cdot returned a transcript assembled from the genome (7207 nt) rather than NCBI's NM_007294.3 record +(7224 nt), and the mapper minted ``SQ.bh0R…`` where reverse translation minted ``SQ.jj1R…``. Nothing failed, +because both digests were internally consistent. + +The two repos run different VRS versions and cannot import each other, so the contract is pinned by a +fixture. ``tests/fixtures/canonical_accession_sequences.json`` is a byte-identical copy of the one in +dcd_mapping; it holds NCBI's records and the refget each must have. If either repo drifts, its copy of +these tests fails. Expected refgets are ``sha512t24u(sequence)``, so they are correct by construction. +""" + +# ruff: noqa: E402 +import json +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + +pytest.importorskip("biocommons.seqrepo") + +from biocommons.seqrepo import SeqRepo +from ga4gh.core import sha512t24u +from hgvs.dataproviders.seqfetcher import SeqFetcher +from hgvs.exceptions import HGVSDataNotAvailableError + +from mavedb.data_providers import services +from mavedb.lib.seqrepo import AmbiguousSequenceError, SequenceNotFoundError, resolve_refget +from mavedb.lib.vrs_utils import AlleleRefgetMismatchError, verify_allele_refget + +FIXTURE = Path(__file__).parents[1] / "fixtures" / "canonical_accession_sequences.json" +ACCESSIONS = json.loads(FIXTURE.read_text())["accessions"] + + +@pytest.fixture +def seqrepo_dir(tmp_path: Path) -> Path: + """Build a small SeqRepo of only the canonical fixture sequences, stored under ``refseq:``.""" + root = tmp_path / "seqrepo" + root.mkdir() + sr = SeqRepo(str(root), writeable=True) + for accession, record in ACCESSIONS.items(): + sr.store(record["sequence"], [{"namespace": "refseq", "alias": accession}]) + sr.commit() + return root + + +@pytest.fixture +def sr(seqrepo_dir: Path) -> SeqRepo: + return SeqRepo(str(seqrepo_dir)) + + +@pytest.mark.parametrize("accession", ACCESSIONS) +def test_fixture_refget_is_digest_of_sequence(accession): + """Guards the fixture itself against a hand-edit that desynchronises sequence and refget.""" + record = ACCESSIONS[accession] + assert record["expected_refget"] == f"SQ.{sha512t24u(record['sequence'].encode())}" + assert record["expected_refget"] not in record["non_canonical_refgets"] + + +@pytest.mark.parametrize("accession", ACCESSIONS) +def test_resolve_refget_is_the_canonical_refget(accession, sr): + record = ACCESSIONS[accession] + refget = resolve_refget(sr, accession) + assert refget == record["expected_refget"] + assert refget not in record["non_canonical_refgets"] + + +def test_missing_accession_is_reported(sr): + with pytest.raises(SequenceNotFoundError): + resolve_refget(sr, "NM_000000.1") + + +def test_more_than_one_sequence_is_an_error_not_a_pick(): + sr = MagicMock() + sr.aliases.find_aliases.return_value = [{"seq_id": "aaa"}, {"seq_id": "bbb"}] + with pytest.raises(AmbiguousSequenceError): + resolve_refget(sr, "NM_007294.3") + + +@pytest.mark.parametrize("accession", ACCESSIONS) +def test_seqfetcher_serves_ncbi_records_ahead_of_the_fasta_fallback(accession, seqrepo_dir, monkeypatch): + """The chain feeding hgvs validation must resolve accessions from SeqRepo before the genome-assembled + FASTA fallback. The FASTA fetcher is stubbed to refuse, so a chain without the SeqRepo-backed fetcher in + front (the original defect in dcd_mapping) cannot serve the sequence. + """ + monkeypatch.setenv("HGVS_SEQREPO_DIR", str(seqrepo_dir)) + fasta = MagicMock(source="stub fasta") + fasta.fetch_seq.side_effect = HGVSDataNotAvailableError("no fasta in test") + with ( + patch.object(services, "FastaSeqFetcher", return_value=fasta), + patch.object(services, "GENOMIC_FASTA_FILES", ["stub.fna"]), + ): + chain = services.seqfetcher() + + assert isinstance(chain.seq_fetchers[0], SeqFetcher) + sequence = chain.fetch_seq(accession) + record = ACCESSIONS[accession] + assert sequence == record["sequence"] + assert f"SQ.{sha512t24u(sequence.encode())}" == record["expected_refget"] + + +def _allele(refget: str) -> dict: + return { + "type": "Allele", + "location": {"type": "SequenceLocation", "sequenceReference": {"refgetAccession": refget}}, + } + + +@pytest.mark.parametrize("accession", ACCESSIONS) +def test_insert_check_accepts_the_canonical_refget(accession, sr): + refget = ACCESSIONS[accession]["expected_refget"] + verify_allele_refget(_allele(refget), f"{accession}:c.1A>G", sr, subject="test") + + +@pytest.mark.parametrize("accession", ACCESSIONS) +def test_insert_check_rejects_a_non_canonical_refget(accession, sr): + """The original failure: an allele built on a sequence that is not SeqRepo's for its accession.""" + bad = ACCESSIONS[accession]["non_canonical_refgets"][0] + with pytest.raises(AlleleRefgetMismatchError): + verify_allele_refget(_allele(bad), f"{accession}:c.1A>G", sr, subject="test") + + +def test_insert_check_inspects_every_member_of_a_block(sr): + good = ACCESSIONS["NM_007294.3"]["expected_refget"] + bad = ACCESSIONS["NM_007294.3"]["non_canonical_refgets"][0] + block = {"type": "CisPhasedBlock", "members": [_allele(good), _allele(bad)]} + with pytest.raises(AlleleRefgetMismatchError): + verify_allele_refget(block, "NM_007294.3:c.[1A>G;5C>T]", sr, subject="test") + + +def test_insert_check_ignores_accessions_seqrepo_is_not_the_authority_for(sr): + verify_allele_refget(_allele("SQ.anything"), "target_label:c.1A>G", sr, subject="test") + verify_allele_refget(_allele("SQ.anything"), "c.1A>G", sr, subject="test") + + +def test_insert_check_alerts_but_does_not_raise_for_an_unresolvable_accession(sr, caplog): + with caplog.at_level("ERROR"): + verify_allele_refget(_allele("SQ.anything"), "NM_000000.1:c.1A>G", sr, subject="test") + assert "Cannot verify the refget" in caplog.text From a8a2593c3087c16697cbc5fc4daca92ca02c322c Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 30 Sep 2026 14:42:43 -0700 Subject: [PATCH 192/252] feat(alleles): enforce one allele per HGVS expression The same HGVS expression under two digests means two writers built the allele on different sequences (NM_007294.3 and NP_001346945.1 on staging), so the copies never deduplicate. Nothing else detects it, because both digests are internally consistent. - Allele.hgvs is a hybrid property: the populated one of hgvs_g/hgvs_c/hgvs_p, None when none is, on the instance and in SQL alike. transcript is derived from it, and the unique index shares its SQL form so the two cannot drift. - uq_alleles_hgvs is a unique index on that expression, built CONCURRENTLY by migration a7c3e5f9b1d2. The migration documents the pre-check query and how to remove existing duplicates. - get_or_create_allele raises AlleleIdentityConflictError, naming both digests and refgets and how to debug it, before inserting an allele whose HGVS another allele holds. The insert is flushed in a savepoint so a concurrent writer that passes the check hits the index and raises the same error. - seed_csv_substrate gives each variant its own HGVS, since distinct variants are distinct alleles; the download tests expect the shifted values. --- ...3e5f9b1d2_unique_allele_hgvs_expression.py | 64 +++++++++++++++++ src/mavedb/lib/variant_detail.py | 2 +- src/mavedb/lib/variant_translations.py | 61 ++++++++++++++-- src/mavedb/lib/vrs_utils.py | 4 +- src/mavedb/models/allele.py | 49 +++++++++---- tests/helpers/util/score_set.py | 21 +++++- tests/lib/test_variant_translations.py | 72 +++++++++++++++++++ tests/models/test_allele_hgvs.py | 47 ++++++++++++ tests/routers/test_score_set.py | 21 ++++-- 9 files changed, 312 insertions(+), 29 deletions(-) create mode 100644 alembic/versions/a7c3e5f9b1d2_unique_allele_hgvs_expression.py create mode 100644 tests/models/test_allele_hgvs.py diff --git a/alembic/versions/a7c3e5f9b1d2_unique_allele_hgvs_expression.py b/alembic/versions/a7c3e5f9b1d2_unique_allele_hgvs_expression.py new file mode 100644 index 000000000..1ead4d392 --- /dev/null +++ b/alembic/versions/a7c3e5f9b1d2_unique_allele_hgvs_expression.py @@ -0,0 +1,64 @@ +"""enforce one allele per HGVS expression + +Revision ID: a7c3e5f9b1d2 +Revises: 024370c4bca7 +Create Date: 2026-09-30 + +Adds a unique index on the allele's HGVS expression (whichever of hgvs_g / hgvs_c / hgvs_p is set; the +model guarantees exactly one). An allele digest covers the refget of the sequence the allele sits on, +and alleles deduplicate on that digest. The same expression under two digests therefore means two +writers built the allele on different sequences (this happened with NM_007294.3 and NP_001346945.1 when +the mapper and reverse translation resolved one accession to different sequences), and the copies never +merge: reverse translation's fold-in misses the measured allele and the convergent / projection +labelling breaks. Nothing else detects it, because both digests are internally consistent. This index +moves the failure to write time, in get_or_create_allele, which raises AlleleIdentityConflictError with +both digests, both refgets and how to debug it. + +Built CONCURRENTLY: mapping and reverse translation write to alleles, and a plain CREATE UNIQUE INDEX +would block them. A concurrent build that fails leaves an INVALID index behind, which must be dropped +before retrying: + + DROP INDEX CONCURRENTLY IF EXISTS uq_alleles_hgvs; + +Assumes no two alleles share an HGVS expression. Check first: + + SELECT coalesce(hgvs_g, hgvs_c, hgvs_p) AS hgvs, count(*), array_agg(id ORDER BY id) + FROM alleles + GROUP BY 1 HAVING count(*) > 1; + +If that returns rows, remove the duplicates before applying this migration. In each group keep the +allele whose refget is the one SeqRepo holds for its accession. Delete the other allele and the rows +that reference it (mapping_record_alleles, gnomad_allele_links, vep_allele_consequences, +clinvar_allele_links, annotation_event; all ON DELETE RESTRICT), then remap the affected score sets. +Delete before remapping: get_or_create_allele refuses to create an allele while another holds its HGVS. +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "a7c3e5f9b1d2" +down_revision = "024370c4bca7" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + with op.get_context().autocommit_block(): + op.create_index( + "uq_alleles_hgvs", + "alleles", + [sa.text("(coalesce(hgvs_g, hgvs_c, hgvs_p))")], + unique=True, + postgresql_concurrently=True, + ) + op.execute( + "COMMENT ON INDEX uq_alleles_hgvs IS " + "'One allele per HGVS expression. A violation means two writers disagree about the sequence an " + "allele sits on (see mavedb.lib.variant_translations.AlleleIdentityConflictError).'" + ) + + +def downgrade() -> None: + with op.get_context().autocommit_block(): + op.drop_index("uq_alleles_hgvs", table_name="alleles", postgresql_concurrently=True) diff --git a/src/mavedb/lib/variant_detail.py b/src/mavedb/lib/variant_detail.py index 1ad8fdf14..6f6f805db 100644 --- a/src/mavedb/lib/variant_detail.py +++ b/src/mavedb/lib/variant_detail.py @@ -236,7 +236,7 @@ def get_variant_detail( is_convergent = is_convergent_encoding(allele.level, link.projection_group, defining_group=defining_group) alleles[allele.vrs_digest] = AlleleIdentity( level=allele.level, - hgvs=allele.hgvs_g or allele.hgvs_c or allele.hgvs_p, + hgvs=allele.hgvs, clingen_allele_id=allele.clingen_allele_id, is_focus=link.is_authoritative, relation=relation.value if relation is not None else None, diff --git a/src/mavedb/lib/variant_translations.py b/src/mavedb/lib/variant_translations.py index 31d89e6df..d4597b4e1 100644 --- a/src/mavedb/lib/variant_translations.py +++ b/src/mavedb/lib/variant_translations.py @@ -16,8 +16,10 @@ from sqlalchemy import select from sqlalchemy.dialects.postgresql import insert from sqlalchemy.engine import CursorResult +from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Session +from mavedb.lib.vrs_utils import location_refgets from mavedb.models.allele import Allele from mavedb.models.variant_translation import VariantTranslation @@ -49,19 +51,70 @@ def upsert_variant_translations(db: Session, translations: list[tuple[str, str]] return created, existing +class AlleleIdentityConflictError(ValueError): + """Raised when an allele's HGVS expression already exists under a different vrs_digest. + + An allele digest covers the refget of the sequence the allele sits on, and alleles deduplicate on + it. Two digests for one HGVS expression therefore mean two writers built the allele on different + sequences (most often by resolving its reference accession to different sequences), so the + copies would never merge. Nothing is wrong with the variant; the writers disagree about the + reference. + + To debug: compare the two refgets in the message with the sequence SeqRepo holds for the accession + (``SELECT * FROM seqalias WHERE alias = ''`` in the SeqRepo's aliases.sqlite3); check that + the mapper and the worker read the same SeqRepo (``SEQREPO_ROOT_DIR`` and ``HGVS_SEQREPO_DIR``) and + that dcd_mapping is current. Duplicates that already exist have to be removed, with the rows + that reference them, and the affected score sets remapped. Remove them first: this check refuses to + create an allele while another allele holds its HGVS. + """ + + +def _identity_conflict(existing: Allele, incoming: Allele) -> AlleleIdentityConflictError: + return AlleleIdentityConflictError( + f"Allele identity conflict for {incoming.hgvs}: an allele with this HGVS already " + f"exists under a different digest. Existing: id={existing.id} digest={existing.vrs_digest} " + f"refgets={location_refgets(existing.post_mapped or {})}. Incoming: digest={incoming.vrs_digest} " + f"refgets={location_refgets(incoming.post_mapped or {})}. Two writers disagree on the sequence " + "this allele sits on; see AlleleIdentityConflictError for how to debug this error." + ) + + def get_or_create_allele(db: Session, allele_draft: Allele) -> Allele: """Return the existing Allele matching ``allele_draft``'s vrs_digest, else add the draft. This is a get-or-create, not an upsert: a matching row is returned untouched, and on - a miss the draft is added to the session (not flushed). The draft is never used to + a miss the draft is added to the session and flushed. The draft is never used to update an existing row. - NOTE: This function does not persist the returned Allele to the database; the caller - is responsible for committing the session. + :raise AlleleIdentityConflictError: if the draft's HGVS already exists under a different digest + + NOTE: This function does not commit; the caller is responsible for committing the session. """ existing = db.scalars(select(Allele).where(Allele.vrs_digest == allele_draft.vrs_digest)).one_or_none() if existing is not None: return existing - db.add(allele_draft) + hgvs = allele_draft.hgvs + if hgvs: + conflicting = db.scalars(select(Allele).where(Allele.hgvs == hgvs)).first() + if conflicting is not None: + raise _identity_conflict(conflicting, allele_draft) + + # Flushed inside a savepoint so a concurrent writer that slipped past the check above surfaces as + # the same error, through the unique index, without poisoning the caller's transaction. + try: + with db.begin_nested(): + db.add(allele_draft) + db.flush() + + except IntegrityError as e: + if "uq_alleles_hgvs" not in str(e.orig) or not hgvs: + raise + + conflicting = db.scalars(select(Allele).where(Allele.hgvs == hgvs)).first() + if conflicting is None: + raise + + raise _identity_conflict(conflicting, allele_draft) from e + return allele_draft diff --git a/src/mavedb/lib/vrs_utils.py b/src/mavedb/lib/vrs_utils.py index 97b6d7de1..f96c30203 100644 --- a/src/mavedb/lib/vrs_utils.py +++ b/src/mavedb/lib/vrs_utils.py @@ -292,7 +292,7 @@ def canonical_variation_document(document: Mapping[str, Any], *, subject: str) - return variation.model_dump(mode="json", exclude_none=True), identifier -def _location_refgets(variation: Mapping[str, Any]) -> list[str]: +def location_refgets(variation: Mapping[str, Any]) -> list[str]: """Collect the refgetAccession of the variation's location, or of each member of a block.""" members = variation.get("members") or [variation] refgets = [] @@ -332,7 +332,7 @@ def verify_allele_refget(variation: Mapping[str, Any], hgvs: str, sr: SeqRepo, * ) return - for refget in _location_refgets(variation): + for refget in location_refgets(variation): if refget != expected: msg = ( f"Allele for {subject} sits on {refget}, but SeqRepo holds {expected} for {accession}. " diff --git a/src/mavedb/models/allele.py b/src/mavedb/models/allele.py index 67eadd519..d5ae12e03 100644 --- a/src/mavedb/models/allele.py +++ b/src/mavedb/models/allele.py @@ -1,9 +1,10 @@ from datetime import date from typing import TYPE_CHECKING, Any, Optional -from sqlalchemy import Column, Date, Index, Integer, String, UniqueConstraint, func +from sqlalchemy import Date, Index, Integer, String, UniqueConstraint, func from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.ext.hybrid import hybrid_property +from sqlalchemy.orm import mapped_column from sqlalchemy.orm import Mapped, relationship from mavedb.db.base import Base @@ -13,23 +14,28 @@ from .mapping_record_allele import MappingRecordAllele +def _coalesce_hgvs(hgvs_g, hgvs_c, hgvs_p): + """SQL form of ``Allele.hgvs``.""" + return func.coalesce(hgvs_g, hgvs_c, hgvs_p) + + class Allele(Base): __tablename__ = "alleles" - id: Mapped[int] = Column(Integer, primary_key=True) + id: Mapped[int] = mapped_column(Integer, primary_key=True) - vrs_digest: Mapped[str] = Column(String, nullable=False) - level = Column(String(length=16), nullable=False) + vrs_digest: Mapped[str] = mapped_column(String, nullable=False) + level: Mapped[str] = mapped_column(String(length=16), nullable=False) - hgvs_g = Column(String, nullable=True) - hgvs_c = Column(String, nullable=True) - hgvs_p = Column(String, nullable=True) + hgvs_g: Mapped[Optional[str]] = mapped_column(String, nullable=True) + hgvs_c: Mapped[Optional[str]] = mapped_column(String, nullable=True) + hgvs_p: Mapped[Optional[str]] = mapped_column(String, nullable=True) - clingen_allele_id = Column(String, nullable=True) - post_mapped: Optional[Any] = Column(JSONB(none_as_null=True), nullable=True) + clingen_allele_id: Mapped[Optional[str]] = mapped_column(String, nullable=True) + post_mapped: Mapped[Optional[Any]] = mapped_column(JSONB(none_as_null=True), nullable=True) - created_at = Column(Date, nullable=False, default=date.today) - updated_at = Column(Date, nullable=False, default=date.today, onupdate=date.today) + created_at: Mapped[date] = mapped_column(Date, nullable=False, default=date.today) + updated_at: Mapped[date] = mapped_column(Date, nullable=False, default=date.today, onupdate=date.today) @hybrid_property def transcript(self) -> str: @@ -39,12 +45,25 @@ def transcript(self) -> str: string's accession. Derived rather than stored so it cannot drift from the HGVS column it duplicates. """ - return extract_accession(self.hgvs_g or self.hgvs_c or self.hgvs_p or "") + return extract_accession(self.hgvs or "") + + @hybrid_property + def hgvs(self) -> Optional[str]: + """The populated HGVS expression: whichever of hgvs_g/hgvs_c/hgvs_p is set (derived, not stored). + + None when none is, on the instance and in SQL alike (``coalesce`` yields NULL). + """ + return self.hgvs_g or self.hgvs_c or self.hgvs_p + + @hgvs.inplace.expression + @classmethod + def _hgvs_expression(cls): + return _coalesce_hgvs(cls.hgvs_g, cls.hgvs_c, cls.hgvs_p) @transcript.inplace.expression @classmethod def _transcript_expression(cls): - return func.split_part(func.coalesce(cls.hgvs_g, cls.hgvs_c, cls.hgvs_p), ":", 1) + return func.split_part(cls.hgvs, ":", 1) mapping_record_links: Mapped[list["MappingRecordAllele"]] = relationship( "MappingRecordAllele", @@ -60,4 +79,8 @@ def _transcript_expression(cls): Index("ix_alleles_vrs_digest", "vrs_digest"), Index("ix_alleles_level", "level"), Index("ix_alleles_clingen_allele_id", "clingen_allele_id"), + # One row per HGVS expression. Two digests for the same expression mean two writers built the + # allele on different sequences, so the copies would never deduplicate. Fails at write time, in + # get_or_create_allele, where the cause is. See AlleleIdentityConflictError. + Index("uq_alleles_hgvs", _coalesce_hgvs(hgvs_g, hgvs_c, hgvs_p), unique=True), ) diff --git a/tests/helpers/util/score_set.py b/tests/helpers/util/score_set.py index 35fd9e382..3801910d5 100644 --- a/tests/helpers/util/score_set.py +++ b/tests/helpers/util/score_set.py @@ -1,3 +1,4 @@ +import re from copy import deepcopy from datetime import date from typing import Any, Dict, Optional @@ -239,6 +240,18 @@ def seed_annotation_substrate(db, score_set, *, skip_first=False, pre_mapped=Non return variants +def shift_hgvs_position(hgvs, offset): + """Move the first position in an HGVS expression by ``offset``; ``None`` and offset 0 pass through.""" + if hgvs is None or offset == 0: + return hgvs + + # DNA positions follow the type letter directly (g.123A>G); a protein position follows a three-letter + # residue (p.Gly123Arg). + return re.sub( + r"(:[gcpn]\.\(?(?:[A-Za-z]{3})?)(\d+)", lambda m: f"{m.group(1)}{int(m.group(2)) + offset}", hgvs, count=1 + ) + + def seed_csv_substrate( db, score_set, @@ -271,10 +284,14 @@ def seed_csv_substrate( .where(ScoreSetDbModel.urn == score_set["urn"]) .order_by(VariantDbModel.id) ).all() - level_hgvs = {"genomic": hgvs_g, "cdna": hgvs_c, "protein": hgvs_p} - assay_hgvs = level_hgvs[assay_level] + hgvs_g_in, hgvs_c_in, hgvs_p_in = hgvs_g, hgvs_c, hgvs_p for index, variant in enumerate(variants): + # Distinct variants are distinct alleles, and alleles are unique per HGVS expression, so each + # variant gets its own position. The first variant keeps the HGVS exactly as given. + hgvs_g, hgvs_c, hgvs_p = (shift_hgvs_position(h, index) for h in (hgvs_g_in, hgvs_c_in, hgvs_p_in)) + level_hgvs = {"genomic": hgvs_g, "cdna": hgvs_c, "protein": hgvs_p} + assay_hgvs = level_hgvs[assay_level] annotated = annotate == "all" or index == 0 specs = [ AlleleSpec( diff --git a/tests/lib/test_variant_translations.py b/tests/lib/test_variant_translations.py index 832152c7f..addb9a0c8 100644 --- a/tests/lib/test_variant_translations.py +++ b/tests/lib/test_variant_translations.py @@ -91,3 +91,75 @@ def test_fully_overlapping_second_call_inserts_nothing(self, session): session.commit() rows = session.scalars(select(VariantTranslation)).all() assert len(rows) == 2 + + +def _draft(digest, *, hgvs_c=None, hgvs_p=None, refget=None, level="cdna"): + from mavedb.models.allele import Allele + + post_mapped = {"type": "Allele", "location": {"sequenceReference": {"refgetAccession": refget}}} if refget else None + return Allele(vrs_digest=digest, level=level, hgvs_c=hgvs_c, hgvs_p=hgvs_p, post_mapped=post_mapped) + + +@pytest.mark.integration +class TestGetOrCreateAllele: + """One allele per digest, and one allele per HGVS expression.""" + + def test_creates_and_flushes_a_new_allele(self, session): + from mavedb.lib.variant_translations import get_or_create_allele + + allele = get_or_create_allele(session, _draft("digest-a", hgvs_c="NM_000001.1:c.1A>G")) + + assert allele.id is not None + + def test_returns_the_existing_allele_for_the_same_digest(self, session): + from mavedb.lib.variant_translations import get_or_create_allele + + first = get_or_create_allele(session, _draft("digest-a", hgvs_c="NM_000001.1:c.1A>G")) + again = get_or_create_allele(session, _draft("digest-a", hgvs_c="NM_000001.1:c.1A>G")) + + assert again is first + + def test_alleles_without_an_hgvs_expression_do_not_conflict(self, session): + from mavedb.lib.variant_translations import get_or_create_allele + + get_or_create_allele(session, _draft("digest-a")) + get_or_create_allele(session, _draft("digest-b")) + + def test_same_hgvs_under_a_different_digest_is_a_conflict_that_explains_itself(self, session): + from mavedb.lib.variant_translations import AlleleIdentityConflictError, get_or_create_allele + + hgvs = "NM_007294.3:c.5448A>T" + get_or_create_allele(session, _draft("digest-good", hgvs_c=hgvs, refget="SQ.jj1R")) + + with pytest.raises(AlleleIdentityConflictError) as excinfo: + get_or_create_allele(session, _draft("digest-bad", hgvs_c=hgvs, refget="SQ.bh0R")) + + message = str(excinfo.value) + for expected in (hgvs, "digest-good", "digest-bad", "SQ.jj1R", "SQ.bh0R", "AlleleIdentityConflictError"): + assert expected in message + + def test_a_conflict_does_not_add_the_draft_or_poison_the_session(self, session): + from mavedb.lib.variant_translations import AlleleIdentityConflictError, get_or_create_allele + from mavedb.models.allele import Allele + + hgvs = "NM_000001.1:c.1A>G" + get_or_create_allele(session, _draft("digest-good", hgvs_c=hgvs)) + with pytest.raises(AlleleIdentityConflictError): + get_or_create_allele(session, _draft("digest-bad", hgvs_c=hgvs)) + + digests = set(session.scalars(select(Allele.vrs_digest))) + assert digests == {"digest-good"} + get_or_create_allele(session, _draft("digest-other", hgvs_c="NM_000001.1:c.2A>G")) + + def test_the_database_index_backstops_a_writer_that_skips_the_check(self, session): + """A concurrent writer past the application check must still be stopped, by name.""" + from sqlalchemy.exc import IntegrityError + + hgvs = "NM_000001.1:c.1A>G" + session.add(_draft("digest-a", hgvs_c=hgvs)) + session.flush() + session.add(_draft("digest-b", hgvs_c=hgvs)) + + with pytest.raises(IntegrityError, match="uq_alleles_hgvs"): + session.flush() + session.rollback() diff --git a/tests/models/test_allele_hgvs.py b/tests/models/test_allele_hgvs.py new file mode 100644 index 000000000..c079feb00 --- /dev/null +++ b/tests/models/test_allele_hgvs.py @@ -0,0 +1,47 @@ +# ruff: noqa: E402 +"""``Allele.hgvs`` must mean the same thing on an instance and in SQL.""" + +import pytest + +pytest.importorskip("psycopg2") + +from sqlalchemy import select + +from mavedb.models.allele import Allele + +G, C, P = "NC_000017.11:g.1A>G", "NM_000001.1:c.2A>G", "NP_000001.1:p.Ala3Gly" + + +def test_instance_value_prefers_genomic_then_coding_then_protein(): + assert Allele(hgvs_g=G, hgvs_c=C, hgvs_p=P).hgvs == G + assert Allele(hgvs_c=C, hgvs_p=P).hgvs == C + assert Allele(hgvs_p=P).hgvs == P + + +def test_instance_value_is_none_when_nothing_is_set(): + assert Allele().hgvs is None + + +@pytest.mark.integration +def test_the_sql_expression_matches_the_allele_at_every_level(session): + """Regression: a hybrid with no SQL expression compiled ``Allele.hgvs`` to ``hgvs_g`` alone, so a query + for a coding or protein allele silently matched nothing. + """ + alleles = { + G: Allele(vrs_digest="g", level="genomic", hgvs_g=G), + C: Allele(vrs_digest="c", level="cdna", hgvs_c=C), + P: Allele(vrs_digest="p", level="protein", hgvs_p=P), + } + session.add_all(alleles.values()) + session.commit() + + for hgvs, allele in alleles.items(): + assert session.scalars(select(Allele.id).where(Allele.hgvs == hgvs)).all() == [allele.id] + + +@pytest.mark.integration +def test_transcript_still_derives_from_the_same_expression(session): + session.add(Allele(vrs_digest="c", level="cdna", hgvs_c=C)) + session.commit() + + assert session.scalars(select(Allele.vrs_digest).where(Allele.transcript == "NM_000001.1")).all() == ["c"] diff --git a/tests/routers/test_score_set.py b/tests/routers/test_score_set.py index 0fe2a8c96..06d13a518 100644 --- a/tests/routers/test_score_set.py +++ b/tests/routers/test_score_set.py @@ -101,6 +101,7 @@ publish_score_set, seed_annotation_substrate, seed_csv_substrate, + shift_hgvs_position, ) from tests.helpers.util.user import change_ownership from tests.helpers.util.variant import ( @@ -3970,17 +3971,23 @@ def test_download_variants_data_file( ] ) rows = list(reader) - for row in rows: + # seed_csv_substrate gives each variant its own allele, so each row's HGVS is the seeded one shifted by + # the row's position (the first row is unshifted). + for index, row in enumerate(rows): if has_hgvs_g: - assert row["mavedb.post_mapped_hgvs_g"] == mapped_variant["hgvs_g"] - assert row["mavedb.post_mapped_hgvs_c"] == mapped_variant["hgvs_c"] - assert row["mavedb.post_mapped_hgvs_p"] == mapped_variant["hgvs_p"] - assert row["mavedb.post_mapped_hgvs_at_assay_level"] == mapped_variant["hgvs_assay_level"] + assert row["mavedb.post_mapped_hgvs_g"] == shift_hgvs_position(mapped_variant["hgvs_g"], index) + assert row["mavedb.post_mapped_hgvs_c"] == shift_hgvs_position(mapped_variant["hgvs_c"], index) + assert row["mavedb.post_mapped_hgvs_p"] == shift_hgvs_position(mapped_variant["hgvs_p"], index) + assert row["mavedb.post_mapped_hgvs_at_assay_level"] == shift_hgvs_position( + mapped_variant["hgvs_assay_level"], index + ) elif has_hgvs_p: assert row["mavedb.post_mapped_hgvs_g"] == "NA" assert row["mavedb.post_mapped_hgvs_c"] == "NA" - assert row["mavedb.post_mapped_hgvs_p"] == mapped_variant["hgvs_p"] - assert row["mavedb.post_mapped_hgvs_at_assay_level"] == mapped_variant["hgvs_assay_level"] + assert row["mavedb.post_mapped_hgvs_p"] == shift_hgvs_position(mapped_variant["hgvs_p"], index) + assert row["mavedb.post_mapped_hgvs_at_assay_level"] == shift_hgvs_position( + mapped_variant["hgvs_assay_level"], index + ) else: assert row["mavedb.post_mapped_hgvs_g"] == "NA" assert row["mavedb.post_mapped_hgvs_c"] == "NA" From 3af9f4d494ac6e1c41f77d493d08224059808d4c Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 30 Sep 2026 15:04:03 -0700 Subject: [PATCH 193/252] fix(alleles): declare Allele columns with Column so the mypy plugin builds its __init__ The SQLAlchemy mypy plugin only synthesizes constructor kwargs for Column attributes, so mapped_column made every Allele(...) call fail type checking. --- src/mavedb/models/allele.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/src/mavedb/models/allele.py b/src/mavedb/models/allele.py index d5ae12e03..379cdd562 100644 --- a/src/mavedb/models/allele.py +++ b/src/mavedb/models/allele.py @@ -1,10 +1,9 @@ from datetime import date from typing import TYPE_CHECKING, Any, Optional -from sqlalchemy import Date, Index, Integer, String, UniqueConstraint, func +from sqlalchemy import Column, Date, Index, Integer, String, UniqueConstraint, func from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.ext.hybrid import hybrid_property -from sqlalchemy.orm import mapped_column from sqlalchemy.orm import Mapped, relationship from mavedb.db.base import Base @@ -22,20 +21,20 @@ def _coalesce_hgvs(hgvs_g, hgvs_c, hgvs_p): class Allele(Base): __tablename__ = "alleles" - id: Mapped[int] = mapped_column(Integer, primary_key=True) + id: Mapped[int] = Column(Integer, primary_key=True) - vrs_digest: Mapped[str] = mapped_column(String, nullable=False) - level: Mapped[str] = mapped_column(String(length=16), nullable=False) + vrs_digest: Mapped[str] = Column(String, nullable=False) + level: Mapped[str] = Column(String(length=16), nullable=False) - hgvs_g: Mapped[Optional[str]] = mapped_column(String, nullable=True) - hgvs_c: Mapped[Optional[str]] = mapped_column(String, nullable=True) - hgvs_p: Mapped[Optional[str]] = mapped_column(String, nullable=True) + hgvs_g = Column(String, nullable=True) + hgvs_c = Column(String, nullable=True) + hgvs_p = Column(String, nullable=True) - clingen_allele_id: Mapped[Optional[str]] = mapped_column(String, nullable=True) - post_mapped: Mapped[Optional[Any]] = mapped_column(JSONB(none_as_null=True), nullable=True) + clingen_allele_id = Column(String, nullable=True) + post_mapped: Mapped[Optional[Any]] = Column(JSONB(none_as_null=True), nullable=True) - created_at: Mapped[date] = mapped_column(Date, nullable=False, default=date.today) - updated_at: Mapped[date] = mapped_column(Date, nullable=False, default=date.today, onupdate=date.today) + created_at: Mapped[date] = Column(Date, nullable=False, default=date.today) + updated_at: Mapped[date] = Column(Date, nullable=False, default=date.today, onupdate=date.today) @hybrid_property def transcript(self) -> str: From 6ae94912b1a4c0096d070889da882639640abf7d Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 30 Sep 2026 16:12:01 -0700 Subject: [PATCH 194/252] chore(deps): update starlette, fastapi and cryptography to close known vulnerabilities Starlette 1.x needs FastAPI 0.133 or later. Also refreshes anyio, urllib3, sqlparse and virtualenv in the lock. --- poetry.lock | 252 +++++++++++++++++++++++++++++-------------------- pyproject.toml | 6 +- 2 files changed, 153 insertions(+), 105 deletions(-) diff --git a/poetry.lock b/poetry.lock index a90018460..51ecdb81d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -93,19 +93,19 @@ files = [ [[package]] name = "anyio" -version = "4.13.0" +version = "4.15.1" description = "High-level concurrency and networking framework on top of asyncio or Trio" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "anyio-4.13.0-py3-none-any.whl", hash = "sha256:08b310f9e24a9594186fd75b4f73f4a4152069e3853f1ed8bfbf58369f4ad708"}, - {file = "anyio-4.13.0.tar.gz", hash = "sha256:334b70e641fd2221c1505b3890c69882fe4a2df910cba14d97019b90b24439dc"}, + {file = "anyio-4.15.1-py3-none-any.whl", hash = "sha256:6152fdbbf9a77fdec97731721bebf7c4c44f7c29b424b0065826173efc7ed101"}, + {file = "anyio-4.15.1.tar.gz", hash = "sha256:9f28306018cbd6d329e64a36d58256edff76dd996fe423bc957326e578b82a94"}, ] [package.dependencies] idna = ">=2.8" -typing_extensions = {version = ">=4.5", markers = "python_version < \"3.13\""} +typing_extensions = {version = ">=4.16.0", markers = "python_version < \"3.15\""} [package.extras] trio = ["trio (>=0.32.0)"] @@ -1334,76 +1334,79 @@ toml = ["tomli ; python_full_version <= \"3.11.0a6\""] [[package]] name = "cryptography" -version = "46.0.7" +version = "50.0.2" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." optional = true -python-versions = "!=3.9.0,!=3.9.1,>=3.8" +python-versions = "!=3.9.0,!=3.9.1,>=3.9" groups = ["main"] markers = "extra == \"server\"" files = [ - {file = "cryptography-46.0.7-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:ea42cbe97209df307fdc3b155f1b6fa2577c0defa8f1f7d3be7d31d189108ad4"}, - {file = "cryptography-46.0.7-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b36a4695e29fe69215d75960b22577197aca3f7a25b9cf9d165dcfe9d80bc325"}, - {file = "cryptography-46.0.7-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5ad9ef796328c5e3c4ceed237a183f5d41d21150f972455a9d926593a1dcb308"}, - {file = "cryptography-46.0.7-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:73510b83623e080a2c35c62c15298096e2a5dc8d51c3b4e1740211839d0dea77"}, - {file = "cryptography-46.0.7-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:cbd5fb06b62bd0721e1170273d3f4d5a277044c47ca27ee257025146c34cbdd1"}, - {file = "cryptography-46.0.7-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:420b1e4109cc95f0e5700eed79908cef9268265c773d3a66f7af1eef53d409ef"}, - {file = "cryptography-46.0.7-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:24402210aa54baae71d99441d15bb5a1919c195398a87b563df84468160a65de"}, - {file = "cryptography-46.0.7-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:8a469028a86f12eb7d2fe97162d0634026d92a21f3ae0ac87ed1c4a447886c83"}, - {file = "cryptography-46.0.7-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:9694078c5d44c157ef3162e3bf3946510b857df5a3955458381d1c7cfc143ddb"}, - {file = "cryptography-46.0.7-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:42a1e5f98abb6391717978baf9f90dc28a743b7d9be7f0751a6f56a75d14065b"}, - {file = "cryptography-46.0.7-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:91bbcb08347344f810cbe49065914fe048949648f6bd5c2519f34619142bbe85"}, - {file = "cryptography-46.0.7-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:5d1c02a14ceb9148cc7816249f64f623fbfee39e8c03b3650d842ad3f34d637e"}, - {file = "cryptography-46.0.7-cp311-abi3-win32.whl", hash = "sha256:d23c8ca48e44ee015cd0a54aeccdf9f09004eba9fc96f38c911011d9ff1bd457"}, - {file = "cryptography-46.0.7-cp311-abi3-win_amd64.whl", hash = "sha256:397655da831414d165029da9bc483bed2fe0e75dde6a1523ec2fe63f3c46046b"}, - {file = "cryptography-46.0.7-cp314-cp314t-macosx_10_9_universal2.whl", hash = "sha256:d151173275e1728cf7839aaa80c34fe550c04ddb27b34f48c232193df8db5842"}, - {file = "cryptography-46.0.7-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:db0f493b9181c7820c8134437eb8b0b4792085d37dbb24da050476ccb664e59c"}, - {file = "cryptography-46.0.7-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ebd6daf519b9f189f85c479427bbd6e9c9037862cf8fe89ee35503bd209ed902"}, - {file = "cryptography-46.0.7-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:b7b412817be92117ec5ed95f880defe9cf18a832e8cafacf0a22337dc1981b4d"}, - {file = "cryptography-46.0.7-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:fbfd0e5f273877695cb93baf14b185f4878128b250cc9f8e617ea0c025dfb022"}, - {file = "cryptography-46.0.7-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:ffca7aa1d00cf7d6469b988c581598f2259e46215e0140af408966a24cf086ce"}, - {file = "cryptography-46.0.7-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:60627cf07e0d9274338521205899337c5d18249db56865f943cbe753aa96f40f"}, - {file = "cryptography-46.0.7-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:80406c3065e2c55d7f49a9550fe0c49b3f12e5bfff5dedb727e319e1afb9bf99"}, - {file = "cryptography-46.0.7-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:c5b1ccd1239f48b7151a65bc6dd54bcfcc15e028c8ac126d3fada09db0e07ef1"}, - {file = "cryptography-46.0.7-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:d5f7520159cd9c2154eb61eb67548ca05c5774d39e9c2c4339fd793fe7d097b2"}, - {file = "cryptography-46.0.7-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:fcd8eac50d9138c1d7fc53a653ba60a2bee81a505f9f8850b6b2888555a45d0e"}, - {file = "cryptography-46.0.7-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:65814c60f8cc400c63131584e3e1fad01235edba2614b61fbfbfa954082db0ee"}, - {file = "cryptography-46.0.7-cp314-cp314t-win32.whl", hash = "sha256:fdd1736fed309b4300346f88f74cd120c27c56852c3838cab416e7a166f67298"}, - {file = "cryptography-46.0.7-cp314-cp314t-win_amd64.whl", hash = "sha256:e06acf3c99be55aa3b516397fe42f5855597f430add9c17fa46bf2e0fb34c9bb"}, - {file = "cryptography-46.0.7-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:462ad5cb1c148a22b2e3bcc5ad52504dff325d17daf5df8d88c17dda1f75f2a4"}, - {file = "cryptography-46.0.7-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:84d4cced91f0f159a7ddacad249cc077e63195c36aac40b4150e7a57e84fffe7"}, - {file = "cryptography-46.0.7-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:128c5edfe5e5938b86b03941e94fac9ee793a94452ad1365c9fc3f4f62216832"}, - {file = "cryptography-46.0.7-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:5e51be372b26ef4ba3de3c167cd3d1022934bc838ae9eaad7e644986d2a3d163"}, - {file = "cryptography-46.0.7-cp38-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:cdf1a610ef82abb396451862739e3fc93b071c844399e15b90726ef7470eeaf2"}, - {file = "cryptography-46.0.7-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:1d25aee46d0c6f1a501adcddb2d2fee4b979381346a78558ed13e50aa8a59067"}, - {file = "cryptography-46.0.7-cp38-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:cdfbe22376065ffcf8be74dc9a909f032df19bc58a699456a21712d6e5eabfd0"}, - {file = "cryptography-46.0.7-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:abad9dac36cbf55de6eb49badd4016806b3165d396f64925bf2999bcb67837ba"}, - {file = "cryptography-46.0.7-cp38-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:935ce7e3cfdb53e3536119a542b839bb94ec1ad081013e9ab9b7cfd478b05006"}, - {file = "cryptography-46.0.7-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:35719dc79d4730d30f1c2b6474bd6acda36ae2dfae1e3c16f2051f215df33ce0"}, - {file = "cryptography-46.0.7-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:7bbc6ccf49d05ac8f7d7b5e2e2c33830d4fe2061def88210a126d130d7f71a85"}, - {file = "cryptography-46.0.7-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a1529d614f44b863a7b480c6d000fe93b59acee9c82ffa027cfadc77521a9f5e"}, - {file = "cryptography-46.0.7-cp38-abi3-win32.whl", hash = "sha256:f247c8c1a1fb45e12586afbb436ef21ff1e80670b2861a90353d9b025583d246"}, - {file = "cryptography-46.0.7-cp38-abi3-win_amd64.whl", hash = "sha256:506c4ff91eff4f82bdac7633318a526b1d1309fc07ca76a3ad182cb5b686d6d3"}, - {file = "cryptography-46.0.7-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:fc9ab8856ae6cf7c9358430e49b368f3108f050031442eaeb6b9d87e4dcf4e4f"}, - {file = "cryptography-46.0.7-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:d3b99c535a9de0adced13d159c5a9cf65c325601aa30f4be08afd680643e9c15"}, - {file = "cryptography-46.0.7-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:d02c738dacda7dc2a74d1b2b3177042009d5cab7c7079db74afc19e56ca1b455"}, - {file = "cryptography-46.0.7-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:04959522f938493042d595a736e7dbdff6eb6cc2339c11465b3ff89343b65f65"}, - {file = "cryptography-46.0.7-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:3986ac1dee6def53797289999eabe84798ad7817f3e97779b5061a95b0ee4968"}, - {file = "cryptography-46.0.7-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:258514877e15963bd43b558917bc9f54cf7cf866c38aa576ebf47a77ddbc43a4"}, - {file = "cryptography-46.0.7.tar.gz", hash = "sha256:e4cfd68c5f3e0bfdad0d38e023239b96a2fe84146481852dffbcca442c245aa5"}, + {file = "cryptography-50.0.2-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:fa8f5efb344d6908a1ce62f4a24e2e5780f825d6f53f5f50ec5ffacac72936cb"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:79def8d059362e7831389ed3be0ecdf58a89386e1271e35dd9f5af84e81bffd0"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:630ebfea3bf689d075f82316324ff7433dc447fe6bc1bfc76524b74b4a9567d2"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:f9f6143a8c75945eb960d9eb98905a441394abfa24afaae239d514ffb2586480"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:a582ab2ae1d34f67112cadc86702774c9ea4374df6bca6afe672817203c99134"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:4061c0079120205fb760c58acab6443e217307dcf05e3702cf970e0689972856"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:ac9ed99d81760c62fe89d5f0815cdfa1ba9a35141cf30f1c2d044f04b4803d2e"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:87e9ce85beb6b328ba370cc6e6aea483c92617b4c95b1d33a49297eb662bfb04"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:f265528741e048bce55c3463ed721fb0aa45a5888d8add8cfeccb3035451bbdc"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:9dab55f57c74c3cad24c323bacbbd04be4705ba6eb0d92e920b1fc4837ed5079"}, + {file = "cryptography-50.0.2-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:25784ce8b9621c90c643efb9e1e2162ab3b0224cae446ad5e70e7fcb1ce18b51"}, + {file = "cryptography-50.0.2-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:85d0d9a31b9098e98534226d5686b47264b95e62ce459dc2e62fdfc809f9fe93"}, + {file = "cryptography-50.0.2-cp311-abi3-win_amd64.whl", hash = "sha256:7afa5a6602a9f29af1f3a2965f831bae7c9d5d597b7cbb716d41ab3b7d89879c"}, + {file = "cryptography-50.0.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f785f6161f202ab04d8ca194158968798e480ca058943907972da5f12e2881e8"}, + {file = "cryptography-50.0.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0ecbc5652bdb6fc9eaf89a7d196e20941adfe812f43bc4ca05d9150496821047"}, + {file = "cryptography-50.0.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ab50ee449bf968271e820086f10a33d101dd060370abc10bcd22279be2656539"}, + {file = "cryptography-50.0.2-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:a9f7355e6fab51f6c369b86fb7571cffa05edee2c2121e0380a37fb9ac1cd5c1"}, + {file = "cryptography-50.0.2-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:94e5e9f108ee10471288214d3d233fbfbb492840a8457eb85178d643ddeb32c7"}, + {file = "cryptography-50.0.2-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:241449bf940a5d27309bd317e6f9a2af6932113818bb2b8f5c59ddc7ef16da18"}, + {file = "cryptography-50.0.2-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:d8947001be83df1394050758ce0e745dd74fb134eef0a4b5124208dfc3a68c37"}, + {file = "cryptography-50.0.2-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:4a20ce1e5cb4284a86692fdcba7cb8754185c6b2e5c56fcef3751cf451d3cdc2"}, + {file = "cryptography-50.0.2-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:84f964e537f916e2cc85199e5a88742e964939b575ac8598b3f9d6cc416cdaf1"}, + {file = "cryptography-50.0.2-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:828d49b0ff5a0e3975865571c5d91dbbdd0d38d8289b249a163e9425413a5e05"}, + {file = "cryptography-50.0.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:deb9fde5c60e437ee4821bc9bc39ff31b42135c27e1dc61ef0a629389c1de62e"}, + {file = "cryptography-50.0.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:8c71ba2cd31fc93748c38e1b613200ff1c2665cbfd5341fe3a61cfde35a1430e"}, + {file = "cryptography-50.0.2-cp314-cp314t-win_amd64.whl", hash = "sha256:78198641e5be9521beea5aa782bb551a58068d10e6eb04c9c680c1b69f2e7d45"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-macosx_11_0_arm64.whl", hash = "sha256:edc3342adf8f697fc5f59c887a304356f147b397809440ed64e2fa6af2f50f37"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d370b8d1dfcdf7130178137f6fbee6140774a1acc6cacefc4b42643ec11d0a3a"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f2f9bd7f90c64fe89253f0a2c05e3c4856072660429ce8831b4235bf29403a67"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_28_aarch64.whl", hash = "sha256:e275096ea1e60cc595cda2836fd4a6c725d1125108b868be17f53684d164e2cc"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_28_ppc64le.whl", hash = "sha256:b13478603dcd0a2479ff8e87e2c19a7d525734686fe3c49542472293a204212d"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_28_x86_64.whl", hash = "sha256:58a0c478eeca76fe5e07993c5a0703def34a6dc6a0cda4f5564639b33112ffe7"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_31_armv7l.whl", hash = "sha256:d38cdff612d06fa6a32840d5e1b1f7a27cee4a349aa9085d94a67789d6bfd408"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_34_aarch64.whl", hash = "sha256:fdd28f912fccfec1846a94e2e1e8f9b0012f557f0c46fe4f3eb0d7a87afcf90b"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_34_ppc64le.whl", hash = "sha256:cbc8738fd8526d80f35cb3a40d41f41a2e7030bb3b18b09a6778ef63d291c2fd"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_34_x86_64.whl", hash = "sha256:e105ab60406787da31fccc883fc0f733af1efd78f0136a4599692c4083a73d0c"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-musllinux_1_2_aarch64.whl", hash = "sha256:6f8700550aa1474a91e5dc07049c46f98b423b5b1ddd0483e0b51362eeeaf5be"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-musllinux_1_2_x86_64.whl", hash = "sha256:c71be1cbfa5cd9a41ee452acf1eccd82b2c05950358b106ec8ceb83411d1a020"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-win_amd64.whl", hash = "sha256:c423ab384a46c4dff7217b2ea5ba2e11cffdeab6441acd04cf65a369caf0366c"}, + {file = "cryptography-50.0.2-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:0ec5f09541743261e66e291b4a0cbf0fb2997aeaab6d9e9c740b9dba1b58d1c2"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c5e67125c7dca78d199ec4e116aa93dbb83494808ecbb8211a2cb09b1bf41dbd"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ee247f5c245c9a2fe7c8e2214e295918838e44e00a45a6718451e4004219e767"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:dfe9763530994147d9af1def057a5b9658b00e8f8fe8743d144d1e0911c2e454"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:58ddb5a8e3179d12f19e4ea34d2d32e9d63a4baa142c875c1eb59f41b7243acd"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f21e8a22c8605750c7af886bab299a363721264061b4ac0a30efb73cfd58efc5"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:9c8402a82ea0dc4ceeab793db05f0fafa8ca139ca34fcde5df0f596103c74107"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:0ddc924c04591c2811ca024d62ecad4f7f6f08af8939c211438f48a16bd23602"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:a6557e5f38e065ca9fbdaf7cfc7435ecb1d113aa81a022d1b51921ee7432e227"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:1981f1db4630889b9ef7803fadef12b056f428cb6b85c27ba57b774793b6093c"}, + {file = "cryptography-50.0.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:7a8701d6b584d76e909e3d305b7d126b41439876a5aaf76cddc67fc230eafa2e"}, + {file = "cryptography-50.0.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:ce47f66801c20ec6c6632453bb5960fe38939e9306970b48b3a5a26de7745d94"}, + {file = "cryptography-50.0.2-cp39-abi3-win_amd64.whl", hash = "sha256:4e81d95e5bafc2d6e34e4bed780e53e4d5b9a2f928573428aa4d35fbec1eb0de"}, + {file = "cryptography-50.0.2-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:92e665960f25fcdc73725b9cec7a3824f279ba97a98653afe9ffac2e43668f67"}, + {file = "cryptography-50.0.2-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:eef4c2f3423810b3070ab391f85436d2f8bbfcb286ac15cbc73190b3563b1f1a"}, + {file = "cryptography-50.0.2-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:7c6d0330c472d96f6a6afe24d80dfdf15176c33096f0a4397ae4c60f3dd3be48"}, + {file = "cryptography-50.0.2-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:1ba34f04897fcdaa73f74145c25f3ec146fbd56593853e88adc2e811303c5f42"}, + {file = "cryptography-50.0.2-pp311-pypy311_pp80-macosx_11_0_arm64.whl", hash = "sha256:3dc4fd8058cea1644971207d530e1a03a184a805ffc8ebdddf0599d78a331b81"}, + {file = "cryptography-50.0.2-pp311-pypy311_pp80-win_amd64.whl", hash = "sha256:7b75de3c8b3be1cdb1052747c929440c3eea46c1bc2cb8a6e3a48388e9b7b452"}, + {file = "cryptography-50.0.2.tar.gz", hash = "sha256:7b46165bb56eb4704e2eaaf86f3c940d19154535d9b0ca7d6d590b04060e00d5"}, ] [package.dependencies] -cffi = {version = ">=2.0.0", markers = "python_full_version >= \"3.9.0\" and platform_python_implementation != \"PyPy\""} +cffi = {version = ">=2.0.0", markers = "platform_python_implementation != \"PyPy\""} [package.extras] -docs = ["sphinx (>=5.3.0)", "sphinx-inline-tabs", "sphinx-rtd-theme (>=3.0.0)"] -docstest = ["pyenchant (>=3)", "readme-renderer (>=30.0)", "sphinxcontrib-spelling (>=7.3.1)"] -nox = ["nox[uv] (>=2024.4.15)"] -pep8test = ["check-sdist", "click (>=8.0.1)", "mypy (>=1.14)", "ruff (>=0.11.11)"] -sdist = ["build (>=1.0.0)"] ssh = ["bcrypt (>=3.1.5)"] -test = ["certifi (>=2024)", "cryptography-vectors (==46.0.7)", "pretend (>=0.7)", "pytest (>=7.4.0)", "pytest-benchmark (>=4.0)", "pytest-cov (>=2.10.1)", "pytest-xdist (>=3.5.0)"] -test-randomorder = ["pytest-randomly"] [[package]] name = "decorator" @@ -1579,27 +1582,30 @@ probabilistic = ["pyprobables (>=0.6,<0.7)"] [[package]] name = "fastapi" -version = "0.121.3" +version = "0.142.2" description = "FastAPI framework, high performance, easy to learn, fast to code, ready for production" optional = true -python-versions = ">=3.8" +python-versions = ">=3.10" groups = ["main"] markers = "extra == \"server\"" files = [ - {file = "fastapi-0.121.3-py3-none-any.whl", hash = "sha256:0c78fc87587fcd910ca1bbf5bc8ba37b80e119b388a7206b39f0ecc95ebf53e9"}, - {file = "fastapi-0.121.3.tar.gz", hash = "sha256:0055bc24fe53e56a40e9e0ad1ae2baa81622c406e548e501e717634e2dfbc40b"}, + {file = "fastapi-0.142.2-py3-none-any.whl", hash = "sha256:bd5f4d81f1e93a88bcd77caf4dfe3c2dbffc3805407a0007e9a114c18b3a670b"}, + {file = "fastapi-0.142.2.tar.gz", hash = "sha256:06366626f2e70576367714d9ab2fe8472e6c8456dba69b399f9f797ab5e92570"}, ] [package.dependencies] annotated-doc = ">=0.0.2" -pydantic = ">=1.7.4,<1.8 || >1.8,<1.8.1 || >1.8.1,<2.0.0 || >2.0.0,<2.0.1 || >2.0.1,<2.1.0 || >2.1.0,<3.0.0" -starlette = ">=0.40.0,<0.51.0" +opentelemetry-api = ">=1.44.0" +pydantic = ">=2.9.0" +starlette = ">=0.46.0" typing-extensions = ">=4.8.0" +typing-inspection = ">=0.4.2" [package.extras] -all = ["email-validator (>=2.0.0)", "fastapi-cli[standard] (>=0.0.8)", "httpx (>=0.23.0,<1.0.0)", "itsdangerous (>=1.1.0)", "jinja2 (>=3.1.5)", "orjson (>=3.2.1)", "pydantic-extra-types (>=2.0.0)", "pydantic-settings (>=2.0.0)", "python-multipart (>=0.0.18)", "pyyaml (>=5.3.1)", "ujson (>=4.0.1,!=4.0.2,!=4.1.0,!=4.2.0,!=4.3.0,!=5.0.0,!=5.1.0)", "uvicorn[standard] (>=0.12.0)"] -standard = ["email-validator (>=2.0.0)", "fastapi-cli[standard] (>=0.0.8)", "httpx (>=0.23.0,<1.0.0)", "jinja2 (>=3.1.5)", "python-multipart (>=0.0.18)", "uvicorn[standard] (>=0.12.0)"] -standard-no-fastapi-cloud-cli = ["email-validator (>=2.0.0)", "fastapi-cli[standard-no-fastapi-cloud-cli] (>=0.0.8)", "httpx (>=0.23.0,<1.0.0)", "jinja2 (>=3.1.5)", "python-multipart (>=0.0.18)", "uvicorn[standard] (>=0.12.0)"] +all = ["email-validator (>=2.0.0)", "fastapi-cli[standard] (>=0.0.32)", "httpx (>=0.23.0,<1.0.0)", "itsdangerous (>=1.1.0)", "jinja2 (>=3.1.5)", "opentelemetry-exporter-otlp-proto-http (>=1.44.0)", "opentelemetry-sdk (>=1.44.0)", "pydantic-extra-types (>=2.0.0)", "pydantic-settings (>=2.0.0)", "python-multipart (>=0.0.18)", "pyyaml (>=5.3.1)", "uvicorn[standard] (>=0.12.0)"] +opentelemetry = ["opentelemetry-exporter-otlp-proto-http (>=1.44.0)", "opentelemetry-sdk (>=1.44.0)"] +standard = ["email-validator (>=2.0.0)", "fastapi-cli[standard] (>=0.0.32)", "fastar (>=0.9.0)", "httpx (>=0.23.0,<1.0.0)", "jinja2 (>=3.1.5)", "opentelemetry-exporter-otlp-proto-http (>=1.44.0)", "opentelemetry-sdk (>=1.44.0)", "pydantic-extra-types (>=2.0.0)", "pydantic-settings (>=2.0.0)", "python-multipart (>=0.0.18)", "uvicorn[standard] (>=0.12.0)"] +standard-no-fastapi-cloud-cli = ["email-validator (>=2.0.0)", "fastapi-cli[standard-no-fastapi-cloud-cli] (>=0.0.32)", "httpx (>=0.23.0,<1.0.0)", "jinja2 (>=3.1.5)", "opentelemetry-exporter-otlp-proto-http (>=1.44.0)", "opentelemetry-sdk (>=1.44.0)", "pydantic-extra-types (>=2.0.0)", "pydantic-settings (>=2.0.0)", "python-multipart (>=0.0.18)", "uvicorn[standard] (>=0.12.0)"] [[package]] name = "filelock" @@ -2977,6 +2983,22 @@ files = [ [package.dependencies] et-xmlfile = "*" +[[package]] +name = "opentelemetry-api" +version = "1.45.0" +description = "OpenTelemetry Python API" +optional = true +python-versions = ">=3.10" +groups = ["main"] +markers = "extra == \"server\"" +files = [ + {file = "opentelemetry_api-1.45.0-py3-none-any.whl", hash = "sha256:80e068aba7cd56c8b58512d6a36f8d25cb1dfaa0c0a4cc1c938ccf9f362d9cb3"}, + {file = "opentelemetry_api-1.45.0.tar.gz", hash = "sha256:711ede81773c8025c2c03dac0450bc89f3d30aea6eabcc815c570d4e35a963f7"}, +] + +[package.dependencies] +typing-extensions = ">=4.5.0" + [[package]] name = "orcid" version = "1.0.3" @@ -3426,7 +3448,7 @@ description = "C parser in Python" optional = true python-versions = ">=3.10" groups = ["main"] -markers = "platform_python_implementation != \"PyPy\" and extra == \"server\" and implementation_name != \"PyPy\"" +markers = "platform_python_implementation != \"PyPy\" and implementation_name != \"PyPy\" and extra == \"server\"" files = [ {file = "pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992"}, {file = "pycparser-3.0.tar.gz", hash = "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29"}, @@ -3777,23 +3799,18 @@ six = ">=1.5" [[package]] name = "python-discovery" -version = "1.3.0" +version = "1.6.1" description = "Python interpreter discovery" optional = false python-versions = ">=3.8" groups = ["dev"] files = [ - {file = "python_discovery-1.3.0-py3-none-any.whl", hash = "sha256:441d9ced3dfce36e113beb35ca302c71c7ef06f3c0f9c227a0b9bb3bd49b9e9f"}, - {file = "python_discovery-1.3.0.tar.gz", hash = "sha256:d098f1e86be5d45fe4d14bf1029294aabbd332f4321179dec85e76cddce834b0"}, + {file = "python_discovery-1.6.1-py3-none-any.whl", hash = "sha256:d43fcdef879fe795352bd13ccf8d185ba5a9f86f36cfcd00529f596e737442b3"}, + {file = "python_discovery-1.6.1.tar.gz", hash = "sha256:cf87d3627dfb4412437fdd5b13eae402607722998d21567993aedbc59b23c15e"}, ] [package.dependencies] filelock = ">=3.15.4" -platformdirs = ">=4.3.6,<5" - -[package.extras] -docs = ["furo (>=2025.12.19)", "sphinx (>=9.1)", "sphinx-autodoc-typehints (>=3.6.3)", "sphinxcontrib-mermaid (>=2)"] -testing = ["covdefaults (>=2.3)", "coverage (>=7.5.4)", "pytest (>=8.3.5)", "pytest-mock (>=3.14)", "setuptools (>=75.1)"] [[package]] name = "python-dotenv" @@ -4522,20 +4539,20 @@ sqlcipher = ["sqlcipher3_binary"] [[package]] name = "sqlparse" -version = "0.5.5" +version = "0.6.0" description = "A non-validating SQL parser." optional = true -python-versions = ">=3.8" +python-versions = ">=3.10" groups = ["main"] markers = "extra == \"server\"" files = [ - {file = "sqlparse-0.5.5-py3-none-any.whl", hash = "sha256:12a08b3bf3eec877c519589833aed092e2444e68240a3577e8e26148acc7b1ba"}, - {file = "sqlparse-0.5.5.tar.gz", hash = "sha256:e20d4a9b0b8585fdf63b10d30066c7c94c5d7a7ec47c889a2d83a3caa93ff28e"}, + {file = "sqlparse-0.6.0-py3-none-any.whl", hash = "sha256:b861c0288ce2fa56209a9a6412d2e066ac664b3873b89c26c9d8415e8e32996f"}, + {file = "sqlparse-0.6.0.tar.gz", hash = "sha256:113c35c75365ab9cc9c7231d68c6428fb11c085fc8e9eb1ad659b7ddbf6cd2b9"}, ] [package.extras] dev = ["build"] -doc = ["sphinx"] +doc = ["furo", "sphinx"] [[package]] name = "stack-data" @@ -4560,23 +4577,23 @@ tests = ["cython", "littleutils", "pygments", "pytest", "typeguard"] [[package]] name = "starlette" -version = "0.49.3" +version = "1.7.0" description = "The little ASGI library that shines." optional = true -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] markers = "extra == \"server\"" files = [ - {file = "starlette-0.49.3-py3-none-any.whl", hash = "sha256:b579b99715fdc2980cf88c8ec96d3bf1ce16f5a8051a7c2b84ef9b1cdecaea2f"}, - {file = "starlette-0.49.3.tar.gz", hash = "sha256:1c14546f299b5901a1ea0e34410575bc33bbd741377a10484a54445588d00284"}, + {file = "starlette-1.7.0-py3-none-any.whl", hash = "sha256:67f8e99895493dd2911a03f11314af6ceebeae4e704bb9f43dfc6a9db151c93e"}, + {file = "starlette-1.7.0.tar.gz", hash = "sha256:c79f74ea63cff761804fbbfb182f1e0b440c2d07b164d24700c5a1bab5d6ff5d"}, ] [package.dependencies] -anyio = ">=3.6.2,<5" +anyio = ">=4.0.0,<5" typing-extensions = {version = ">=4.10.0", markers = "python_version < \"3.13\""} [package.extras] -full = ["httpx (>=0.27.0,<0.29.0)", "itsdangerous", "jinja2", "python-multipart (>=0.0.18)", "pyyaml"] +full = ["httpx (>=0.27.0,<0.29.0)", "httpx2 (>=2.0.0)", "itsdangerous", "jinja2", "opentelemetry-api", "python-multipart (>=0.0.18)", "pyyaml"] [[package]] name = "starlette-context" @@ -4752,11 +4769,41 @@ description = "Backported and Experimental Type Hints for Python 3.9+" optional = false python-versions = ">=3.9" groups = ["main", "dev"] +markers = "python_version >= \"3.15\"" files = [ {file = "typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548"}, {file = "typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466"}, ] +[[package]] +name = "typing-extensions" +version = "4.16.0" +description = "Backported and Experimental Type Hints for Python 3.9+" +optional = false +python-versions = ">=3.9" +groups = ["main", "dev"] +markers = "python_version < \"3.15\"" +files = [ + {file = "typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8"}, + {file = "typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5"}, +] + +[[package]] +name = "typing-inspection" +version = "0.4.4" +description = "Runtime typing introspection tools" +optional = true +python-versions = ">=3.10" +groups = ["main"] +markers = "extra == \"server\"" +files = [ + {file = "typing_inspection-0.4.4-py3-none-any.whl", hash = "sha256:65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147"}, + {file = "typing_inspection-0.4.4.tar.gz", hash = "sha256:547274fa6b0a561ccf549cc9524b999a578e737d015d8709d021f9d0d13bea47"}, +] + +[package.dependencies] +typing-extensions = ">=4.15.0" + [[package]] name = "tzdata" version = "2026.2" @@ -4772,14 +4819,14 @@ markers = {dev = "sys_platform == \"win32\""} [[package]] name = "urllib3" -version = "2.7.0" +version = "2.8.0" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.10" groups = ["main", "dev"] files = [ - {file = "urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897"}, - {file = "urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c"}, + {file = "urllib3-2.8.0-py3-none-any.whl", hash = "sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3"}, + {file = "urllib3-2.8.0.tar.gz", hash = "sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63"}, ] [package.extras] @@ -4940,21 +4987,22 @@ publish = ["build", "twine"] [[package]] name = "virtualenv" -version = "21.3.1" +version = "21.14.1" description = "Virtual Python Environment builder" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["dev"] files = [ - {file = "virtualenv-21.3.1-py3-none-any.whl", hash = "sha256:d1a71cf58f2f9228fff23a1f6ec15d39785c6b32e03658d104974247145edd35"}, - {file = "virtualenv-21.3.1.tar.gz", hash = "sha256:c2305bc1fddeec40699b8370d13f8d431b0701f00ce895061ce493aeded4426b"}, + {file = "virtualenv-21.14.1-py3-none-any.whl", hash = "sha256:6fd04089fc0dc33549e7abdff70fc3b63d4e15799f2dbf3281f80d13b9fce522"}, + {file = "virtualenv-21.14.1.tar.gz", hash = "sha256:719b189804e66678017d9f63bbfc590c44f6b96ab4829513806394f9b375929c"}, ] [package.dependencies] distlib = ">=0.3.7,<1" -filelock = {version = ">=3.24.2,<4", markers = "python_version >= \"3.10\""} +filelock = {version = ">=3.24.2,<5", markers = "python_version >= \"3.10\""} +packaging = ">=23.1" platformdirs = ">=3.9.1,<5" -python-discovery = ">=1.2.2" +python-discovery = ">=1.6" [[package]] name = "watchfiles" @@ -5245,4 +5293,4 @@ server = ["aiocache", "alembic", "alembic-utils", "arq", "authlib", "biocommons" [metadata] lock-version = "2.1" python-versions = "^3.11" -content-hash = "7c51ea285ac86d12da6956251af1c55460ec203441ab8adfe9ccbdd47c545cc6" +content-hash = "bf3730983cb78a69ea69c1ecbb92bfd18d430f8843005480b4b9f19e8b7bb686" diff --git a/pyproject.toml b/pyproject.toml index 8e41e55b8..299a1efe1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,9 +50,9 @@ arq = { version = "~0.25.0", optional = true } authlib = { version = "~1.6.6", optional = true } boto3 = { version = "~1.34.97", optional = true } biocommons = { version = "~0.0.0", optional = true } -cryptography = { version = "~46.0.5", optional = true } +cryptography = { version = "~50.0.2", optional = true } cdot = { version = "~0.2.21", optional = true } -fastapi = { version = "~0.121.0", optional = true } +fastapi = { version = "~0.142.2", optional = true } hgvs = { version = "~1.5.4", optional = true } orcid = { version = "~1.0.3", optional = true } pyathena = { version = "~3.14.1", optional = true } @@ -60,7 +60,7 @@ psycopg2 = { version = "~2.9.3", optional = true } python-jose = { extras = ["cryptography"], version = "~3.5.0", optional = true } python-multipart = { version = "~0.0.22", optional = true } requests = { version = "^2.33", optional = true } -starlette = { version = "~0.49.0", optional = true } +starlette = { version = "^1.7.0", optional = true } starlette-context = { version = "^0.3.6", optional = true } slack-sdk = { version = "~3.21.3", optional = true } uvicorn = { extras = ["standard"], version = "*", optional = true } From 2dbf711fc9eb419d8ddee9930e8ec8ad9cce5a63 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 30 Sep 2026 16:12:02 -0700 Subject: [PATCH 195/252] chore: bump version to 2026.3.0 --- pyproject.toml | 2 +- src/mavedb/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 299a1efe1..ce255b293 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "mavedb" -version = "2026.2.7.4" +version = "2026.3.0" description = "API for MaveDB, the database of Multiplexed Assays of Variant Effect." license = "AGPL-3.0-only" readme = "README.md" diff --git a/src/mavedb/__init__.py b/src/mavedb/__init__.py index b37965a21..4d74f711e 100644 --- a/src/mavedb/__init__.py +++ b/src/mavedb/__init__.py @@ -6,7 +6,7 @@ logger = module_logging.getLogger(__name__) __project__ = "mavedb-api" -__version__ = "2026.2.7.4" +__version__ = "2026.3.0" logger.info(f"MaveDB {__version__}") From 8e8abed7e19bb16a5e14f1fedde96216c1914626 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Thu, 1 Oct 2026 15:57:36 -0700 Subject: [PATCH 196/252] feat(cat-vrs): cross-reference ClinGen, gnomAD and ClinVar records in CategoricalVariant mappings Every CategoricalVariant now carries `mappings` to its members' ClinGen allele IDs, gnomAD variant IDs and ClinVar variation IDs: exactMatch for records of the measured change in either nucleotide frame, relatedMatch for its protein consequence and other encodings. This reaches the variant detail, /variant-details, the dump's vrs.ndjson and the VA-Spec proposition subject. - get_allele_cross_references fetches identifiers only (gnomAD and ClinVar, no VEP), time-scoped by as_of. CAIDs are read off the allele. The variant detail derives them from the annotations it already loads. - categorical_member_links selects members before anything is fetched, so a VA-Spec subject that is a lone allele skips both the fetch and the build. The mode rule moves to CatVrsMode.for_defining_level. - Mappings are sorted by (system, code), so output no longer depends on link order. gnomAD codings carry the release as systemVersion and link to that release's browser dataset. - Registry link builders live with their registries (clingen_allele_url, clinvar_variation_url, gnomad_variant_url), and registry base URLs are constants in each registry's module. - categorical_variant_for_variant is removed; callers build from links they already hold. --- src/mavedb/lib/allele_annotations.py | 71 +++++++ src/mavedb/lib/annotation/context.py | 18 +- src/mavedb/lib/cat_vrs.py | 174 ++++++++++++---- src/mavedb/lib/clingen/allele_registry.py | 9 +- src/mavedb/lib/clingen/constants.py | 7 + src/mavedb/lib/clingen/services.py | 9 +- src/mavedb/lib/clinvar/constants.py | 2 + src/mavedb/lib/clinvar/utils.py | 7 + src/mavedb/lib/gnomad.py | 16 ++ src/mavedb/lib/term_systems.py | 20 +- src/mavedb/lib/variant_detail.py | 8 +- src/mavedb/scripts/export_public_data.py | 4 +- src/mavedb/scripts/resources/README.md | 14 +- tests/lib/annotation/test_context.py | 105 ++++++++++ tests/lib/clingen/test_allele_registry.py | 9 + tests/lib/clinvar/test_utils.py | 6 + tests/lib/test_cat_vrs.py | 243 +++++++++++++++------- tests/lib/test_gnomad.py | 22 ++ tests/lib/test_variant_detail.py | 102 +++++++++ 19 files changed, 713 insertions(+), 133 deletions(-) diff --git a/src/mavedb/lib/allele_annotations.py b/src/mavedb/lib/allele_annotations.py index 3d179f055..1cd1fcf9e 100644 --- a/src/mavedb/lib/allele_annotations.py +++ b/src/mavedb/lib/allele_annotations.py @@ -16,6 +16,8 @@ This is the shared assembler for both ``GET /variants/{urn}`` (all of a variant's alleles) and ``GET /alleles/{digest}`` (a single allele) — pass whichever allele set the caller has in hand. +:func:`get_allele_cross_references` is the identifiers-only counterpart for callers that need only +the gnomAD and ClinVar identifiers (the Cat-VRS ``mappings``), skipping VEP and the payload columns. """ from dataclasses import dataclass, field @@ -64,6 +66,22 @@ class ClinvarAnnotation: db_version: str +@dataclass(frozen=True) +class GnomadReference: + """An allele's gnomAD variant ID and the gnomAD release it was matched in.""" + + db_identifier: str + db_version: str + + +@dataclass +class AlleleCrossReferences: + """An allele's identifiers in external variant registries. Identifiers only; no annotation payload.""" + + gnomad: Optional[GnomadReference] = None + clinvar_variation_ids: list[str] = field(default_factory=list) + + @dataclass class AlleleAnnotations: """The external annotations for a single allele. Every field is sparse — a source with no data @@ -73,6 +91,17 @@ class AlleleAnnotations: gnomad: Optional[GnomadAnnotation] = None clinvar: list[ClinvarAnnotation] = field(default_factory=list) + def cross_references(self) -> AlleleCrossReferences: + """The identifiers these annotations carry, for a caller that already fetched the full map.""" + return AlleleCrossReferences( + gnomad=( + GnomadReference(db_identifier=self.gnomad.db_identifier, db_version=self.gnomad.db_version) + if self.gnomad is not None + else None + ), + clinvar_variation_ids=[c.clinvar_variation_id for c in self.clinvar if c.clinvar_variation_id], + ) + def get_allele_annotations( db: Session, alleles: Sequence[Allele], *, as_of: Optional[datetime] = None @@ -149,3 +178,45 @@ def get_allele_annotations( ) return annotations + + +def get_allele_cross_references( + db: Session, alleles: Sequence[Allele], *, as_of: Optional[datetime] = None +) -> dict[str, AlleleCrossReferences]: + """Assemble the digest-keyed gnomAD and ClinVar identifiers for ``alleles``. + + Keyed and time-scoped like :func:`get_allele_annotations`, but reads only the identifier columns: + two queries, no VEP. An allele with no identifiers still gets an (empty) entry. + """ + digest_by_id = {allele.id: allele.vrs_digest for allele in alleles} + references: dict[str, AlleleCrossReferences] = { + digest: AlleleCrossReferences() for digest in digest_by_id.values() if digest is not None + } + if not digest_by_id: + return references + + allele_ids = list(digest_by_id.keys()) + + gnomad_rows = db.execute( + select(GnomadAlleleLink.allele_id, GnomADVariant.db_identifier, GnomADVariant.db_version) + .join(GnomADVariant, GnomADVariant.id == GnomadAlleleLink.gnomad_variant_id) + .where(GnomadAlleleLink.allele_id.in_(allele_ids)) + .where(GnomadAlleleLink.live_at(as_of)) + ).tuples() + for allele_id, db_identifier, db_version in gnomad_rows: + digest = digest_by_id[allele_id] + if digest is not None: + references[digest].gnomad = GnomadReference(db_identifier=db_identifier, db_version=db_version) + + clinvar_rows = db.execute( + select(ClinvarAlleleLink.allele_id, ClinvarControl.clinvar_variation_id) + .join(ClinvarControl, ClinvarControl.id == ClinvarAlleleLink.clinvar_control_id) + .where(ClinvarAlleleLink.allele_id.in_(allele_ids)) + .where(ClinvarAlleleLink.live_at(as_of)) + ).tuples() + for allele_id, variation_id in clinvar_rows: + digest = digest_by_id[allele_id] + if digest is not None and variation_id is not None: + references[digest].clinvar_variation_ids.append(variation_id) + + return references diff --git a/src/mavedb/lib/annotation/context.py b/src/mavedb/lib/annotation/context.py index bf1e3d13b..5c9255cb7 100644 --- a/src/mavedb/lib/annotation/context.py +++ b/src/mavedb/lib/annotation/context.py @@ -14,8 +14,9 @@ from sqlalchemy import select from sqlalchemy.orm import Session +from mavedb.lib.allele_annotations import get_allele_cross_references from mavedb.lib.alleles import get_live_record_allele_links -from mavedb.lib.cat_vrs import build_categorical_variant +from mavedb.lib.cat_vrs import build_categorical_variant, categorical_member_links from mavedb.lib.vrs import vrs_object_from_mapped_variant from mavedb.models.allele import Allele from mavedb.models.mapping_record import MappingRecord @@ -46,8 +47,9 @@ def variant_annotation_context( """Assemble the annotation context for ``variant`` from the live (or as-of) mapping substrate. Returns ``None`` when the variant is unmapped at ``as_of`` or an authoritative allele that carries - no ``post_mapped`` VRS (nothing to annotate). One record fetch + one allele-link fetch; the Cat-VRS - transit is built from the same links, so the subject costs no redundant query. + no ``post_mapped`` VRS (nothing to annotate). One record fetch and one allele-link fetch; a subject with + projection members adds one gnomAD and one ClinVar identifier fetch, scoped to those members, for its + ``mappings``. The Cat-VRS transit is built from the same links. """ record = db.scalar( select(MappingRecord).where(MappingRecord.variant_id == variant.id).where(MappingRecord.live_at(as_of)) @@ -64,8 +66,14 @@ def variant_annotation_context( # projection member, else the concrete measured variation. The VA subject is deliberately *narrow* # (include_convergent=False): the convergent encodings are dropped, because VA-Spec carries no per-member # provenance to mark them as unmeasured, and StudyResult.focusVariant already pins the concrete measured - # allele. - transit = build_categorical_variant(links, name=variant.urn or "", include_convergent=False) + # allele. A lone measured allele is served bare, so it skips the cross-reference fetch and the build. + member_links = categorical_member_links(links, include_convergent=False) + transit = None + if len(member_links) > 1: + cross_references = get_allele_cross_references(db, [link.allele for link in member_links], as_of=as_of) + transit = build_categorical_variant( + links, name=variant.urn or "", include_convergent=False, cross_references=cross_references + ) if transit is not None and len(transit.categorical_variant.members) > 1: subject_variant: MolecularVariation | CategoricalVariant = transit.categorical_variant else: diff --git a/src/mavedb/lib/cat_vrs.py b/src/mavedb/lib/cat_vrs.py index b551472a2..566184819 100644 --- a/src/mavedb/lib/cat_vrs.py +++ b/src/mavedb/lib/cat_vrs.py @@ -6,20 +6,29 @@ import logging from dataclasses import dataclass -from datetime import datetime from enum import Enum -from typing import Optional +from typing import Mapping, Optional from ga4gh.cat_vrs.models import CategoricalVariant, DefiningAlleleConstraint, MappableConcept, Relation -from ga4gh.core.models import ConceptMapping, iriReference +from ga4gh.core.models import Coding, ConceptMapping, iriReference from ga4gh.core.models import Relation as MappingRelation from ga4gh.vrs.models import Allele as VrsAllele from ga4gh.vrs.models import CisPhasedBlock -from sqlalchemy.orm import Session -from mavedb.lib.alleles import get_live_record_allele_links +from mavedb.lib.allele_annotations import AlleleCrossReferences +from mavedb.lib.clingen.allele_registry import clingen_allele_url +from mavedb.lib.clinvar.utils import clinvar_variation_url +from mavedb.lib.gnomad import gnomad_variant_url from mavedb.lib.logging.context import logging_context -from mavedb.lib.term_systems import GKS_ALLELE_RELATION, MAVEDB_CAT_VRS_RELATION, SEQUENCE_ONTOLOGY, TermSystem +from mavedb.lib.term_systems import ( + CLINGEN_ALLELE_REGISTRY, + CLINVAR_VARIATION, + GKS_ALLELE_RELATION, + GNOMAD, + MAVEDB_CAT_VRS_RELATION, + SEQUENCE_ONTOLOGY, + TermSystem, +) from mavedb.lib.term_systems import coding as term_coding from mavedb.lib.vrs import vrs_object_from_mapped_variant from mavedb.models.allele import Allele @@ -75,6 +84,11 @@ class CatVrsMode(str, Enum): PROJECTION = "projection" # Mode 1 — nt measured; score rides faithfully. REVERSE_TRANSLATION = "reverse_translation" # Mode 2 — protein measured; score is implied. + @classmethod + def for_defining_level(cls, level: Optional[str]) -> "CatVrsMode": + """The mode a categorical variant takes from its defining (measured) allele's level.""" + return cls.REVERSE_TRANSLATION if level == SequenceLevel.protein.value else cls.PROJECTION + @dataclass class CategoricalVariantTransit: @@ -178,6 +192,99 @@ def _relation_concept(relation: CatVrsRelation) -> MappableConcept: ) +def _external_codings(allele: Allele, references: Optional[AlleleCrossReferences]) -> list[Coding]: + """The allele's ClinGen, gnomAD and ClinVar identifiers as resolvable codings. Identifiers only.""" + codings: list[Coding] = [] + if allele.clingen_allele_id: + caid = allele.clingen_allele_id + codings.append(term_coding(CLINGEN_ALLELE_REGISTRY, caid, iri=clingen_allele_url(caid))) + + if references is None: + return codings + + if references.gnomad is not None: + gnomad = references.gnomad + codings.append( + term_coding( + GNOMAD, + gnomad.db_identifier, + iri=gnomad_variant_url(gnomad.db_identifier, gnomad.db_version), + system_version=gnomad.db_version, + ) + ) + + for variation_id in references.clinvar_variation_ids: + codings.append(term_coding(CLINVAR_VARIATION, variation_id, iri=clinvar_variation_url(variation_id))) + + return codings + + +def _external_mappings( + members: list[tuple[Allele, Optional[CatVrsRelation]]], references: Mapping[str, AlleleCrossReferences] +) -> list[ConceptMapping]: + """Cross-reference the members' external records as ``CategoricalVariant.mappings``. + + ``members`` pairs each included allele with its relation to the defining allele, ``None`` marking the + defining allele itself. The defining allele and its coordinate representations are the categorical + variant, so their records are an ``exactMatch``. Every other member is a distinct variant tied to it by + encoding or translation, so its records are a ``relatedMatch``, following the ga4gh/cat-vrs v1 + protein-consequence examples (proteinSequenceConsequence-ex1.json). + + The c↔g twins of one change share a CAID, gnomAD ID and ClinVar variation, and a ClinVar variation + recurs once per live release, so mappings are deduplicated by (system, code) with ``exactMatch`` + taking precedence. Mappings are sorted by (system, code) so the output does not depend on the order + the database returned the links in. + """ + mappings: dict[tuple[str, str], ConceptMapping] = {} + for allele, relation in members: + is_exact = relation is None or relation is CatVrsRelation.COORDINATE_REPRESENTATION_OF + match = MappingRelation.EXACT_MATCH if is_exact else MappingRelation.RELATED_MATCH + + for coding in _external_codings(allele, references.get(allele.vrs_digest) if allele.vrs_digest else None): + key = (coding.system, coding.code.root) + existing = mappings.get(key) + if existing is None or (is_exact and existing.relation is not MappingRelation.EXACT_MATCH): + mappings[key] = ConceptMapping(coding=coding, relation=match) + + return [mappings[key] for key in sorted(mappings)] + + +def categorical_member_links( + links: list[MappingRecordAllele], *, include_convergent: bool = True +) -> list[MappingRecordAllele]: + """The links :func:`build_categorical_variant` draws members from, the defining (authoritative) link first. + + Empty when no link is authoritative. Selection is by level and projection group only; a selected + allele with no ``post_mapped`` is skipped later, at hydration. Callers use this to size or prefetch + for the categorical variant before building it. + """ + defining_link = next((link for link in links if link.is_authoritative), None) + if defining_link is None: + return [] + + # Projection mode, narrow object (include_convergent=False): drop any convergent encodings the + # reverse-translation fan left on the record, keeping only the measured change's precise coordinate + # projection and its protein consequence. + mode = CatVrsMode.for_defining_level(defining_link.allele.level) + members = [defining_link] + for link in links: + if link is defining_link: + continue + + if ( + mode is CatVrsMode.PROJECTION + and not include_convergent + and not _is_precise_projection_member( + link.allele.level, link.projection_group, defining_group=defining_link.projection_group + ) + ): + continue + + members.append(link) + + return members + + def _hydrate_vrs(allele: Allele) -> Optional[VrsAllele | CisPhasedBlock]: """Bare VRS variation (Allele or CisPhasedBlock) from the stored post_mapped JSONB. @@ -194,7 +301,11 @@ def _hydrate_vrs(allele: Allele) -> Optional[VrsAllele | CisPhasedBlock]: def build_categorical_variant( - links: list[MappingRecordAllele], *, name: str, include_convergent: bool = True + links: list[MappingRecordAllele], + *, + name: str, + include_convergent: bool = True, + cross_references: Optional[Mapping[str, AlleleCrossReferences]] = None, ) -> Optional[CategoricalVariantTransit]: """Assemble a Cat-VRS ``CategoricalVariant`` from a variant's live allele links. @@ -217,13 +328,20 @@ def build_categorical_variant( projection and protein consequence stay ``coordinate_representation_of`` / ``translation_of``. - ``False`` (the VA-Spec subject): the encodings are dropped (see :func:`_is_precise_projection_member`) and only the measured change's precise projection and protein consequence remain. + + ``mappings`` cross-references the included members' external records (see :func:`_external_mappings`). + CAIDs come from the alleles themselves; gnomAD and ClinVar identifiers come from ``cross_references``, + the digest-keyed map from :func:`~mavedb.lib.allele_annotations.get_allele_cross_references` fetched at + the same ``as_of`` as ``links``. Without it, only CAIDs are mapped. """ - defining_link = next((link for link in links if link.is_authoritative), None) - if defining_link is None: + member_links = categorical_member_links(links, include_convergent=include_convergent) + if not member_links: return None + defining_link, *other_links = member_links defining_allele = defining_link.allele defining_vrs = _hydrate_vrs(defining_allele) + # The authoritative (measured) allele having no post_mapped breaks an invariant the mapping # job upholds. Returning None here is analogous to "unmapped". if defining_vrs is None: @@ -237,30 +355,15 @@ def build_categorical_variant( return None defining_level = defining_allele.level - mode = CatVrsMode.REVERSE_TRANSLATION if defining_level == SequenceLevel.protein.value else CatVrsMode.PROJECTION + mode = CatVrsMode.for_defining_level(defining_level) members: list[VrsAllele | CisPhasedBlock | iriReference] = [defining_vrs] + mapped_members: list[tuple[Allele, Optional[CatVrsRelation]]] = [(defining_allele, None)] member_relations: dict[str, CatVrsRelation] = {} relations_present: dict[CatVrsRelation, None] = {} # insertion-ordered set of relation kinds - for link in links: - if link is defining_link: - continue - + for link in other_links: allele = link.allele - - # Projection mode, narrow object (include_convergent=False): drop any convergent encodings the - # reverse-translation fan left on the record, keeping only the measured change's precise coordinate - # projection and its protein consequence. - if ( - mode is CatVrsMode.PROJECTION - and not include_convergent - and not _is_precise_projection_member( - allele.level, link.projection_group, defining_group=defining_link.projection_group - ) - ): - continue - member_vrs = _hydrate_vrs(allele) # A live link to an allele with no post_mapped is an unexpected data state. if member_vrs is None: @@ -280,6 +383,8 @@ def build_categorical_variant( defining_group=defining_link.projection_group, member_group=link.projection_group, ) + if relation is not None: + mapped_members.append((allele, relation)) if relation is not None and allele.vrs_digest is not None: member_relations[allele.vrs_digest] = relation relations_present[relation] = None @@ -300,6 +405,7 @@ def build_categorical_variant( relations=[_relation_concept(relation) for relation in relations_present] or None, ) ], + mappings=_external_mappings(mapped_members, cross_references or {}) or None, ) return CategoricalVariantTransit( @@ -307,17 +413,3 @@ def build_categorical_variant( mode=mode, member_relations=member_relations, ) - - -def categorical_variant_for_variant( - db: Session, variant_id: int, *, name: str, as_of: Optional[datetime] = None -) -> Optional[CategoricalVariantTransit]: - """Fetch a variant's live allele links and assemble its Cat-VRS transit object. - - The DB-backed entry point routers use. ``as_of`` is a fetch concern, threaded into the link query - only: membership is what varies over time, while each allele's VRS is immutable and content-addressed, - so :func:`build_categorical_variant` stays pure and time-agnostic over the links it is handed. Returns - ``None`` for an unmapped variant (no authoritative link at ``as_of``). - """ - links = get_live_record_allele_links(db, variant_id, as_of=as_of) - return build_categorical_variant(links, name=name) diff --git a/src/mavedb/lib/clingen/allele_registry.py b/src/mavedb/lib/clingen/allele_registry.py index b28a184d1..62e3bf9bf 100644 --- a/src/mavedb/lib/clingen/allele_registry.py +++ b/src/mavedb/lib/clingen/allele_registry.py @@ -1,16 +1,21 @@ import asyncio import logging from typing import Optional +from urllib.parse import quote import requests from aiocache import cached from mavedb.lib.clingen.cache import CACHE_CLASS, CACHE_CONFIG, CACHE_TTL_SECONDS, clingen_cache_key_builder +from mavedb.lib.clingen.constants import CLINGEN_ALLELE_REGISTRY_API_URL, CLINGEN_ALLELE_REGISTRY_PAGE_URL logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) -CLINGEN_API_URL = "https://reg.genome.network/allele" + +def clingen_allele_url(clingen_allele_id: str) -> str: + """Allele Registry page for a CAID or PAID, in the form the ga4gh/cat-vrs examples use.""" + return f"{CLINGEN_ALLELE_REGISTRY_PAGE_URL}?caid={quote(clingen_allele_id, safe='')}" @cached(ttl=CACHE_TTL_SECONDS, key_builder=clingen_cache_key_builder, cache=CACHE_CLASS, **CACHE_CONFIG) @@ -33,7 +38,7 @@ async def get_clingen_allele_data(clingen_allele_id: str) -> Optional[dict]: (excluding 404, which returns None). """ loop = asyncio.get_running_loop() - response = await loop.run_in_executor(None, requests.get, f"{CLINGEN_API_URL}/{clingen_allele_id}") + response = await loop.run_in_executor(None, requests.get, f"{CLINGEN_ALLELE_REGISTRY_API_URL}/{clingen_allele_id}") if response.status_code == 404: return None diff --git a/src/mavedb/lib/clingen/constants.py b/src/mavedb/lib/clingen/constants.py index 60f3a26cb..cf0f7c56c 100644 --- a/src/mavedb/lib/clingen/constants.py +++ b/src/mavedb/lib/clingen/constants.py @@ -9,6 +9,13 @@ CAR_SUBMISSION_ENDPOINT = os.getenv("CAR_SUBMISSION_ENDPOINT") +CLINGEN_ALLELE_REGISTRY_API_URL = "https://reg.genome.network/allele" +"""ClinGen Allele Registry REST endpoint for looking up an allele by CAID, PAID or HGVS.""" +CLINGEN_ALLELE_REGISTRY_PAGE_URL = "https://reg.clinicalgenome.org/redmine/projects/registry/genboree_registry/by_caid" +"""Allele Registry web page for a CAID or PAID, selected with a ``caid`` query parameter.""" +GENBOREE_AUTH_URL = "https://genboree.org/auth/usr" +"""Genboree account authentication, which issues the JWT for ClinGen submissions.""" + LDH_SUBMISSION_TYPE = "cg-ldh-ld-submission" LDH_ENTITY_NAME = "MaveDBMapping" LDH_ENTITY_ENDPOINT = "maveDb" # for some reason, not the same :/ diff --git a/src/mavedb/lib/clingen/services.py b/src/mavedb/lib/clingen/services.py index c8e80259f..fb491f825 100644 --- a/src/mavedb/lib/clingen/services.py +++ b/src/mavedb/lib/clingen/services.py @@ -8,7 +8,12 @@ import requests from jose import jwt -from mavedb.lib.clingen.constants import CLINGEN_HTTP_TIMEOUT, GENBOREE_ACCOUNT_NAME, GENBOREE_ACCOUNT_PASSWORD +from mavedb.lib.clingen.constants import ( + CLINGEN_HTTP_TIMEOUT, + GENBOREE_ACCOUNT_NAME, + GENBOREE_ACCOUNT_PASSWORD, + GENBOREE_AUTH_URL, +) from mavedb.lib.logging.context import format_raised_exception_info_as_dict, logging_context, save_to_logging_context from mavedb.lib.types.clingen import ClinGenAllele, ClinGenSubmissionError, LdhSubmission from mavedb.lib.utils import batched @@ -164,7 +169,7 @@ def authenticate(self) -> str: extra=logging_context(), ) - auth_url = f"https://genboree.org/auth/usr/gb:{GENBOREE_ACCOUNT_NAME}/auth" + auth_url = f"{GENBOREE_AUTH_URL}/gb:{GENBOREE_ACCOUNT_NAME}/auth" auth_body = {"type": "plain", "val": GENBOREE_ACCOUNT_PASSWORD} try: auth_response = requests.post(auth_url, json=auth_body, timeout=CLINGEN_HTTP_TIMEOUT) diff --git a/src/mavedb/lib/clinvar/constants.py b/src/mavedb/lib/clinvar/constants.py index fd80212be..11f028fc8 100644 --- a/src/mavedb/lib/clinvar/constants.py +++ b/src/mavedb/lib/clinvar/constants.py @@ -4,6 +4,8 @@ from urllib3.util.retry import Retry TSV_VARIANT_ARCHIVE_BASE_URL = "https://ftp.ncbi.nlm.nih.gov/pub/clinvar/tab_delimited/archive" +CLINVAR_VARIATION_URL = "https://www.ncbi.nlm.nih.gov/clinvar/variation" +"""ClinVar web page for a variation, addressed as ``{CLINVAR_VARIATION_URL}/{variation_id}/``.""" NCBI_REQUEST_HEADERS = { "User-Agent": "MaveDB/1.0 (https://mavedb.org)", diff --git a/src/mavedb/lib/clinvar/utils.py b/src/mavedb/lib/clinvar/utils.py index d72968da8..7ac13e5d1 100644 --- a/src/mavedb/lib/clinvar/utils.py +++ b/src/mavedb/lib/clinvar/utils.py @@ -9,6 +9,7 @@ from datetime import datetime from pathlib import Path from typing import Dict +from urllib.parse import quote import requests from filelock import FileLock @@ -17,6 +18,7 @@ from mavedb.lib.clinvar.constants import ( CLINVAR_CACHE_DIR, CLINVAR_FIELDS_TO_KEEP, + CLINVAR_VARIATION_URL, NCBI_REQUEST_HEADERS, NCBI_RETRY_STRATEGY, TSV_VARIANT_ARCHIVE_BASE_URL, @@ -28,6 +30,11 @@ logger = logging.getLogger(__name__) +def clinvar_variation_url(variation_id: str) -> str: + """Link to a ClinVar variation record. Mirrors ``clinvarVariationUrl`` in the UI's ``src/lib/clinvar-controls.ts``.""" + return f"{CLINVAR_VARIATION_URL}/{quote(variation_id, safe='')}/" + + def _ncbi_session() -> requests.Session: session = requests.Session() session.headers.update(NCBI_REQUEST_HEADERS) diff --git a/src/mavedb/lib/gnomad.py b/src/mavedb/lib/gnomad.py index 712792dbd..c536231eb 100644 --- a/src/mavedb/lib/gnomad.py +++ b/src/mavedb/lib/gnomad.py @@ -6,6 +6,7 @@ from datetime import datetime from enum import Enum from typing import Any, Optional, Sequence, Union +from urllib.parse import quote from sqlalchemy import Connection, Row, func, select, text from sqlalchemy.orm import Session @@ -24,12 +25,15 @@ GNOMAD_DB_NAME = "gnomAD" GNOMAD_DATA_VERSION = os.getenv("GNOMAD_DATA_VERSION", "v4.1") +GNOMAD_BROWSER_URL = "https://gnomad.broadinstitute.org" _CAID_LEADING_ZERO_RE = r"^(CA)0+([0-9])" """ Strip leading zeros from a CAID's numeric portion, keeping at least one digit. Kept byte-for-byte in sync with the SQL form used to normalize Allele.clingen_allele_id in link_gnomad_variants_to_alleles. """ +_GNOMAD_BROWSER_DATASETS = {"2": "gnomad_r2_1", "3": "gnomad_r3", "4": "gnomad_r4"} +"""Mapping of gnomAD major release numbers to browser dataset names.""" class GnomadLinkVerdict(str, Enum): @@ -124,6 +128,18 @@ def gnomad_identifier(contig: str, position: Union[str, int], alleles: list[str] return f"{contig}-{position}-{'-'.join(alleles)}" +def gnomad_variant_url(db_identifier: str, db_version: str) -> str: + """Browser link for a gnomAD variant, pinned to the dataset of ``db_version`` (e.g. ``v4.1``). + + An unrecognized ``db_version`` gets no ``dataset`` parameter rather than a guessed one. Mirrors + ``gnomadVariantUrl`` in the UI's ``src/lib/gnomad.ts``. + """ + url = f"{GNOMAD_BROWSER_URL}/variant/{quote(db_identifier, safe='')}" + major = re.match(r"v?(\d+)", db_version) + dataset = _GNOMAD_BROWSER_DATASETS.get(major.group(1)) if major else None + return f"{url}?dataset={dataset}" if dataset else url + + def gnomad_table_name() -> str: """ Generate the gnomAD table name based on the data version. diff --git a/src/mavedb/lib/term_systems.py b/src/mavedb/lib/term_systems.py index b9998433a..6f8028ca3 100644 --- a/src/mavedb/lib/term_systems.py +++ b/src/mavedb/lib/term_systems.py @@ -5,7 +5,7 @@ lookup tables could. """ -from ga4gh.core.models import Coding +from ga4gh.core.models import Coding, iriReference TermSystem = tuple[str, str] @@ -29,8 +29,20 @@ # systems under the same vocabulary owner, not this one. GKS_ALLELE_RELATION: TermSystem = ("ga4gh-gks-term:allele-relation", "ga4gh-gks-term") +# External variant registries. Identifiers consumers match on, so deliberately not built from the registries' +# link constants: a registry moving its web pages must not rename a system in served output. +CLINVAR_VARIATION: TermSystem = ("https://www.ncbi.nlm.nih.gov/clinvar/variation/", "clinvar") +CLINGEN_ALLELE_REGISTRY: TermSystem = ("https://reg.clinicalgenome.org/", "clingen") +GNOMAD: TermSystem = ("https://gnomad.broadinstitute.org", "gnomad") -def coding(term_system: TermSystem, code: str) -> Coding: - """Build a ``Coding`` for `code`, drawn from `term_system`.""" + +def coding(term_system: TermSystem, code: str, *, iri: str | None = None, system_version: str | None = None) -> Coding: + """Build a ``Coding`` for `code`, drawn from `term_system` at `system_version`, optionally resolvable at `iri`.""" uri, prefix = term_system - return Coding(id=f"{prefix}:{code}", code=code, system=uri) + return Coding( + id=f"{prefix}:{code}", + code=code, + system=uri, + systemVersion=system_version, + iris=[iriReference(root=iri)] if iri is not None else None, + ) diff --git a/src/mavedb/lib/variant_detail.py b/src/mavedb/lib/variant_detail.py index 6f6f805db..1454cb9c4 100644 --- a/src/mavedb/lib/variant_detail.py +++ b/src/mavedb/lib/variant_detail.py @@ -22,7 +22,7 @@ from mavedb.lib.allele_annotations import AlleleAnnotations, get_allele_annotations from mavedb.lib.allele_identity import AlleleDerivation, AlleleIdentity from mavedb.lib.alleles import get_live_record_allele_links -from mavedb.lib.cat_vrs import categorical_variant_for_variant, is_convergent_encoding +from mavedb.lib.cat_vrs import build_categorical_variant, is_convergent_encoding from mavedb.lib.score_calibrations import calibration_preference_key from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.models.mapping_record import MappingRecord @@ -194,7 +194,11 @@ def get_variant_detail( # Spec-pure Cat-VRS built on the fly, plus mode + per-member relations. The defining allele is # deliberately absent from `relations`, so it gets relation=None below. - transit = categorical_variant_for_variant(db, variant.id, name=variant.urn or "", as_of=as_of) + transit = build_categorical_variant( + links, + name=variant.urn or "", + cross_references={digest: entry.cross_references() for digest, entry in annotations.items()}, + ) if transit is not None: molecular_representation = transit.categorical_variant.model_dump(mode="json", exclude_none=True) mode: Optional[str] = transit.mode.value diff --git a/src/mavedb/scripts/export_public_data.py b/src/mavedb/scripts/export_public_data.py index c4c868819..b4f7d86ad 100644 --- a/src/mavedb/scripts/export_public_data.py +++ b/src/mavedb/scripts/export_public_data.py @@ -31,6 +31,7 @@ from mavedb.lib.annotation.annotate import variant_highest_level_annotation from mavedb.lib.annotation.context import variant_annotation_context +from mavedb.lib.allele_annotations import get_allele_cross_references from mavedb.lib.alleles import get_live_record_allele_links from mavedb.lib.cat_vrs import build_categorical_variant from mavedb.lib.csv.entries import score_sets_have_current_mappings @@ -239,7 +240,8 @@ def vrs_ndjson(db: Session, score_set: ScoreSet) -> str: links = get_live_record_allele_links(db, variant.id) authoritative = next((link.allele for link in links if link.is_authoritative), None) record = next((link.mapping_record for link in links), None) - transit = build_categorical_variant(links, name=variant.urn or "") + cross_references = get_allele_cross_references(db, [link.allele for link in links]) + transit = build_categorical_variant(links, name=variant.urn or "", cross_references=cross_references) lines.append( json.dumps( diff --git a/src/mavedb/scripts/resources/README.md b/src/mavedb/scripts/resources/README.md index aab9e7d5f..7b8582b50 100644 --- a/src/mavedb/scripts/resources/README.md +++ b/src/mavedb/scripts/resources/README.md @@ -202,7 +202,13 @@ A separate artifact rather than columns in `csv/{urn}.annotations.csv` because t CSV cell can carry an identifier or an HGVS string, but not a VRS object. Join the two on `mavedb.post_mapped_vrs_id`, which is the same identifier as `post_mapped.id`. -`categorical_variant` is spec-pure — it contains no MaveDB-specific fields. A variant the pipeline could +`categorical_variant` is spec-pure — it contains no MaveDB-specific fields. Its `mappings` cross-reference the +members' ClinGen allele IDs, gnomAD variant IDs and ClinVar variation IDs: `exactMatch` for records of the +measured change itself (in any coordinate frame), `relatedMatch` for records of other members, such as the +protein consequence or another nucleotide change encoding the same protein change. A gnomAD mapping names the +gnomAD release it was matched in as `coding.systemVersion`. Mappings are sorted by `coding.system`, then +`coding.code`. They carry identifiers only; frequencies and classifications are in +`csv/{urn}.annotations.csv`. A variant the pipeline could not place carries no reference coordinates and is omitted from this file, so every line carries a non-null `post_mapped` and the line count equals the number of variants with reference coordinates. `pre_mapped` is `null` where no assay-frame VRS was recorded, and `categorical_variant` is `null` where the stored VRS @@ -282,6 +288,12 @@ be combined with other evidence downstream, not as a standalone clinical verdict Research-use-only calibrations are excluded from this file, unlike `csv/{urn}.annotations.csv`, which includes them under a `research_use_only` flag. +The proposition subject is the measured change. When it has another representation (its genomic or +transcript twin, or its protein consequence) the subject is a Cat-VRS `CategoricalVariant` over those +representations, carrying the same `mappings` as `categorical_variant` in `vrs/{urn}.vrs.ndjson`. Unlike +that file, it leaves out other nucleotide changes that encode the same protein change, so it carries none of +their mappings. Otherwise the subject is the bare VRS allele and has no `mappings`. + `annotation` is `null` where no VA-Spec layer could be built for a variant that does have reference coordinates. A variant the pipeline could not place has none, and is omitted from this file entirely. Every variant it currently places produces exactly one line, so the line count equals the number of diff --git a/tests/lib/annotation/test_context.py b/tests/lib/annotation/test_context.py index e75256512..f0a5824bb 100644 --- a/tests/lib/annotation/test_context.py +++ b/tests/lib/annotation/test_context.py @@ -2,6 +2,9 @@ """Tests for mavedb.lib.annotation.context — the VA proposition-subject grain (Slice 5.1).""" +from datetime import datetime, timezone +from unittest import mock + import pytest pytest.importorskip("psycopg2") @@ -9,11 +12,39 @@ from ga4gh.cat_vrs.models import CategoricalVariant from ga4gh.vrs.models import MolecularVariation +from sqlalchemy import select + from mavedb.lib.annotation.context import variant_annotation_context +from mavedb.models.gnomad_allele_link import GnomadAlleleLink +from mavedb.models.gnomad_variant import GnomADVariant from tests.helpers.constants import TEST_VALID_POST_MAPPED_VRS_ALLELE from tests.helpers.util.annotation import AlleleSpec, seed_mapping_record +T0 = datetime(2020, 1, 1, tzinfo=timezone.utc) +T1 = datetime(2021, 1, 1, tzinfo=timezone.utc) + + +def _gnomad_variant(session, db_identifier: str) -> int: + gnomad_variant = GnomADVariant( + db_name="gnomAD", + db_identifier=db_identifier, + db_version="v4.1", + allele_count=1, + allele_number=100, + allele_frequency=0.01, + ) + session.add(gnomad_variant) + session.commit() + return gnomad_variant.id + + +def _gnomad_codes(subject) -> list[str]: + return [ + m.coding.code.root for m in subject.mappings or [] if m.coding.system == "https://gnomad.broadinstitute.org" + ] + + @pytest.mark.integration class TestVariantAnnotationContextSubject: """variant_annotation_context — the VA *proposition* subject grain (Slice 5.1). @@ -106,3 +137,77 @@ def test_unmapped_variant_has_no_context(self, session, setup_lib_db_with_mapped context = variant_annotation_context(session, setup_lib_db_with_mapped_variant.variant) assert context is None + + def test_subject_mappings_cover_only_the_narrow_members_at_as_of(self, session, setup_lib_db_with_mapped_variant): + """The subject cross-references its own members only (not the dropped sibling encoder), as of the + requested instant: a gnomAD match replaced at T1 still maps to the old ID before T1.""" + mapped_variant = setup_lib_db_with_mapped_variant + old_id = _gnomad_variant(session, "1-100-A-G") + new_id = _gnomad_variant(session, "1-100-A-T") + sibling_id = _gnomad_variant(session, "1-101-C-T") + seed_mapping_record( + session, + mapped_variant.variant, + assay_level="cdna", + valid_from=T0, + alleles=[ + AlleleSpec( + digest="cdna", + level="cdna", + is_authoritative=True, + projection_group=0, + post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE, + ), + AlleleSpec( + digest="gen", + level="genomic", + projection_group=0, + post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE, + gnomad_variant_ids=[old_id], + ), + AlleleSpec( + digest="sibling", + level="cdna", + projection_group=1, + post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE, + gnomad_variant_ids=[sibling_id], + ), + ], + ) + old_link = session.scalar(select(GnomadAlleleLink).where(GnomadAlleleLink.gnomad_variant_id == old_id)) + assert old_link is not None + old_link.retire(at=T1) + new_link = GnomadAlleleLink(allele_id=old_link.allele_id, gnomad_variant_id=new_id) + new_link.valid_from = T1 + session.add(new_link) + session.commit() + + past = variant_annotation_context( + session, mapped_variant.variant, as_of=datetime(2020, 6, 1, tzinfo=timezone.utc) + ) + current = variant_annotation_context(session, mapped_variant.variant) + + assert past is not None and isinstance(past.subject_variant, CategoricalVariant) + assert current is not None and isinstance(current.subject_variant, CategoricalVariant) + assert _gnomad_codes(past.subject_variant) == ["1-100-A-G"] + assert _gnomad_codes(current.subject_variant) == ["1-100-A-T"] + + def test_single_allele_subject_skips_the_cross_reference_fetch(self, session, setup_lib_db_with_mapped_variant): + """A lone measured allele is served bare, so nothing would carry the cross-references.""" + mapped_variant = setup_lib_db_with_mapped_variant + seed_mapping_record( + session, + mapped_variant.variant, + assay_level="cdna", + alleles=[ + AlleleSpec( + digest="cdna", level="cdna", is_authoritative=True, post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE + ), + ], + ) + + with mock.patch("mavedb.lib.annotation.context.get_allele_cross_references") as fetch: + context = variant_annotation_context(session, mapped_variant.variant) + + assert context is not None + fetch.assert_not_called() diff --git a/tests/lib/clingen/test_allele_registry.py b/tests/lib/clingen/test_allele_registry.py index 45a04db6d..68cbf4327 100644 --- a/tests/lib/clingen/test_allele_registry.py +++ b/tests/lib/clingen/test_allele_registry.py @@ -9,6 +9,7 @@ import requests from mavedb.lib.clingen.allele_registry import ( + clingen_allele_url, get_associated_clinvar_allele_id, get_canonical_pa_ids, get_clingen_allele_data, @@ -511,3 +512,11 @@ async def test_cache_fully_down_calls_api_every_time(self, mock_request, clear_c assert result1 == "333333" assert result2 == "333333" assert mock_request.call_count == 2 # No caching — both calls hit the API + + +@pytest.mark.unit +@pytest.mark.parametrize("allele_id", ["CA123", "PA456"]) +def test_clingen_allele_url_links_the_registry_page(allele_id): + assert clingen_allele_url(allele_id) == ( + f"https://reg.clinicalgenome.org/redmine/projects/registry/genboree_registry/by_caid?caid={allele_id}" + ) diff --git a/tests/lib/clinvar/test_utils.py b/tests/lib/clinvar/test_utils.py index 082c55586..58b2d29f3 100644 --- a/tests/lib/clinvar/test_utils.py +++ b/tests/lib/clinvar/test_utils.py @@ -8,6 +8,7 @@ from mavedb.lib.clinvar.constants import CLINVAR_FIELDS_TO_KEEP from mavedb.lib.clinvar.utils import ( + clinvar_variation_url, fetch_clinvar_variant_data, validate_clinvar_variant_summary_date, ) @@ -240,3 +241,8 @@ def mock_get(url, **kwargs): assert not stale_file.exists() pkl_files = list(tmp_path.glob("variant_summary_2020-10.parsed.*.pkl")) assert len(pkl_files) == 1 + + +@pytest.mark.unit +def test_clinvar_variation_url_links_the_variation_record(): + assert clinvar_variation_url("376654") == "https://www.ncbi.nlm.nih.gov/clinvar/variation/376654/" diff --git a/tests/lib/test_cat_vrs.py b/tests/lib/test_cat_vrs.py index 92d427d64..ba905cc53 100644 --- a/tests/lib/test_cat_vrs.py +++ b/tests/lib/test_cat_vrs.py @@ -3,12 +3,9 @@ The pure builder is unit-tested over transient ``MappingRecordAllele`` instances (no DB) — asserting the mode, the member->defining relations, and the spec-pure ``CategoricalVariant`` shape for both -score-collapse modes. The DB-backed wrapper ``categorical_variant_for_variant`` is exercised at the -bottom against a real session to pin the fetch + ``as_of`` threading. +score-collapse modes, and the external cross-references carried in ``mappings``. """ -from datetime import datetime, timezone - import pytest pytest.importorskip("psycopg2") @@ -22,13 +19,11 @@ CatVrsMode, CatVrsRelation, build_categorical_variant, - categorical_variant_for_variant, + categorical_member_links, ) +from mavedb.lib.allele_annotations import AlleleCrossReferences, GnomadReference from mavedb.models.allele import Allele -from mavedb.models.mapping_record import MappingRecord from mavedb.models.mapping_record_allele import MappingRecordAllele -from mavedb.models.variant import Variant -from tests.helpers.constants import TEST_MINIMAL_VARIANT # A spec-valid 32-char VRS digest; the internal VRS digest is irrelevant to the builder, which keys # member relations on the `vrs_digest` *column*, so one fixed valid value across members is fine. @@ -61,7 +56,13 @@ def _post_mapped() -> dict: def _link( - *, level: str, digest: str, is_authoritative: bool, post_mapped=_DEFAULT_POST_MAPPED, projection_group=None + *, + level: str, + digest: str, + is_authoritative: bool, + post_mapped=_DEFAULT_POST_MAPPED, + projection_group=None, + caid=None, ) -> MappingRecordAllele: """A transient (record, allele) link with its allele attached — no session needed. @@ -69,10 +70,11 @@ def _link( deliberately distinct from the spec-valid VRS digest embedded in post_mapped. Pass ``post_mapped=None`` to model an un-hydratable allele. ``projection_group`` pairs a c↔g projection (the two links of one precise change share a value; the protein apex carries ``None``); the builder - uses it in projection mode to keep only the measured change's precise coordinate partner. + uses it in projection mode to keep only the measured change's precise coordinate partner. ``caid`` is the + allele's ClinGen id, which the builder cross-references in ``mappings``. """ pm = _post_mapped() if post_mapped is _DEFAULT_POST_MAPPED else post_mapped - allele = Allele(level=level, vrs_digest=digest, post_mapped=pm) + allele = Allele(level=level, vrs_digest=digest, post_mapped=pm, clingen_allele_id=caid) return MappingRecordAllele(is_authoritative=is_authoritative, allele=allele, projection_group=projection_group) @@ -269,91 +271,182 @@ def test_unhydratable_member_allele_is_skipped(): assert len(transit.categorical_variant.members) == 2 -# --- DB-backed wrapper: categorical_variant_for_variant (fetch + build, as_of threaded) --- +# --- External cross-references: CategoricalVariant.mappings --- -T0 = datetime(2020, 1, 1, tzinfo=timezone.utc) -T1 = datetime(2021, 1, 1, tzinfo=timezone.utc) +_CLINGEN = "https://reg.clinicalgenome.org/" +_GNOMAD = "https://gnomad.broadinstitute.org" +_CLINVAR = "https://www.ncbi.nlm.nih.gov/clinvar/variation/" -def _db_allele(session, digest, level): - allele = Allele(vrs_digest=digest, level=level, post_mapped=_post_mapped()) - session.add(allele) - session.commit() - return allele +def _gnomad(db_identifier: str, db_version: str = "v4.1") -> AlleleCrossReferences: + return AlleleCrossReferences(gnomad=GnomadReference(db_identifier=db_identifier, db_version=db_version)) -def _db_variant(session, score_set, suffix): - variant = Variant(**TEST_MINIMAL_VARIANT, urn=f"{score_set.urn}#{suffix}", score_set_id=score_set.id) - session.add(variant) - session.commit() - return variant +def _clinvar(*variation_ids: str) -> AlleleCrossReferences: + """One ClinVar variation ID per live release; a repeated ID models one variation across releases.""" + return AlleleCrossReferences(clinvar_variation_ids=list(variation_ids)) -def _db_record(session, variant, *, assay_level, valid_from=None): - record = MappingRecord( - variant_id=variant.id, - score_set_id=variant.score_set_id, - assay_level=assay_level, - mapping_api_version="test.0.0", - valid_from=valid_from, - ) - session.add(record) - session.commit() - return record - - -def _db_link(session, record, allele, *, is_authoritative=False, valid_from=None): - link = MappingRecordAllele( - mapping_record_id=record.id, - score_set_id=record.score_set_id, - allele_id=allele.id, - is_authoritative=is_authoritative, - valid_from=valid_from, +def _mappings(transit) -> dict[tuple[str, str], MappingRelation]: + """(system, code) -> relation, for the categorical variant's external cross-references.""" + return {(m.coding.system, m.coding.code.root): m.relation for m in transit.categorical_variant.mappings or []} + + +@pytest.mark.unit +def test_projection_mode_maps_the_measured_change_as_exact_and_other_variants_as_related(): + """The measured change and its coordinate twin are the categorical variant (exactMatch); the protein + consequence and convergent encodings are distinct variants (relatedMatch). The twins' shared CAID and + a ClinVar variation repeated across releases each collapse to one mapping.""" + links = [ + _link(level="cdna", digest="cdna", is_authoritative=True, projection_group=0, caid="CA1"), + _link(level="genomic", digest="gen", is_authoritative=False, projection_group=0, caid="CA1"), + _link(level="protein", digest="prot", is_authoritative=False, caid="PA1"), + _link(level="genomic", digest="sibling_gen", is_authoritative=False, projection_group=1, caid="CA2"), + ] + references = { + "cdna": _clinvar("100", "100"), + "gen": _gnomad("1-100-A-G"), + "sibling_gen": _gnomad("1-101-C-T"), + } + + transit = build_categorical_variant(links, name="urn:mavedb:test#map1", cross_references=references) + + assert transit is not None + assert _mappings(transit) == { + (_CLINGEN, "CA1"): MappingRelation.EXACT_MATCH, + (_CLINVAR, "100"): MappingRelation.EXACT_MATCH, + (_GNOMAD, "1-100-A-G"): MappingRelation.EXACT_MATCH, + (_CLINGEN, "PA1"): MappingRelation.RELATED_MATCH, + (_CLINGEN, "CA2"): MappingRelation.RELATED_MATCH, + (_GNOMAD, "1-101-C-T"): MappingRelation.RELATED_MATCH, + } + + +@pytest.mark.unit +def test_narrow_object_omits_mappings_for_dropped_convergent_encodings(): + """Mappings follow the member set: the VA subject drops convergent encodings, so their records go too.""" + links = [ + _link(level="cdna", digest="cdna", is_authoritative=True, projection_group=0), + _link(level="genomic", digest="sibling_gen", is_authoritative=False, projection_group=1), + ] + references = {"cdna": _gnomad("1-100-A-G"), "sibling_gen": _gnomad("1-101-C-T")} + + transit = build_categorical_variant( + links, name="urn:mavedb:test#map2", include_convergent=False, cross_references=references ) - session.add(link) - session.commit() - return link + + assert transit is not None + assert _mappings(transit) == {(_GNOMAD, "1-100-A-G"): MappingRelation.EXACT_MATCH} -@pytest.mark.integration -def test_wrapper_builds_transit_from_live_links(session, setup_lib_db_with_score_set): - """Fetch + build: a protein-measured variant's record yields a Mode 2 transit with nt `encodes`.""" - variant = _db_variant(session, setup_lib_db_with_score_set, 1) - record = _db_record(session, variant, assay_level="protein") - _db_link(session, record, _db_allele(session, "prot", "protein"), is_authoritative=True) - _db_link(session, record, _db_allele(session, "cdna", "cdna")) +@pytest.mark.unit +def test_reverse_translation_maps_each_encoding_as_related(): + """Protein measured: the protein's own CAID is exact; every nt encoding is a distinct variant.""" + links = [ + _link(level="protein", digest="prot", is_authoritative=True, caid="PA1"), + _link(level="cdna", digest="cdna_a", is_authoritative=False, projection_group=0, caid="CA1"), + _link(level="cdna", digest="cdna_b", is_authoritative=False, projection_group=1, caid="CA2"), + ] + references = {"cdna_a": _clinvar("100"), "cdna_b": _clinvar("200")} - transit = categorical_variant_for_variant(session, variant.id, name=variant.urn) + transit = build_categorical_variant(links, name="urn:mavedb:test#map3", cross_references=references) assert transit is not None - assert transit.mode == CatVrsMode.REVERSE_TRANSLATION - assert transit.member_relations == {"cdna": CatVrsRelation.ENCODES} + assert _mappings(transit) == { + (_CLINGEN, "PA1"): MappingRelation.EXACT_MATCH, + (_CLINGEN, "CA1"): MappingRelation.RELATED_MATCH, + (_CLINVAR, "100"): MappingRelation.RELATED_MATCH, + (_CLINGEN, "CA2"): MappingRelation.RELATED_MATCH, + (_CLINVAR, "200"): MappingRelation.RELATED_MATCH, + } -@pytest.mark.integration -def test_wrapper_returns_none_for_unmapped_variant(session, setup_lib_db_with_score_set): - variant = _db_variant(session, setup_lib_db_with_score_set, 1) - assert categorical_variant_for_variant(session, variant.id, name=variant.urn) is None +@pytest.mark.unit +def test_exact_match_wins_when_a_related_member_shares_the_identifier(): + """A record reached first through a related member is upgraded when an exact member also carries it.""" + links = [ + _link(level="cdna", digest="cdna", is_authoritative=True, projection_group=0), + _link(level="protein", digest="prot", is_authoritative=False), + _link(level="genomic", digest="gen", is_authoritative=False, projection_group=0), + ] + references = {"prot": _clinvar("100"), "gen": _clinvar("100")} + + transit = build_categorical_variant(links, name="urn:mavedb:test#map4", cross_references=references) + + assert transit is not None + assert _mappings(transit) == {(_CLINVAR, "100"): MappingRelation.EXACT_MATCH} -@pytest.mark.integration -def test_wrapper_threads_as_of_to_the_historical_record(session, setup_lib_db_with_score_set): - """as_of selects the past record, changing the built object — Mode 2 then, Mode 1 now.""" - variant = _db_variant(session, setup_lib_db_with_score_set, 1) +@pytest.mark.unit +def test_mappings_are_resolvable_and_absent_without_identifiers(): + """Each coding carries an IRI; a variant with no external identifiers emits no ``mappings`` at all.""" + mapped = build_categorical_variant( + [_link(level="genomic", digest="gen", is_authoritative=True)], + name="urn:mavedb:test#map5", + cross_references={"gen": _gnomad("1-100-A-G")}, + ) + unmapped = build_categorical_variant( + [_link(level="genomic", digest="gen", is_authoritative=True)], name="urn:mavedb:test#map6" + ) - old_record = _db_record(session, variant, assay_level="protein", valid_from=T0) - _db_link(session, old_record, _db_allele(session, "old-prot", "protein"), is_authoritative=True, valid_from=T0) - old_record.retire(session, at=T1) - session.commit() + assert mapped is not None and mapped.categorical_variant.mappings is not None + coding = mapped.categorical_variant.mappings[0].coding + assert [iri.root for iri in coding.iris or []] == [ + "https://gnomad.broadinstitute.org/variant/1-100-A-G?dataset=gnomad_r4" + ] + assert unmapped is not None and unmapped.categorical_variant.mappings is None - new_record = _db_record(session, variant, assay_level="genomic", valid_from=T1) - _db_link(session, new_record, _db_allele(session, "new-gen", "genomic"), is_authoritative=True, valid_from=T1) - past = categorical_variant_for_variant(session, variant.id, name=variant.urn, as_of=T0) - current = categorical_variant_for_variant(session, variant.id, name=variant.urn) +@pytest.mark.unit +def test_gnomad_mapping_names_its_release(): + """The gnomAD coding carries the release it was matched in; its link is pinned by ``gnomad_variant_url``.""" + transit = build_categorical_variant( + [_link(level="genomic", digest="gen", is_authoritative=True)], + name="urn:mavedb:test#map7", + cross_references={"gen": _gnomad("1-100-A-G", "v2.1.1")}, + ) - assert past is not None and past.mode == CatVrsMode.REVERSE_TRANSLATION - assert current is not None and current.mode == CatVrsMode.PROJECTION + assert transit is not None and transit.categorical_variant.mappings is not None + coding = transit.categorical_variant.mappings[0].coding + assert coding.systemVersion == "v2.1.1" + assert [iri.root for iri in coding.iris or []] == [ + "https://gnomad.broadinstitute.org/variant/1-100-A-G?dataset=gnomad_r2_1" + ] + + +@pytest.mark.unit +def test_mappings_do_not_depend_on_link_order(): + """The database returns links in no fixed order; the serialized mappings must not follow it.""" + links = [ + _link(level="cdna", digest="cdna", is_authoritative=True, projection_group=0, caid="CA1"), + _link(level="protein", digest="prot", is_authoritative=False, caid="PA1"), + _link(level="genomic", digest="sibling_gen", is_authoritative=False, projection_group=1, caid="CA2"), + ] + references = {"cdna": _clinvar("200", "100"), "sibling_gen": _gnomad("1-101-C-T")} + + forward = build_categorical_variant(links, name="urn:mavedb:test#map8", cross_references=references) + backward = build_categorical_variant( + list(reversed(links)), name="urn:mavedb:test#map8", cross_references=references + ) + + assert forward is not None and backward is not None + assert forward.categorical_variant.mappings == backward.categorical_variant.mappings + assert [(m.coding.system, m.coding.code.root) for m in forward.categorical_variant.mappings or []] == sorted( + _mappings(forward) + ) + + +@pytest.mark.unit +def test_member_links_put_the_defining_link_first_and_follow_the_narrow_selection(): + """The prefetch scope matches the built object: defining first, convergent encodings only when wide.""" + defining = _link(level="cdna", digest="cdna", is_authoritative=True, projection_group=0) + twin = _link(level="genomic", digest="gen", is_authoritative=False, projection_group=0) + sibling = _link(level="genomic", digest="sibling_gen", is_authoritative=False, projection_group=1) + links = [twin, sibling, defining] + + assert categorical_member_links(links) == [defining, twin, sibling] + assert categorical_member_links(links, include_convergent=False) == [defining, twin] + assert categorical_member_links([twin, sibling]) == [] # --------------------------------------------------------------------------- diff --git a/tests/lib/test_gnomad.py b/tests/lib/test_gnomad.py index b3dc62c28..4c9dc9bc0 100644 --- a/tests/lib/test_gnomad.py +++ b/tests/lib/test_gnomad.py @@ -13,6 +13,7 @@ allele_list_from_list_like_string, gnomad_identifier, gnomad_table_name, + gnomad_variant_url, link_gnomad_variants_to_alleles, normalize_caid, ) @@ -381,3 +382,24 @@ def test_returns_empty_map_when_no_alleles_match(session, mocked_gnomad_variant_ assert len(session.scalars(select(GnomadAlleleLink)).all()) == 0 # No gnomAD variant is created when nothing matches the CAID. assert len(session.scalars(select(GnomADVariant)).all()) == 0 + + +@pytest.mark.unit +@pytest.mark.parametrize( + "db_version,expected_url", + [ + ("v4.1", "https://gnomad.broadinstitute.org/variant/1-100-A-G?dataset=gnomad_r4"), + ("v3.1.2", "https://gnomad.broadinstitute.org/variant/1-100-A-G?dataset=gnomad_r3"), + ("v2.1.1", "https://gnomad.broadinstitute.org/variant/1-100-A-G?dataset=gnomad_r2_1"), + ("custom", "https://gnomad.broadinstitute.org/variant/1-100-A-G"), + ], +) +def test_gnomad_variant_url_pins_the_dataset_of_a_known_release(db_version, expected_url): + assert gnomad_variant_url("1-100-A-G", db_version) == expected_url + + +@pytest.mark.unit +def test_gnomad_variant_url_encodes_the_identifier(): + assert gnomad_variant_url("1-100-A/G", "v4.1") == ( + "https://gnomad.broadinstitute.org/variant/1-100-A%2FG?dataset=gnomad_r4" + ) diff --git a/tests/lib/test_variant_detail.py b/tests/lib/test_variant_detail.py index 256594e7e..2c9970c07 100644 --- a/tests/lib/test_variant_detail.py +++ b/tests/lib/test_variant_detail.py @@ -9,6 +9,7 @@ we assert they are wired in and keyed correctly. """ +from datetime import datetime, timezone from types import SimpleNamespace import pytest @@ -18,6 +19,10 @@ from mavedb.lib.variant_detail import get_variant_detail from mavedb.lib.mondo import get_generic_disease_term from mavedb.models.allele import Allele +from mavedb.models.clinical_control import ClinvarControl +from mavedb.models.clinvar_allele_link import ClinvarAlleleLink +from mavedb.models.gnomad_allele_link import GnomadAlleleLink +from mavedb.models.gnomad_variant import GnomADVariant from mavedb.models.mapping_record import MappingRecord from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.score_calibration import ScoreCalibration @@ -293,6 +298,103 @@ def test_pre_reverse_translation_data_degrades_projection_of_to_null(session, se assert detail.alleles["prot-digest"].projection_of is None +T0 = datetime(2020, 1, 1, tzinfo=timezone.utc) +T1 = datetime(2021, 1, 1, tzinfo=timezone.utc) + + +def _gnomad_link(session, allele, gnomad_id, *, valid_from=None): + gnomad_variant = GnomADVariant( + db_name="gnomAD", + db_identifier=gnomad_id, + db_version="v4.1", + allele_count=1, + allele_number=100, + allele_frequency=0.01, + ) + session.add(gnomad_variant) + session.flush() + link = GnomadAlleleLink(allele_id=allele.id, gnomad_variant_id=gnomad_variant.id) + if valid_from is not None: + link.valid_from = valid_from + session.add(link) + session.flush() + return link + + +@pytest.mark.integration +def test_categorical_variant_cross_references_linked_gnomad_and_clinvar_records(session, setup_lib_db_with_score_set): + """The live gnomAD and ClinVar links reach the Cat-VRS as identifier-only ``mappings``: the measured + change's records are an exactMatch, a convergent encoding's a relatedMatch.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, data={"score_data": {"score": -2.3}}, hgvs_nt="c.1216G>A") + record = _record(session, variant, assay_level="cdna") + measured = _allele(session, "cdna-digest", level="cdna", clingen_allele_id="CA123") + genomic = _allele(session, "gen-digest", level="genomic", clingen_allele_id="CA123") + cousin = _allele(session, "cousin-digest", level="cdna") + _link(session, record, measured, is_authoritative=True, projection_group=0) + _link(session, record, genomic, projection_group=0) + _link(session, record, cousin, projection_group=1) + + _gnomad_link(session, genomic, "17-7676154-C-T") + _gnomad_link(session, cousin, "17-7676152-G-A") + control = ClinvarControl( + db_identifier="12345", + gene_symbol="TP53", + clinical_significance="Pathogenic", + clinical_review_status="reviewed by expert panel", + db_name="ClinVar", + db_version="11_2024", + clinvar_variation_id="376654", + ) + session.add(control) + session.flush() + session.add(ClinvarAlleleLink(allele_id=measured.id, clinvar_control_id=control.id)) + session.commit() + + detail = get_variant_detail(session, variant) + + assert detail.molecular_representation is not None + mappings = { + (m["coding"]["system"], m["coding"]["code"]): m["relation"] for m in detail.molecular_representation["mappings"] + } + assert mappings == { + ("https://reg.clinicalgenome.org/", "CA123"): "exactMatch", + ("https://www.ncbi.nlm.nih.gov/clinvar/variation/", "376654"): "exactMatch", + ("https://gnomad.broadinstitute.org", "17-7676154-C-T"): "exactMatch", + ("https://gnomad.broadinstitute.org", "17-7676152-G-A"): "relatedMatch", + } + + +@pytest.mark.integration +def test_categorical_variant_mappings_follow_as_of(session, setup_lib_db_with_score_set): + """``as_of`` reaches the cross-references: a gnomAD match replaced at T1 maps to the old ID at T0.""" + variant = _variant(session, setup_lib_db_with_score_set, 1, data={"score_data": {"score": -2.3}}) + record = _record(session, variant, assay_level="cdna") + measured = _allele(session, "cdna-digest", level="cdna") + genomic = _allele(session, "gen-digest", level="genomic") + for link in ( + _link(session, record, measured, is_authoritative=True, projection_group=0), + _link(session, record, genomic, projection_group=0), + ): + link.valid_from = T0 + record.valid_from = T0 + _gnomad_link(session, genomic, "17-1-A-G", valid_from=T0).retire(at=T1) + _gnomad_link(session, genomic, "17-1-A-T", valid_from=T1) + session.commit() + + def gnomad_ids(as_of): + detail = get_variant_detail(session, variant, as_of=as_of) + assert detail.molecular_representation is not None + return [ + m["coding"]["code"] + for m in detail.molecular_representation["mappings"] + if m["coding"]["system"] == "https://gnomad.broadinstitute.org" + ] + + assert gnomad_ids(datetime(2020, 6, 1, tzinfo=timezone.utc)) == ["17-1-A-G"] + assert gnomad_ids(None) == ["17-1-A-T"] + + @pytest.mark.integration def test_unmapped_variant_has_null_molecular_layer(session, setup_lib_db_with_score_set): """No live record: flat mapped fields and Cat-VRS are null, the annotation map is empty, and the From 8108d170a43e0f14b1cb01deeb872a23268ea5cd Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Fri, 2 Oct 2026 10:45:56 -0700 Subject: [PATCH 197/252] feat(deps)!: upgrade GA4GH models to VA-Spec 1.1.0 and Cat-VRS 1.1.1 Moves ga4gh.vrs to 2.4.0a6, ga4gh.cat-vrs to 0.8.0a6 and ga4gh.va-spec to 0.5.0a6, the versions the GKM Starter Kit targets, and pydantic to 2.12, which va-spec 0.5 requires. VRS digests are unchanged for every allele shape MaveDB stores. BREAKING CHANGE: served VA-Spec and Cat-VRS objects follow the 1.1 specifications. - StudyResult focusVariant is now focus; proposition subjectVariant is now subject, and objectCondition and objectSequenceFeature are now object. - The functional classification system is ga4gh-gkm-term:experimental-var-func-impact-classification. - ACMG evidence lines carry specifiedBy.methodType functional_data_assessment, not the criterion, which VA-Spec 1.1 validates. - Every MappableConcept, including calibration disease, carries type: MappableConcept. - Cat-VRS relation mappings use the spec's own relation concepts. va-spec 0.5 declares nested evidence as base classes, so a plain model_dump() drops a nested study result's focus and score. serialize_annotation dumps with serialize_as_any, and every emission path uses it: the /variants/{urn}/va/* routes, the annotated-variant streams and the public dump. The hand-written ga4gh stubs in mypy_stubs/ hid the breaking API changes, so they are removed and mypy reads the installed packages directly (follow_untyped_imports). That surfaced three typing gaps in vrs_utils, reverse translation and the annotation context, now narrowed. --- mypy_stubs/ga4gh/core/__init__.pyi | 10 - mypy_stubs/ga4gh/core/identifiers.pyi | 10 - mypy_stubs/ga4gh/core/models.pyi | 55 ---- mypy_stubs/ga4gh/va_spec/__init__.pyi | 0 mypy_stubs/ga4gh/va_spec/acmg_2015.pyi | 51 ---- mypy_stubs/ga4gh/va_spec/base/__init__.pyi | 4 - mypy_stubs/ga4gh/va_spec/base/core.pyi | 193 ------------ .../ga4gh/va_spec/base/domain_entities.pyi | 11 - mypy_stubs/ga4gh/va_spec/base/enums.pyi | 12 - mypy_stubs/ga4gh/vrs/__init__.pyi | 4 - mypy_stubs/ga4gh/vrs/dataproxy.pyi | 8 - mypy_stubs/ga4gh/vrs/extras/__init__.pyi | 0 mypy_stubs/ga4gh/vrs/extras/translator.pyi | 19 -- mypy_stubs/ga4gh/vrs/models.pyi | 230 -------------- mypy_stubs/ga4gh/vrs/normalize.pyi | 5 - poetry.lock | 286 +++++++++--------- pyproject.toml | 14 +- src/mavedb/lib/annotation/condition.py | 2 +- src/mavedb/lib/annotation/conformance.py | 24 +- src/mavedb/lib/annotation/context.py | 4 +- src/mavedb/lib/annotation/evidence_line.py | 13 +- src/mavedb/lib/annotation/method.py | 9 +- src/mavedb/lib/annotation/proposition.py | 8 +- src/mavedb/lib/annotation/statement.py | 2 +- src/mavedb/lib/annotation/study_result.py | 4 +- src/mavedb/lib/cat_vrs.py | 30 +- src/mavedb/lib/term_systems.py | 10 - src/mavedb/lib/vrs_utils.py | 5 +- src/mavedb/routers/score_sets.py | 3 +- src/mavedb/routers/variants.py | 34 ++- src/mavedb/scripts/export_public_data.py | 3 +- .../variant_processing/reverse_translation.py | 2 + tests/helpers/constants.py | 1 + tests/lib/annotation/test_annotate.py | 4 +- tests/lib/annotation/test_condition.py | 12 +- tests/lib/annotation/test_method.py | 9 +- tests/lib/annotation/test_proposition.py | 22 +- tests/lib/annotation/test_statement.py | 2 +- tests/lib/annotation/test_study_result.py | 4 +- tests/lib/test_cat_vrs.py | 3 +- tests/routers/test_variant_annotation.py | 4 + 41 files changed, 271 insertions(+), 855 deletions(-) delete mode 100644 mypy_stubs/ga4gh/core/__init__.pyi delete mode 100644 mypy_stubs/ga4gh/core/identifiers.pyi delete mode 100644 mypy_stubs/ga4gh/core/models.pyi delete mode 100644 mypy_stubs/ga4gh/va_spec/__init__.pyi delete mode 100644 mypy_stubs/ga4gh/va_spec/acmg_2015.pyi delete mode 100644 mypy_stubs/ga4gh/va_spec/base/__init__.pyi delete mode 100644 mypy_stubs/ga4gh/va_spec/base/core.pyi delete mode 100644 mypy_stubs/ga4gh/va_spec/base/domain_entities.pyi delete mode 100644 mypy_stubs/ga4gh/va_spec/base/enums.pyi delete mode 100644 mypy_stubs/ga4gh/vrs/__init__.pyi delete mode 100644 mypy_stubs/ga4gh/vrs/dataproxy.pyi delete mode 100644 mypy_stubs/ga4gh/vrs/extras/__init__.pyi delete mode 100644 mypy_stubs/ga4gh/vrs/extras/translator.pyi delete mode 100644 mypy_stubs/ga4gh/vrs/models.pyi delete mode 100644 mypy_stubs/ga4gh/vrs/normalize.pyi diff --git a/mypy_stubs/ga4gh/core/__init__.pyi b/mypy_stubs/ga4gh/core/__init__.pyi deleted file mode 100644 index 8044c175f..000000000 --- a/mypy_stubs/ga4gh/core/__init__.pyi +++ /dev/null @@ -1,10 +0,0 @@ -from . import models as models -from .identifiers import PrevVrsVersion as PrevVrsVersion - -def ga4gh_identify( - vro: object, - in_place: str = ..., - as_version: PrevVrsVersion | None = ..., -) -> str | None: ... - -__all__ = ["PrevVrsVersion", "ga4gh_identify", "models"] diff --git a/mypy_stubs/ga4gh/core/identifiers.pyi b/mypy_stubs/ga4gh/core/identifiers.pyi deleted file mode 100644 index 04fddede0..000000000 --- a/mypy_stubs/ga4gh/core/identifiers.pyi +++ /dev/null @@ -1,10 +0,0 @@ -from enum import Enum -from re import Pattern - -GA4GH_IR_REGEXP: Pattern[str] - -class PrevVrsVersion(str, Enum): - V1_3 = "1.3" - - @classmethod - def validate(cls, version) -> None: ... diff --git a/mypy_stubs/ga4gh/core/models.pyi b/mypy_stubs/ga4gh/core/models.pyi deleted file mode 100644 index 2acc13a5e..000000000 --- a/mypy_stubs/ga4gh/core/models.pyi +++ /dev/null @@ -1,55 +0,0 @@ -from enum import Enum - -from pydantic import BaseModel, RootModel -from typing import Annotated, Any - -class Relation(str, Enum): - CLOSE_MATCH = "closeMatch" - EXACT_MATCH = "exactMatch" - BROAD_MATCH = "broadMatch" - NARROW_MATCH = "narrowMatch" - RELATED_MATCH = "relatedMatch" - -class Element(BaseModel): - id: str | None = None - extensions: list[Extension] | None = None - -class Code(RootModel): - root: Annotated[str, None] - -class Coding(BaseModel): - label: str | None = None - system: str - systemVersion: str | None = None - code: Code - -class Extension(Element): - name: str - value: float | str | bool | dict[str, Any] | list[Any] | None = None - description: str | None = None - -class MappableConcept(BaseModel): - name: str | None = None - conceptType: str | None = None - primaryCoding: Coding | None = None - mappings: list[ConceptMapping] | None = None - - def require_name_or_primary_coding(cls, v): ... - -class Entity(BaseModel): - id: str | None = None - type: str - name: str | None = None - description: str | None = None - aliases: list[str] | None = None - extensions: list[Extension] | None = None - -class iriReference(RootModel): # noqa: N801 - def __hash__(self) -> int: ... - def ga4gh_serialize(self) -> str: ... - - root: str - -class ConceptMapping(Element): - coding: Coding - relation: Relation diff --git a/mypy_stubs/ga4gh/va_spec/__init__.pyi b/mypy_stubs/ga4gh/va_spec/__init__.pyi deleted file mode 100644 index e69de29bb..000000000 diff --git a/mypy_stubs/ga4gh/va_spec/acmg_2015.pyi b/mypy_stubs/ga4gh/va_spec/acmg_2015.pyi deleted file mode 100644 index 1e5e8b573..000000000 --- a/mypy_stubs/ga4gh/va_spec/acmg_2015.pyi +++ /dev/null @@ -1,51 +0,0 @@ -from enum import Enum - -from ..core.models import MappableConcept -from .base.core import EvidenceLine, Method, Statement, VariantPathogenicityProposition, iriReference - -class VariantPathogenicityEvidenceLine(EvidenceLine): - targetProposition: VariantPathogenicityProposition | None # type: ignore - strengthOfEvidenceProvided: MappableConcept | None - specifiedBy: Method | iriReference | None - - class Criterion(str, Enum): - """Define ACMG 2015 criterion values""" - - PVS1 = "PVS1" - PS1 = "PS1" - PS2 = "PS2" - PS3 = "PS3" - PS4 = "PS4" - PM1 = "PM1" - PM2 = "PM2" - PM3 = "PM3" - PM4 = "PM4" - PM5 = "PM5" - PM6 = "PM6" - PP1 = "PP1" - PP2 = "PP2" - PP3 = "PP3" - PP4 = "PP4" - PP5 = "PP5" - BA1 = "BA1" - BS1 = "BS1" - BS2 = "BS2" - BS3 = "BS3" - BS4 = "BS4" - BP1 = "BP1" - BP2 = "BP2" - BP3 = "BP3" - BP4 = "BP4" - BP5 = "BP5" - BP6 = "BP6" - BP7 = "BP7" - -class VariantPathogenicityStatement(Statement): - proposition: VariantPathogenicityProposition - -class AcmgClassification(str, Enum): - PATHOGENIC = "pathogenic" - BENIGN = "benign" - LIKELY_PATHOGENIC = "likely pathogenic" - LIKELY_BENIGN = "likely benign" - UNCERTAIN_SIGNIFICANCE = "uncertain significance" diff --git a/mypy_stubs/ga4gh/va_spec/base/__init__.pyi b/mypy_stubs/ga4gh/va_spec/base/__init__.pyi deleted file mode 100644 index 5fcb5c274..000000000 --- a/mypy_stubs/ga4gh/va_spec/base/__init__.pyi +++ /dev/null @@ -1,4 +0,0 @@ -from . import core as core -from . import enums as enums - -__all__ = ["core", "enums"] diff --git a/mypy_stubs/ga4gh/va_spec/base/core.pyi b/mypy_stubs/ga4gh/va_spec/base/core.pyi deleted file mode 100644 index 2fd15c263..000000000 --- a/mypy_stubs/ga4gh/va_spec/base/core.pyi +++ /dev/null @@ -1,193 +0,0 @@ -from _typeshed import Incomplete -from abc import ABC -from datetime import date -from enum import Enum -from pydantic import BaseModel, RootModel, StringConstraints as StringConstraints -from typing import Annotated, Any, Literal, TypeVar - -from ...core.models import Coding, MappableConcept -from ...vrs.models import MolecularVariation - -StatementType = TypeVar("StatementType") -EvidenceLineType = TypeVar("EvidenceLineType") - -class CoreImType(str, Enum): - AGENT = str - CONTRIBUTION = str - DOCUMENT = str - METHOD = str - DATA_SET = str - EVIDENCE_LINE = str - INFORMATION_ENTITY = str - STUDY_GROUP = str - -class Direction(str, Enum): - SUPPORTS = str - NEUTRAL = str - DISPUTES = str - -class Code(RootModel): - root: Annotated[str, None] - -class iriReference(RootModel): - def __hash__(self): ... - def ga4gh_serialize(self): ... - root: str - -class Extension(BaseModel): - name: str - value: float | str | bool | dict[str, Any] | list[Any] | None = None - description: str | None = None - -class Entity(BaseModel, ABC): - id: str | None = None - type: str - label: str | None = None - description: str | None = None - alternativeLabels: list[str] | None = None - extensions: list[Extension] | None = None - -class Agent(Entity): - type: Literal["Agent"] = "Agent" - name: str | None = None - -class ActivityBase(Entity, ABC): - subtype: Coding | None = None - date: str | None = None - specifiedBy: list[Method] | None = None - @classmethod - def date_format(cls, v: str | None) -> str | None: ... - -class Activity(ActivityBase): - performedBy: list[Agent] | None = None - -class Contribution(ActivityBase): - type: Literal["Contribution"] = "Contribution" - contributor: list[Agent] | None = None - activityType: Coding | None = None - -class InformationEntityBase(Entity, ABC): - type: Literal["InformationEntity"] = "InformationEntity" - specifiedBy: Method | iriReference | None = None - contributions: list[Contribution] | None = None - reportedIn: list[Document | iriReference] | None = None - dateAuthored: str | None = None - recordMetadata: RecordMetadata | None = None - -class InformationEntity(InformationEntityBase): - derivedFrom: list[InformationEntity] | None = None - -class Document(InformationEntity): - type: Literal["Document"] = "Document" - subtype: Coding | None = None - title: str | None = None - urls: list[Annotated[str, None]] | None = None - doi: Annotated[str, None] | None = None - pmid: int | None = None - -class Method(InformationEntity): - type: Literal["Method"] = "Method" - subtype: Coding | None = None - license: str | None = None - -class RecordMetadata(BaseModel): - recordIdentifier: str | None = None - recordVersion: str | None = None - derivedFrom: str | None = None - dateRecordCreated: str | None = None - contributions: list[Contribution] | None = None - -class DataSet(Entity): - type: Literal["DataSet"] = "DataSet" - subtype: MappableConcept | None = None - reportedIn: Document | iriReference | None = None - releaseDate: date | None = None - version: str | None = None - license: MappableConcept | None = None - -class EvidenceLine(InformationEntity): - type: Literal["EvidenceLine"] = "EvidenceLine" - targetProposition: Proposition | SubjectVariantProposition | None = None - hasEvidenceItems: list[StudyResult | Statement | EvidenceLine | iriReference] | None = None - directionOfEvidenceProvided: Direction - strengthOfEvidenceProvided: MappableConcept | None = None - scoreOfEvidenceProvided: float | None = None - evidenceOutcome: MappableConcept | None = None - -class Proposition(Entity): - subject: dict - predicate: str - object: dict - -class Statement(InformationEntity): - type: Literal["Statement"] = "Statement" - proposition: ( - ExperimentalVariantFunctionalImpactProposition - | VariantDiagnosticProposition - | VariantOncogenicityProposition - | VariantPathogenicityProposition - | VariantPrognosticProposition - | VariantTherapeuticResponseProposition - ) - direction: Direction - strength: MappableConcept | None = None - score: float | None = None - classification: MappableConcept | None = None - hasEvidenceLines: list[EvidenceLine | iriReference] | None = None - -class StudyResult(RootModel): - root: CohortAlleleFrequencyStudyResult | ExperimentalVariantFunctionalImpactStudyResult - -class _StudyResult(InformationEntity, ABC): - sourceDataSet: DataSet | None = None - ancillaryResults: dict | None = None - qualityMeasures: dict | None = None - -class _SubjectVariantPropositionBase(Entity, ABC): - subjectVariant: MolecularVariation | CategoricalVariant | iriReference - -class SubjectVariantProposition(RootModel): - root: ( - ExperimentalVariantFunctionalImpactProposition - | VariantPathogenicityProposition - | VariantDiagnosticProposition - | VariantPrognosticProposition - | VariantOncogenicityProposition - | VariantTherapeuticResponseProposition - ) - -class ClinicalVariantProposition(_SubjectVariantPropositionBase): - geneContextQualifier: MappableConcept | iriReference | None = None - alleleOriginQualifier: MappableConcept | iriReference | None = None - -class VariantPathogenicityProposition(ClinicalVariantProposition): - type: Literal["VariantPathogenicityProposition"] = "VariantPathogenicityProposition" - targetProposition: VariantPathogenicityProposition | None = None - strengthOfEvidenceProvided: Coding | iriReference | None = None - specifiedBy: Method | iriReference | None = None - hasEvidenceLines: list[EvidenceLine] | None = None - -class ExperimentalVariantFunctionalImpactStudyResult(_StudyResult): - type: Literal["ExperimentalVariantFunctionalImpactStudyResult"] = "ExperimentalVariantFunctionalImpactStudyResult" - focusVariant: MolecularVariation | iriReference - functionalImpactScore: float | None = None - specifiedBy: Method | iriReference | None = None - sourceDataSet: DataSet | None = None - -class ExperimentalVariantFunctionalImpactProposition(_SubjectVariantPropositionBase): - type: Literal["ExperimentalVariantFunctionalImpactProposition"] = "ExperimentalVariantFunctionalImpactProposition" - predicate: str = "impactsFunctionOf" - objectSequenceFeature: iriReference | MappableConcept - experimentalContextQualifier: iriReference | Document | dict | None = None - -# TODO#319: When we pull in CatVRS as a dependency, we can fully type this but as of now there is no need. -CategoricalVariant = Incomplete - -# Proposition types we don't care about and thus have no need to strongly type. -VariantDiagnosticProposition = Incomplete -VariantOncogenicityProposition = Incomplete -VariantPrognosticProposition = Incomplete -VariantTherapeuticResponseProposition = Incomplete - -# Study result types we don't care about and thus have no need to strongly type. -CohortAlleleFrequencyStudyResult = Incomplete diff --git a/mypy_stubs/ga4gh/va_spec/base/domain_entities.pyi b/mypy_stubs/ga4gh/va_spec/base/domain_entities.pyi deleted file mode 100644 index 10b57d368..000000000 --- a/mypy_stubs/ga4gh/va_spec/base/domain_entities.pyi +++ /dev/null @@ -1,11 +0,0 @@ -from pydantic import RootModel - -from ...core.models import Element, MappableConcept -from .enums import MembershipOperator - -class ConditionSet(Element): - conditions: list[MappableConcept] - membershipOperator: MembershipOperator - -class Condition(RootModel): - root: ConditionSet | MappableConcept diff --git a/mypy_stubs/ga4gh/va_spec/base/enums.pyi b/mypy_stubs/ga4gh/va_spec/base/enums.pyi deleted file mode 100644 index 04adba73c..000000000 --- a/mypy_stubs/ga4gh/va_spec/base/enums.pyi +++ /dev/null @@ -1,12 +0,0 @@ -from enum import Enum - -class MembershipOperator(str, Enum): - AND = "AND" - OR = "OR" - -class StrengthOfEvidenceProvided(str, Enum): - STANDALONE = "standalone" - VERY_STRONG = "very strong" - STRONG = "strong" - MODERATE = "moderate" - SUPPORTING = "supporting" diff --git a/mypy_stubs/ga4gh/vrs/__init__.pyi b/mypy_stubs/ga4gh/vrs/__init__.pyi deleted file mode 100644 index 3bafe7db9..000000000 --- a/mypy_stubs/ga4gh/vrs/__init__.pyi +++ /dev/null @@ -1,4 +0,0 @@ -from . import models as models -from .models import VrsType as VrsType - -__all__ = ["VrsType", "models"] diff --git a/mypy_stubs/ga4gh/vrs/dataproxy.pyi b/mypy_stubs/ga4gh/vrs/dataproxy.pyi deleted file mode 100644 index 25cabfda9..000000000 --- a/mypy_stubs/ga4gh/vrs/dataproxy.pyi +++ /dev/null @@ -1,8 +0,0 @@ -from abc import ABC - -class _DataProxy(ABC): - def get_metadata(self, identifier: str) -> dict: ... - def get_sequence(self, identifier: str, start: int | None = ..., end: int | None = ...) -> str: ... - -class SeqRepoDataProxy(_DataProxy): - def __init__(self, sr: object) -> None: ... diff --git a/mypy_stubs/ga4gh/vrs/extras/__init__.pyi b/mypy_stubs/ga4gh/vrs/extras/__init__.pyi deleted file mode 100644 index e69de29bb..000000000 diff --git a/mypy_stubs/ga4gh/vrs/extras/translator.pyi b/mypy_stubs/ga4gh/vrs/extras/translator.pyi deleted file mode 100644 index d3bb6aeb4..000000000 --- a/mypy_stubs/ga4gh/vrs/extras/translator.pyi +++ /dev/null @@ -1,19 +0,0 @@ -from typing import Any - -from ga4gh.vrs.dataproxy import _DataProxy - -class _Translator: - default_assembly_name: str - data_proxy: _DataProxy - identify: bool - def __init__( - self, - data_proxy: _DataProxy, - default_assembly_name: str = ..., - identify: bool = ..., - ) -> None: ... - # Returns a VRS variation (Allele/CisPhasedBlock/...); typed loosely so callers - # can annotate the concrete subtype they expect without a redundant cast. - def translate_from(self, var: str, fmt: str | None = ..., **kwargs: Any) -> Any: ... - -class AlleleTranslator(_Translator): ... diff --git a/mypy_stubs/ga4gh/vrs/models.pyi b/mypy_stubs/ga4gh/vrs/models.pyi deleted file mode 100644 index 73c491d88..000000000 --- a/mypy_stubs/ga4gh/vrs/models.pyi +++ /dev/null @@ -1,230 +0,0 @@ -from _typeshed import Incomplete -from abc import ABC -from collections.abc import Generator -from enum import Enum -from ..core.identifiers import PrevVrsVersion as PrevVrsVersion -from ..core.models import Entity as Entity, iriReference as IRI -from pydantic import BaseModel, RootModel, StringConstraints as StringConstraints, ValidationInfo as ValidationInfo -from typing import Annotated, Literal - -def flatten(vals) -> Generator[Incomplete, None, Incomplete]: ... -def flatten_type(t): ... -def overlaps(a: list, b: list): ... -def pydantic_class_refatt_map(): ... - -class VrsType(str, Enum): - LEN_EXPR = "LengthExpression" - REF_LEN_EXPR = "ReferenceLengthExpression" - LIT_SEQ_EXPR = "LiteralSequenceExpression" - SEQ_REF = "SequenceReference" - SEQ_LOC = "SequenceLocation" - ALLELE = "Allele" - CIS_PHASED_BLOCK = "CisPhasedBlock" - ADJACENCY = "Adjacency" - TERMINUS = "Terminus" - TRAVERSAL_BLOCK = "TraversalBlock" - DERIVATIVE_MOL = "DerivativeMolecule" - CN_COUNT = "CopyNumberCount" - CN_CHANGE = "CopyNumberChange" - -class Orientation(str, Enum): - FORWARD = "forward" - REVERSE_COMPLEMENT = "reverse_complement" - -class ResidueAlphabet(str, Enum): - AA = "aa" - NA = "na" - -class CopyChange(str, Enum): - COMPLETE_GENOMIC_LOSS = "complete genomic loss" - HIGH_LEVEL_LOSS = "high-level loss" - LOW_LEVEL_LOSS = "low-level loss" - LOSS = "loss" - REGIONAL_BASE_PLOIDY = "regional base ploidy" - GAIN = "gain" - LOW_LEVEL_GAIN = "low-level gain" - HIGH_LEVEL_GAIN = "high-level gain" - -class Syntax(str, Enum): - HGVS_C = "hgvs.c" - HGVS_P = "hgvs.p" - HGVS_G = "hgvs.g" - HGVS_M = "hgvs.m" - HGVS_N = "hgvs.n" - HGVS_R = "hgvs.r" - HGVS_ISCN = "iscn" - GNOMAD = "gnomad" - SPDI = "spdi" - -class _ValueObject(Entity, ABC): - def __hash__(self): ... - def ga4gh_serialize(self) -> dict: ... - class ga4gh: - keys: list[str] - - @staticmethod - def is_ga4gh_identifiable(): ... - -class _Ga4ghIdentifiableObject(_ValueObject, ABC): - type: str - digest: Annotated[str, None] | None = None - def __lt__(self, other): ... - @staticmethod - def is_ga4gh_identifiable(): ... - def has_valid_ga4gh_id(self): ... - def compute_digest(self, store: bool = True, as_version: PrevVrsVersion | None = None) -> str: ... - id: Incomplete - def get_or_create_ga4gh_identifier( - self, in_place: str = "default", recompute: bool = False, as_version: Incomplete | None = None - ) -> str: ... - def compute_ga4gh_identifier(self, recompute: bool = False, as_version: Incomplete | None = None): ... - def get_or_create_digest(self, recompute: bool = False) -> str: ... - class ga4gh(_ValueObject.ga4gh): - prefix: str - -class Expression(BaseModel): - syntax: Syntax - value: str - syntax_version: str | None = None - -class Range(RootModel): - root: list[int | None] - def validate_range(cls, v: list[int | None]) -> list[int | None]: ... - -class Residue(RootModel): - root: Annotated[str, None] - -class SequenceString(RootModel): - root: Annotated[str, None] - -class LengthExpression(_ValueObject): - type: Literal["LengthExpression"] = "LengthExpression" - length: Range | int | None = None - class ga4gh(_ValueObject.ga4gh): - keys: Incomplete - -class ReferenceLengthExpression(_ValueObject): - type: Literal["ReferenceLengthExpression"] = "ReferenceLengthExpression" - length: Range | int - sequence: SequenceString | None = None - repeatSubunitLength: int - class ga4gh(_ValueObject.ga4gh): - keys: Incomplete - -class LiteralSequenceExpression(_ValueObject): - type: Literal["LiteralSequenceExpression"] = "LiteralSequenceExpression" - sequence: SequenceString - class ga4gh(_ValueObject.ga4gh): - keys: Incomplete - -class SequenceReference(_ValueObject): - type: Literal["SequenceReference"] = "SequenceReference" - refgetAccession: Annotated[str, None] - residueAlphabet: ResidueAlphabet | None = None - circular: bool | None = None - class ga4gh(_ValueObject.ga4gh): - keys: Incomplete - -class SequenceLocation(_Ga4ghIdentifiableObject): - type: Literal["SequenceLocation"] = "SequenceLocation" - sequenceReference: IRI | SequenceReference | None = None - start: Range | int | None = None - end: Range | int | None = None - sequence: SequenceString | None = None - def validate_start_end(cls, v: Range | int | None, info: ValidationInfo) -> Range | int | None: ... - def ga4gh_serialize_as_version(self, as_version: PrevVrsVersion): ... - def get_refget_accession(self): ... - class ga4gh(_Ga4ghIdentifiableObject.ga4gh): - prefix: str - priorPrefix: Incomplete - keys: Incomplete - -class _VariationBase(_Ga4ghIdentifiableObject, ABC): - expressions: list[Expression] | None = None - -class Allele(_VariationBase): - type: Literal["Allele"] = "Allele" - location: IRI | SequenceLocation - state: LiteralSequenceExpression | ReferenceLengthExpression | LengthExpression - def ga4gh_serialize_as_version(self, as_version: PrevVrsVersion): ... - class ga4gh(_Ga4ghIdentifiableObject.ga4gh): - prefix: str - priorPrefix: Incomplete - keys: Incomplete - -class CisPhasedBlock(_VariationBase): - type: Literal["CisPhasedBlock"] = "CisPhasedBlock" - members: list[Allele | IRI] - sequenceReference: SequenceReference | None = None - def ga4gh_serialize(self) -> dict: ... - class ga4gh(_Ga4ghIdentifiableObject.ga4gh): - prefix: str - keys: Incomplete - -class Adjacency(_VariationBase): - type: Literal["Adjacency"] = "Adjacency" - adjoinedSequences: list[IRI | SequenceLocation] - linker: LiteralSequenceExpression | ReferenceLengthExpression | LengthExpression | None = None - homology: bool | None = None - class ga4gh(_Ga4ghIdentifiableObject.ga4gh): - prefix: str - keys: Incomplete - -class Terminus(_VariationBase): - type: Literal["Terminus"] = "Terminus" - location: IRI | SequenceLocation - class ga4gh(_Ga4ghIdentifiableObject.ga4gh): - prefix: str - keys: Incomplete - -class TraversalBlock(_ValueObject): - type: Literal["TraversalBlock"] = "TraversalBlock" - orientation: Orientation | None = None - component: Allele | CisPhasedBlock | Adjacency | Terminus | None = None - class ga4gh(_ValueObject.ga4gh): - keys: Incomplete - -class DerivativeMolecule(_VariationBase): - type: Literal["DerivativeMolecule"] = "DerivativeMolecule" - components: list[IRI | TraversalBlock] - circular: bool | None = None - class ga4gh(_Ga4ghIdentifiableObject.ga4gh): - prefix: str - keys: Incomplete - -class _CopyNumber(_VariationBase, ABC): - location: IRI | SequenceLocation - -class CopyNumberCount(_CopyNumber): - type: Literal["CopyNumberCount"] = "CopyNumberCount" - copies: Range | int - class ga4gh(_Ga4ghIdentifiableObject.ga4gh): - prefix: str - keys: Incomplete - -class CopyNumberChange(_CopyNumber): - type: Literal["CopyNumberChange"] = "CopyNumberChange" - copyChange: CopyChange - class ga4gh(_Ga4ghIdentifiableObject.ga4gh): - prefix: str - keys: Incomplete - -class MolecularVariation(RootModel): - root: Allele | CisPhasedBlock | Adjacency | Terminus | DerivativeMolecule - -class SequenceExpression(RootModel): - root: LiteralSequenceExpression | ReferenceLengthExpression | LengthExpression - -class Location(RootModel): - root: SequenceLocation - -class Variation(RootModel): - root: Allele | CisPhasedBlock | Adjacency | Terminus | DerivativeMolecule | CopyNumberChange | CopyNumberCount - -class SystemicVariation(RootModel): - root: CopyNumberChange | CopyNumberCount - -reffable_classes: Incomplete -union_reffable_classes: Incomplete -class_refatt_map: Incomplete -class_keys: Incomplete diff --git a/mypy_stubs/ga4gh/vrs/normalize.pyi b/mypy_stubs/ga4gh/vrs/normalize.pyi deleted file mode 100644 index 1aea15b23..000000000 --- a/mypy_stubs/ga4gh/vrs/normalize.pyi +++ /dev/null @@ -1,5 +0,0 @@ -from typing import Any - -from ga4gh.vrs.dataproxy import _DataProxy - -def normalize(vo: Any, data_proxy: _DataProxy | None = ..., **kwargs: Any) -> Any: ... diff --git a/poetry.lock b/poetry.lock index 51ecdb81d..027b54a21 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1693,18 +1693,18 @@ tqdm = ["tqdm"] [[package]] name = "ga4gh-cat-vrs" -version = "0.7.2" +version = "0.8.0a6" description = "GA4GH Categorical Variation Representation (Cat-VRS) reference implementation" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "ga4gh_cat_vrs-0.7.2-py3-none-any.whl", hash = "sha256:153da80cbe305a4a2b81d9d1576f7c9fc85eb110f7a80f0e708ed9f7358925c8"}, - {file = "ga4gh_cat_vrs-0.7.2.tar.gz", hash = "sha256:f06530d28aa21bf007ce0cfeee4a98458fc6506ce552029866414fd303fa4de6"}, + {file = "ga4gh_cat_vrs-0.8.0a6-py3-none-any.whl", hash = "sha256:3d9fb48dda3bb4ddb250e042716d0aaa679ee963e281f9609de6d979a774941c"}, + {file = "ga4gh_cat_vrs-0.8.0a6.tar.gz", hash = "sha256:8720fd3104a679853b63eee5b5bd5229fd279105f5d9d08d00dc2de7e7a86700"}, ] [package.dependencies] -"ga4gh.vrs" = ">=2.1.3,<3.0" +"ga4gh.vrs" = ">=2.4.0-a6,<2.5.0" pydantic = ">=2.0,<3.0" [package.extras] @@ -1714,20 +1714,21 @@ tests = ["pytest", "pytest-cov", "pyyaml"] [[package]] name = "ga4gh-va-spec" -version = "0.4.3" +version = "0.5.0a6" description = "GA4GH Variant Annotation (VA) reference implementation" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "ga4gh_va_spec-0.4.3-py3-none-any.whl", hash = "sha256:f54330b59fe1a3d27b901d15edbf9f1e3e66e16438a16654f8ef9ac0271876fa"}, - {file = "ga4gh_va_spec-0.4.3.tar.gz", hash = "sha256:a8813745d261e218932608a7905571354e14180be10c2972ed5d03f8b9a7cf72"}, + {file = "ga4gh_va_spec-0.5.0a6-py3-none-any.whl", hash = "sha256:0136e47c6f7ede65daf8981608cf190fc69f3ce94d5c7c1e30c30c86d7bbe97e"}, + {file = "ga4gh_va_spec-0.5.0a6.tar.gz", hash = "sha256:640b4a11a69a87bc80adb1c82c15a9cb2acfd5bf1bf95f9404d3329127e80cef"}, ] [package.dependencies] -"ga4gh.cat_vrs" = ">=0.7.1,<0.8.0" -"ga4gh.vrs" = ">=2.2.0,<3.0" -pydantic = ">=2.0,<3.0" +"ga4gh.cat_vrs" = ">=0.8.0-a6,<0.9.0" +"ga4gh.vrs" = ">=2.4.0-a6,<2.5.0" +pydantic = ">=2.12,<3.0" +typing_extensions = "*" [package.extras] dev = ["pre-commit", "ruff (==0.4.9)"] @@ -1736,14 +1737,14 @@ tests = ["pytest", "pytest-cov", "pyyaml"] [[package]] name = "ga4gh-vrs" -version = "2.3.1" +version = "2.4.0a6" description = "GA4GH Variation Representation Specification (VRS) reference implementation" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "ga4gh_vrs-2.3.1-py3-none-any.whl", hash = "sha256:0e0eb6436eb6b229a313eac49e58c36057c20656f3d1c8b3da14a3aaafebcf61"}, - {file = "ga4gh_vrs-2.3.1.tar.gz", hash = "sha256:911cff7eadc8a7ca9e746147d85a90fe1affbb78d68c3f9aca177bdfa9bb07db"}, + {file = "ga4gh_vrs-2.4.0a6-py3-none-any.whl", hash = "sha256:ce6850338ecea7819a5aa49504d0bfe15a1211694e199f20a09139c14f490080"}, + {file = "ga4gh_vrs-2.4.0a6.tar.gz", hash = "sha256:be68882acb659ef25af8b154440c54619e9a921993bae82feb336fd78d2588c3"}, ] [package.dependencies] @@ -3456,20 +3457,21 @@ files = [ [[package]] name = "pydantic" -version = "2.10.6" +version = "2.12.5" description = "Data validation using Python type hints" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["main"] files = [ - {file = "pydantic-2.10.6-py3-none-any.whl", hash = "sha256:427d664bf0b8a2b34ff5dd0f5a18df00591adcee7198fbd71981054cef37b584"}, - {file = "pydantic-2.10.6.tar.gz", hash = "sha256:ca5daa827cce33de7a42be142548b0096bf05a7e7b365aebfa5f8eeec7128236"}, + {file = "pydantic-2.12.5-py3-none-any.whl", hash = "sha256:e561593fccf61e8a20fc46dfc2dfe075b8be7d0188df33f221ad1f0139180f9d"}, + {file = "pydantic-2.12.5.tar.gz", hash = "sha256:4d351024c75c0f085a9febbb665ce8c0c6ec5d30e903bdb6394b7ede26aebb49"}, ] [package.dependencies] annotated-types = ">=0.6.0" -pydantic-core = "2.27.2" -typing-extensions = ">=4.12.2" +pydantic-core = "2.41.5" +typing-extensions = ">=4.14.1" +typing-inspection = ">=0.4.2" [package.extras] email = ["email-validator (>=2.0.0)"] @@ -3477,116 +3479,137 @@ timezone = ["tzdata ; python_version >= \"3.9\" and platform_system == \"Windows [[package]] name = "pydantic-core" -version = "2.27.2" +version = "2.41.5" description = "Core functionality for Pydantic validation and serialization" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["main"] files = [ - {file = "pydantic_core-2.27.2-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:2d367ca20b2f14095a8f4fa1210f5a7b78b8a20009ecced6b12818f455b1e9fa"}, - {file = "pydantic_core-2.27.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:491a2b73db93fab69731eaee494f320faa4e093dbed776be1a829c2eb222c34c"}, - {file = "pydantic_core-2.27.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7969e133a6f183be60e9f6f56bfae753585680f3b7307a8e555a948d443cc05a"}, - {file = "pydantic_core-2.27.2-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3de9961f2a346257caf0aa508a4da705467f53778e9ef6fe744c038119737ef5"}, - {file = "pydantic_core-2.27.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e2bb4d3e5873c37bb3dd58714d4cd0b0e6238cebc4177ac8fe878f8b3aa8e74c"}, - {file = "pydantic_core-2.27.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:280d219beebb0752699480fe8f1dc61ab6615c2046d76b7ab7ee38858de0a4e7"}, - {file = "pydantic_core-2.27.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47956ae78b6422cbd46f772f1746799cbb862de838fd8d1fbd34a82e05b0983a"}, - {file = "pydantic_core-2.27.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:14d4a5c49d2f009d62a2a7140d3064f686d17a5d1a268bc641954ba181880236"}, - {file = "pydantic_core-2.27.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:337b443af21d488716f8d0b6164de833e788aa6bd7e3a39c005febc1284f4962"}, - {file = "pydantic_core-2.27.2-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:03d0f86ea3184a12f41a2d23f7ccb79cdb5a18e06993f8a45baa8dfec746f0e9"}, - {file = "pydantic_core-2.27.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7041c36f5680c6e0f08d922aed302e98b3745d97fe1589db0a3eebf6624523af"}, - {file = "pydantic_core-2.27.2-cp310-cp310-win32.whl", hash = "sha256:50a68f3e3819077be2c98110c1f9dcb3817e93f267ba80a2c05bb4f8799e2ff4"}, - {file = "pydantic_core-2.27.2-cp310-cp310-win_amd64.whl", hash = "sha256:e0fd26b16394ead34a424eecf8a31a1f5137094cabe84a1bcb10fa6ba39d3d31"}, - {file = "pydantic_core-2.27.2-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:8e10c99ef58cfdf2a66fc15d66b16c4a04f62bca39db589ae8cba08bc55331bc"}, - {file = "pydantic_core-2.27.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:26f32e0adf166a84d0cb63be85c562ca8a6fa8de28e5f0d92250c6b7e9e2aff7"}, - {file = "pydantic_core-2.27.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c19d1ea0673cd13cc2f872f6c9ab42acc4e4f492a7ca9d3795ce2b112dd7e15"}, - {file = "pydantic_core-2.27.2-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5e68c4446fe0810e959cdff46ab0a41ce2f2c86d227d96dc3847af0ba7def306"}, - {file = "pydantic_core-2.27.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d9640b0059ff4f14d1f37321b94061c6db164fbe49b334b31643e0528d100d99"}, - {file = "pydantic_core-2.27.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:40d02e7d45c9f8af700f3452f329ead92da4c5f4317ca9b896de7ce7199ea459"}, - {file = "pydantic_core-2.27.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1c1fd185014191700554795c99b347d64f2bb637966c4cfc16998a0ca700d048"}, - {file = "pydantic_core-2.27.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d81d2068e1c1228a565af076598f9e7451712700b673de8f502f0334f281387d"}, - {file = "pydantic_core-2.27.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:1a4207639fb02ec2dbb76227d7c751a20b1a6b4bc52850568e52260cae64ca3b"}, - {file = "pydantic_core-2.27.2-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:3de3ce3c9ddc8bbd88f6e0e304dea0e66d843ec9de1b0042b0911c1663ffd474"}, - {file = "pydantic_core-2.27.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:30c5f68ded0c36466acede341551106821043e9afaad516adfb6e8fa80a4e6a6"}, - {file = "pydantic_core-2.27.2-cp311-cp311-win32.whl", hash = "sha256:c70c26d2c99f78b125a3459f8afe1aed4d9687c24fd677c6a4436bc042e50d6c"}, - {file = "pydantic_core-2.27.2-cp311-cp311-win_amd64.whl", hash = "sha256:08e125dbdc505fa69ca7d9c499639ab6407cfa909214d500897d02afb816e7cc"}, - {file = "pydantic_core-2.27.2-cp311-cp311-win_arm64.whl", hash = "sha256:26f0d68d4b235a2bae0c3fc585c585b4ecc51382db0e3ba402a22cbc440915e4"}, - {file = "pydantic_core-2.27.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:9e0c8cfefa0ef83b4da9588448b6d8d2a2bf1a53c3f1ae5fca39eb3061e2f0b0"}, - {file = "pydantic_core-2.27.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:83097677b8e3bd7eaa6775720ec8e0405f1575015a463285a92bfdfe254529ef"}, - {file = "pydantic_core-2.27.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:172fce187655fece0c90d90a678424b013f8fbb0ca8b036ac266749c09438cb7"}, - {file = "pydantic_core-2.27.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:519f29f5213271eeeeb3093f662ba2fd512b91c5f188f3bb7b27bc5973816934"}, - {file = "pydantic_core-2.27.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:05e3a55d124407fffba0dd6b0c0cd056d10e983ceb4e5dbd10dda135c31071d6"}, - {file = "pydantic_core-2.27.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9c3ed807c7b91de05e63930188f19e921d1fe90de6b4f5cd43ee7fcc3525cb8c"}, - {file = "pydantic_core-2.27.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6fb4aadc0b9a0c063206846d603b92030eb6f03069151a625667f982887153e2"}, - {file = "pydantic_core-2.27.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:28ccb213807e037460326424ceb8b5245acb88f32f3d2777427476e1b32c48c4"}, - {file = "pydantic_core-2.27.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:de3cd1899e2c279b140adde9357c4495ed9d47131b4a4eaff9052f23398076b3"}, - {file = "pydantic_core-2.27.2-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:220f892729375e2d736b97d0e51466252ad84c51857d4d15f5e9692f9ef12be4"}, - {file = "pydantic_core-2.27.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a0fcd29cd6b4e74fe8ddd2c90330fd8edf2e30cb52acda47f06dd615ae72da57"}, - {file = "pydantic_core-2.27.2-cp312-cp312-win32.whl", hash = "sha256:1e2cb691ed9834cd6a8be61228471d0a503731abfb42f82458ff27be7b2186fc"}, - {file = "pydantic_core-2.27.2-cp312-cp312-win_amd64.whl", hash = "sha256:cc3f1a99a4f4f9dd1de4fe0312c114e740b5ddead65bb4102884b384c15d8bc9"}, - {file = "pydantic_core-2.27.2-cp312-cp312-win_arm64.whl", hash = "sha256:3911ac9284cd8a1792d3cb26a2da18f3ca26c6908cc434a18f730dc0db7bfa3b"}, - {file = "pydantic_core-2.27.2-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:7d14bd329640e63852364c306f4d23eb744e0f8193148d4044dd3dacdaacbd8b"}, - {file = "pydantic_core-2.27.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:82f91663004eb8ed30ff478d77c4d1179b3563df6cdb15c0817cd1cdaf34d154"}, - {file = "pydantic_core-2.27.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:71b24c7d61131bb83df10cc7e687433609963a944ccf45190cfc21e0887b08c9"}, - {file = "pydantic_core-2.27.2-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fa8e459d4954f608fa26116118bb67f56b93b209c39b008277ace29937453dc9"}, - {file = "pydantic_core-2.27.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ce8918cbebc8da707ba805b7fd0b382816858728ae7fe19a942080c24e5b7cd1"}, - {file = "pydantic_core-2.27.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:eda3f5c2a021bbc5d976107bb302e0131351c2ba54343f8a496dc8783d3d3a6a"}, - {file = "pydantic_core-2.27.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bd8086fa684c4775c27f03f062cbb9eaa6e17f064307e86b21b9e0abc9c0f02e"}, - {file = "pydantic_core-2.27.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8d9b3388db186ba0c099a6d20f0604a44eabdeef1777ddd94786cdae158729e4"}, - {file = "pydantic_core-2.27.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:7a66efda2387de898c8f38c0cf7f14fca0b51a8ef0b24bfea5849f1b3c95af27"}, - {file = "pydantic_core-2.27.2-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:18a101c168e4e092ab40dbc2503bdc0f62010e95d292b27827871dc85450d7ee"}, - {file = "pydantic_core-2.27.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:ba5dd002f88b78a4215ed2f8ddbdf85e8513382820ba15ad5ad8955ce0ca19a1"}, - {file = "pydantic_core-2.27.2-cp313-cp313-win32.whl", hash = "sha256:1ebaf1d0481914d004a573394f4be3a7616334be70261007e47c2a6fe7e50130"}, - {file = "pydantic_core-2.27.2-cp313-cp313-win_amd64.whl", hash = "sha256:953101387ecf2f5652883208769a79e48db18c6df442568a0b5ccd8c2723abee"}, - {file = "pydantic_core-2.27.2-cp313-cp313-win_arm64.whl", hash = "sha256:ac4dbfd1691affb8f48c2c13241a2e3b60ff23247cbcf981759c768b6633cf8b"}, - {file = "pydantic_core-2.27.2-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:d3e8d504bdd3f10835468f29008d72fc8359d95c9c415ce6e767203db6127506"}, - {file = "pydantic_core-2.27.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:521eb9b7f036c9b6187f0b47318ab0d7ca14bd87f776240b90b21c1f4f149320"}, - {file = "pydantic_core-2.27.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:85210c4d99a0114f5a9481b44560d7d1e35e32cc5634c656bc48e590b669b145"}, - {file = "pydantic_core-2.27.2-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d716e2e30c6f140d7560ef1538953a5cd1a87264c737643d481f2779fc247fe1"}, - {file = "pydantic_core-2.27.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f66d89ba397d92f840f8654756196d93804278457b5fbede59598a1f9f90b228"}, - {file = "pydantic_core-2.27.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:669e193c1c576a58f132e3158f9dfa9662969edb1a250c54d8fa52590045f046"}, - {file = "pydantic_core-2.27.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9fdbe7629b996647b99c01b37f11170a57ae675375b14b8c13b8518b8320ced5"}, - {file = "pydantic_core-2.27.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d262606bf386a5ba0b0af3b97f37c83d7011439e3dc1a9298f21efb292e42f1a"}, - {file = "pydantic_core-2.27.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:cabb9bcb7e0d97f74df8646f34fc76fbf793b7f6dc2438517d7a9e50eee4f14d"}, - {file = "pydantic_core-2.27.2-cp38-cp38-musllinux_1_1_armv7l.whl", hash = "sha256:d2d63f1215638d28221f664596b1ccb3944f6e25dd18cd3b86b0a4c408d5ebb9"}, - {file = "pydantic_core-2.27.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:bca101c00bff0adb45a833f8451b9105d9df18accb8743b08107d7ada14bd7da"}, - {file = "pydantic_core-2.27.2-cp38-cp38-win32.whl", hash = "sha256:f6f8e111843bbb0dee4cb6594cdc73e79b3329b526037ec242a3e49012495b3b"}, - {file = "pydantic_core-2.27.2-cp38-cp38-win_amd64.whl", hash = "sha256:fd1aea04935a508f62e0d0ef1f5ae968774a32afc306fb8545e06f5ff5cdf3ad"}, - {file = "pydantic_core-2.27.2-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:c10eb4f1659290b523af58fa7cffb452a61ad6ae5613404519aee4bfbf1df993"}, - {file = "pydantic_core-2.27.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ef592d4bad47296fb11f96cd7dc898b92e795032b4894dfb4076cfccd43a9308"}, - {file = "pydantic_core-2.27.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c61709a844acc6bf0b7dce7daae75195a10aac96a596ea1b776996414791ede4"}, - {file = "pydantic_core-2.27.2-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:42c5f762659e47fdb7b16956c71598292f60a03aa92f8b6351504359dbdba6cf"}, - {file = "pydantic_core-2.27.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4c9775e339e42e79ec99c441d9730fccf07414af63eac2f0e48e08fd38a64d76"}, - {file = "pydantic_core-2.27.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:57762139821c31847cfb2df63c12f725788bd9f04bc2fb392790959b8f70f118"}, - {file = "pydantic_core-2.27.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0d1e85068e818c73e048fe28cfc769040bb1f475524f4745a5dc621f75ac7630"}, - {file = "pydantic_core-2.27.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:097830ed52fd9e427942ff3b9bc17fab52913b2f50f2880dc4a5611446606a54"}, - {file = "pydantic_core-2.27.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:044a50963a614ecfae59bb1eaf7ea7efc4bc62f49ed594e18fa1e5d953c40e9f"}, - {file = "pydantic_core-2.27.2-cp39-cp39-musllinux_1_1_armv7l.whl", hash = "sha256:4e0b4220ba5b40d727c7f879eac379b822eee5d8fff418e9d3381ee45b3b0362"}, - {file = "pydantic_core-2.27.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5e4f4bb20d75e9325cc9696c6802657b58bc1dbbe3022f32cc2b2b632c3fbb96"}, - {file = "pydantic_core-2.27.2-cp39-cp39-win32.whl", hash = "sha256:cca63613e90d001b9f2f9a9ceb276c308bfa2a43fafb75c8031c4f66039e8c6e"}, - {file = "pydantic_core-2.27.2-cp39-cp39-win_amd64.whl", hash = "sha256:77d1bca19b0f7021b3a982e6f903dcd5b2b06076def36a652e3907f596e29f67"}, - {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:2bf14caea37e91198329b828eae1618c068dfb8ef17bb33287a7ad4b61ac314e"}, - {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:b0cb791f5b45307caae8810c2023a184c74605ec3bcbb67d13846c28ff731ff8"}, - {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:688d3fd9fcb71f41c4c015c023d12a79d1c4c0732ec9eb35d96e3388a120dcf3"}, - {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3d591580c34f4d731592f0e9fe40f9cc1b430d297eecc70b962e93c5c668f15f"}, - {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:82f986faf4e644ffc189a7f1aafc86e46ef70372bb153e7001e8afccc6e54133"}, - {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:bec317a27290e2537f922639cafd54990551725fc844249e64c523301d0822fc"}, - {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:0296abcb83a797db256b773f45773da397da75a08f5fcaef41f2044adec05f50"}, - {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:0d75070718e369e452075a6017fbf187f788e17ed67a3abd47fa934d001863d9"}, - {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:7e17b560be3c98a8e3aa66ce828bdebb9e9ac6ad5466fba92eb74c4c95cb1151"}, - {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:c33939a82924da9ed65dab5a65d427205a73181d8098e79b6b426bdf8ad4e656"}, - {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:00bad2484fa6bda1e216e7345a798bd37c68fb2d97558edd584942aa41b7d278"}, - {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c817e2b40aba42bac6f457498dacabc568c3b7a986fc9ba7c8d9d260b71485fb"}, - {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:251136cdad0cb722e93732cb45ca5299fb56e1344a833640bf93b2803f8d1bfd"}, - {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d2088237af596f0a524d3afc39ab3b036e8adb054ee57cbb1dcf8e09da5b29cc"}, - {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:d4041c0b966a84b4ae7a09832eb691a35aec90910cd2dbe7a208de59be77965b"}, - {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:8083d4e875ebe0b864ffef72a4304827015cff328a1be6e22cc850753bfb122b"}, - {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f141ee28a0ad2123b6611b6ceff018039df17f32ada8b534e6aa039545a3efb2"}, - {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7d0c8399fcc1848491f00e0314bd59fb34a9c008761bcb422a057670c3f65e35"}, - {file = "pydantic_core-2.27.2.tar.gz", hash = "sha256:eb026e5a4c1fee05726072337ff51d1efb6f59090b7da90d30ea58625b1ffb39"}, + {file = "pydantic_core-2.41.5-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:77b63866ca88d804225eaa4af3e664c5faf3568cea95360d21f4725ab6e07146"}, + {file = "pydantic_core-2.41.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:dfa8a0c812ac681395907e71e1274819dec685fec28273a28905df579ef137e2"}, + {file = "pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5921a4d3ca3aee735d9fd163808f5e8dd6c6972101e4adbda9a4667908849b97"}, + {file = "pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e25c479382d26a2a41b7ebea1043564a937db462816ea07afa8a44c0866d52f9"}, + {file = "pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f547144f2966e1e16ae626d8ce72b4cfa0caedc7fa28052001c94fb2fcaa1c52"}, + {file = "pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6f52298fbd394f9ed112d56f3d11aabd0d5bd27beb3084cc3d8ad069483b8941"}, + {file = "pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:100baa204bb412b74fe285fb0f3a385256dad1d1879f0a5cb1499ed2e83d132a"}, + {file = "pydantic_core-2.41.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:05a2c8852530ad2812cb7914dc61a1125dc4e06252ee98e5638a12da6cc6fb6c"}, + {file = "pydantic_core-2.41.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:29452c56df2ed968d18d7e21f4ab0ac55e71dc59524872f6fc57dcf4a3249ed2"}, + {file = "pydantic_core-2.41.5-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:d5160812ea7a8a2ffbe233d8da666880cad0cbaf5d4de74ae15c313213d62556"}, + {file = "pydantic_core-2.41.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:df3959765b553b9440adfd3c795617c352154e497a4eaf3752555cfb5da8fc49"}, + {file = "pydantic_core-2.41.5-cp310-cp310-win32.whl", hash = "sha256:1f8d33a7f4d5a7889e60dc39856d76d09333d8a6ed0f5f1190635cbec70ec4ba"}, + {file = "pydantic_core-2.41.5-cp310-cp310-win_amd64.whl", hash = "sha256:62de39db01b8d593e45871af2af9e497295db8d73b085f6bfd0b18c83c70a8f9"}, + {file = "pydantic_core-2.41.5-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:a3a52f6156e73e7ccb0f8cced536adccb7042be67cb45f9562e12b319c119da6"}, + {file = "pydantic_core-2.41.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7f3bf998340c6d4b0c9a2f02d6a400e51f123b59565d74dc60d252ce888c260b"}, + {file = "pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:378bec5c66998815d224c9ca994f1e14c0c21cb95d2f52b6021cc0b2a58f2a5a"}, + {file = "pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e7b576130c69225432866fe2f4a469a85a54ade141d96fd396dffcf607b558f8"}, + {file = "pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6cb58b9c66f7e4179a2d5e0f849c48eff5c1fca560994d6eb6543abf955a149e"}, + {file = "pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:88942d3a3dff3afc8288c21e565e476fc278902ae4d6d134f1eeda118cc830b1"}, + {file = "pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f31d95a179f8d64d90f6831d71fa93290893a33148d890ba15de25642c5d075b"}, + {file = "pydantic_core-2.41.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c1df3d34aced70add6f867a8cf413e299177e0c22660cc767218373d0779487b"}, + {file = "pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:4009935984bd36bd2c774e13f9a09563ce8de4abaa7226f5108262fa3e637284"}, + {file = "pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:34a64bc3441dc1213096a20fe27e8e128bd3ff89921706e83c0b1ac971276594"}, + {file = "pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:c9e19dd6e28fdcaa5a1de679aec4141f691023916427ef9bae8584f9c2fb3b0e"}, + {file = "pydantic_core-2.41.5-cp311-cp311-win32.whl", hash = "sha256:2c010c6ded393148374c0f6f0bf89d206bf3217f201faa0635dcd56bd1520f6b"}, + {file = "pydantic_core-2.41.5-cp311-cp311-win_amd64.whl", hash = "sha256:76ee27c6e9c7f16f47db7a94157112a2f3a00e958bc626e2f4ee8bec5c328fbe"}, + {file = "pydantic_core-2.41.5-cp311-cp311-win_arm64.whl", hash = "sha256:4bc36bbc0b7584de96561184ad7f012478987882ebf9f9c389b23f432ea3d90f"}, + {file = "pydantic_core-2.41.5-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:f41a7489d32336dbf2199c8c0a215390a751c5b014c2c1c5366e817202e9cdf7"}, + {file = "pydantic_core-2.41.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:070259a8818988b9a84a449a2a7337c7f430a22acc0859c6b110aa7212a6d9c0"}, + {file = "pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e96cea19e34778f8d59fe40775a7a574d95816eb150850a85a7a4c8f4b94ac69"}, + {file = "pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ed2e99c456e3fadd05c991f8f437ef902e00eedf34320ba2b0842bd1c3ca3a75"}, + {file = "pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:65840751b72fbfd82c3c640cff9284545342a4f1eb1586ad0636955b261b0b05"}, + {file = "pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e536c98a7626a98feb2d3eaf75944ef6f3dbee447e1f841eae16f2f0a72d8ddc"}, + {file = "pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eceb81a8d74f9267ef4081e246ffd6d129da5d87e37a77c9bde550cb04870c1c"}, + {file = "pydantic_core-2.41.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d38548150c39b74aeeb0ce8ee1d8e82696f4a4e16ddc6de7b1d8823f7de4b9b5"}, + {file = "pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c23e27686783f60290e36827f9c626e63154b82b116d7fe9adba1fda36da706c"}, + {file = "pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:482c982f814460eabe1d3bb0adfdc583387bd4691ef00b90575ca0d2b6fe2294"}, + {file = "pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:bfea2a5f0b4d8d43adf9d7b8bf019fb46fdd10a2e5cde477fbcb9d1fa08c68e1"}, + {file = "pydantic_core-2.41.5-cp312-cp312-win32.whl", hash = "sha256:b74557b16e390ec12dca509bce9264c3bbd128f8a2c376eaa68003d7f327276d"}, + {file = "pydantic_core-2.41.5-cp312-cp312-win_amd64.whl", hash = "sha256:1962293292865bca8e54702b08a4f26da73adc83dd1fcf26fbc875b35d81c815"}, + {file = "pydantic_core-2.41.5-cp312-cp312-win_arm64.whl", hash = "sha256:1746d4a3d9a794cacae06a5eaaccb4b8643a131d45fbc9af23e353dc0a5ba5c3"}, + {file = "pydantic_core-2.41.5-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:941103c9be18ac8daf7b7adca8228f8ed6bb7a1849020f643b3a14d15b1924d9"}, + {file = "pydantic_core-2.41.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:112e305c3314f40c93998e567879e887a3160bb8689ef3d2c04b6cc62c33ac34"}, + {file = "pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0cbaad15cb0c90aa221d43c00e77bb33c93e8d36e0bf74760cd00e732d10a6a0"}, + {file = "pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:03ca43e12fab6023fc79d28ca6b39b05f794ad08ec2feccc59a339b02f2b3d33"}, + {file = "pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dc799088c08fa04e43144b164feb0c13f9a0bc40503f8df3e9fde58a3c0c101e"}, + {file = "pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:97aeba56665b4c3235a0e52b2c2f5ae9cd071b8a8310ad27bddb3f7fb30e9aa2"}, + {file = "pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:406bf18d345822d6c21366031003612b9c77b3e29ffdb0f612367352aab7d586"}, + {file = "pydantic_core-2.41.5-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b93590ae81f7010dbe380cdeab6f515902ebcbefe0b9327cc4804d74e93ae69d"}, + {file = "pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:01a3d0ab748ee531f4ea6c3e48ad9dac84ddba4b0d82291f87248f2f9de8d740"}, + {file = "pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:6561e94ba9dacc9c61bce40e2d6bdc3bfaa0259d3ff36ace3b1e6901936d2e3e"}, + {file = "pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:915c3d10f81bec3a74fbd4faebe8391013ba61e5a1a8d48c4455b923bdda7858"}, + {file = "pydantic_core-2.41.5-cp313-cp313-win32.whl", hash = "sha256:650ae77860b45cfa6e2cdafc42618ceafab3a2d9a3811fcfbd3bbf8ac3c40d36"}, + {file = "pydantic_core-2.41.5-cp313-cp313-win_amd64.whl", hash = "sha256:79ec52ec461e99e13791ec6508c722742ad745571f234ea6255bed38c6480f11"}, + {file = "pydantic_core-2.41.5-cp313-cp313-win_arm64.whl", hash = "sha256:3f84d5c1b4ab906093bdc1ff10484838aca54ef08de4afa9de0f5f14d69639cd"}, + {file = "pydantic_core-2.41.5-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:3f37a19d7ebcdd20b96485056ba9e8b304e27d9904d233d7b1015db320e51f0a"}, + {file = "pydantic_core-2.41.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1d1d9764366c73f996edd17abb6d9d7649a7eb690006ab6adbda117717099b14"}, + {file = "pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25e1c2af0fce638d5f1988b686f3b3ea8cd7de5f244ca147c777769e798a9cd1"}, + {file = "pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:506d766a8727beef16b7adaeb8ee6217c64fc813646b424d0804d67c16eddb66"}, + {file = "pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4819fa52133c9aa3c387b3328f25c1facc356491e6135b459f1de698ff64d869"}, + {file = "pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2b761d210c9ea91feda40d25b4efe82a1707da2ef62901466a42492c028553a2"}, + {file = "pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22f0fb8c1c583a3b6f24df2470833b40207e907b90c928cc8d3594b76f874375"}, + {file = "pydantic_core-2.41.5-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2782c870e99878c634505236d81e5443092fba820f0373997ff75f90f68cd553"}, + {file = "pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:0177272f88ab8312479336e1d777f6b124537d47f2123f89cb37e0accea97f90"}, + {file = "pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:63510af5e38f8955b8ee5687740d6ebf7c2a0886d15a6d65c32814613681bc07"}, + {file = "pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:e56ba91f47764cc14f1daacd723e3e82d1a89d783f0f5afe9c364b8bb491ccdb"}, + {file = "pydantic_core-2.41.5-cp314-cp314-win32.whl", hash = "sha256:aec5cf2fd867b4ff45b9959f8b20ea3993fc93e63c7363fe6851424c8a7e7c23"}, + {file = "pydantic_core-2.41.5-cp314-cp314-win_amd64.whl", hash = "sha256:8e7c86f27c585ef37c35e56a96363ab8de4e549a95512445b85c96d3e2f7c1bf"}, + {file = "pydantic_core-2.41.5-cp314-cp314-win_arm64.whl", hash = "sha256:e672ba74fbc2dc8eea59fb6d4aed6845e6905fc2a8afe93175d94a83ba2a01a0"}, + {file = "pydantic_core-2.41.5-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:8566def80554c3faa0e65ac30ab0932b9e3a5cd7f8323764303d468e5c37595a"}, + {file = "pydantic_core-2.41.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b80aa5095cd3109962a298ce14110ae16b8c1aece8b72f9dafe81cf597ad80b3"}, + {file = "pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3006c3dd9ba34b0c094c544c6006cc79e87d8612999f1a5d43b769b89181f23c"}, + {file = "pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:72f6c8b11857a856bcfa48c86f5368439f74453563f951e473514579d44aa612"}, + {file = "pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5cb1b2f9742240e4bb26b652a5aeb840aa4b417c7748b6f8387927bc6e45e40d"}, + {file = "pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bd3d54f38609ff308209bd43acea66061494157703364ae40c951f83ba99a1a9"}, + {file = "pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ff4321e56e879ee8d2a879501c8e469414d948f4aba74a2d4593184eb326660"}, + {file = "pydantic_core-2.41.5-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d0d2568a8c11bf8225044aa94409e21da0cb09dcdafe9ecd10250b2baad531a9"}, + {file = "pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:a39455728aabd58ceabb03c90e12f71fd30fa69615760a075b9fec596456ccc3"}, + {file = "pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:239edca560d05757817c13dc17c50766136d21f7cd0fac50295499ae24f90fdf"}, + {file = "pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:2a5e06546e19f24c6a96a129142a75cee553cc018ffee48a460059b1185f4470"}, + {file = "pydantic_core-2.41.5-cp314-cp314t-win32.whl", hash = "sha256:b4ececa40ac28afa90871c2cc2b9ffd2ff0bf749380fbdf57d165fd23da353aa"}, + {file = "pydantic_core-2.41.5-cp314-cp314t-win_amd64.whl", hash = "sha256:80aa89cad80b32a912a65332f64a4450ed00966111b6615ca6816153d3585a8c"}, + {file = "pydantic_core-2.41.5-cp314-cp314t-win_arm64.whl", hash = "sha256:35b44f37a3199f771c3eaa53051bc8a70cd7b54f333531c59e29fd4db5d15008"}, + {file = "pydantic_core-2.41.5-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:8bfeaf8735be79f225f3fefab7f941c712aaca36f1128c9d7e2352ee1aa87bdf"}, + {file = "pydantic_core-2.41.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:346285d28e4c8017da95144c7f3acd42740d637ff41946af5ce6e5e420502dd5"}, + {file = "pydantic_core-2.41.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a75dafbf87d6276ddc5b2bf6fae5254e3d0876b626eb24969a574fff9149ee5d"}, + {file = "pydantic_core-2.41.5-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7b93a4d08587e2b7e7882de461e82b6ed76d9026ce91ca7915e740ecc7855f60"}, + {file = "pydantic_core-2.41.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e8465ab91a4bd96d36dde3263f06caa6a8a6019e4113f24dc753d79a8b3a3f82"}, + {file = "pydantic_core-2.41.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:299e0a22e7ae2b85c1a57f104538b2656e8ab1873511fd718a1c1c6f149b77b5"}, + {file = "pydantic_core-2.41.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:707625ef0983fcfb461acfaf14de2067c5942c6bb0f3b4c99158bed6fedd3cf3"}, + {file = "pydantic_core-2.41.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f41eb9797986d6ebac5e8edff36d5cef9de40def462311b3eb3eeded1431e425"}, + {file = "pydantic_core-2.41.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0384e2e1021894b1ff5a786dbf94771e2986ebe2869533874d7e43bc79c6f504"}, + {file = "pydantic_core-2.41.5-cp39-cp39-musllinux_1_1_armv7l.whl", hash = "sha256:f0cd744688278965817fd0839c4a4116add48d23890d468bc436f78beb28abf5"}, + {file = "pydantic_core-2.41.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:753e230374206729bf0a807954bcc6c150d3743928a73faffee51ac6557a03c3"}, + {file = "pydantic_core-2.41.5-cp39-cp39-win32.whl", hash = "sha256:873e0d5b4fb9b89ef7c2d2a963ea7d02879d9da0da8d9d4933dee8ee86a8b460"}, + {file = "pydantic_core-2.41.5-cp39-cp39-win_amd64.whl", hash = "sha256:e4f4a984405e91527a0d62649ee21138f8e3d0ef103be488c1dc11a80d7f184b"}, + {file = "pydantic_core-2.41.5-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:b96d5f26b05d03cc60f11a7761a5ded1741da411e7fe0909e27a5e6a0cb7b034"}, + {file = "pydantic_core-2.41.5-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:634e8609e89ceecea15e2d61bc9ac3718caaaa71963717bf3c8f38bfde64242c"}, + {file = "pydantic_core-2.41.5-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:93e8740d7503eb008aa2df04d3b9735f845d43ae845e6dcd2be0b55a2da43cd2"}, + {file = "pydantic_core-2.41.5-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f15489ba13d61f670dcc96772e733aad1a6f9c429cc27574c6cdaed82d0146ad"}, + {file = "pydantic_core-2.41.5-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:7da7087d756b19037bc2c06edc6c170eeef3c3bafcb8f532ff17d64dc427adfd"}, + {file = "pydantic_core-2.41.5-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:aabf5777b5c8ca26f7824cb4a120a740c9588ed58df9b2d196ce92fba42ff8dc"}, + {file = "pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c007fe8a43d43b3969e8469004e9845944f1a80e6acd47c150856bb87f230c56"}, + {file = "pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:76d0819de158cd855d1cbb8fcafdf6f5cf1eb8e470abe056d5d161106e38062b"}, + {file = "pydantic_core-2.41.5-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:b5819cd790dbf0c5eb9f82c73c16b39a65dd6dd4d1439dcdea7816ec9adddab8"}, + {file = "pydantic_core-2.41.5-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:5a4e67afbc95fa5c34cf27d9089bca7fcab4e51e57278d710320a70b956d1b9a"}, + {file = "pydantic_core-2.41.5-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ece5c59f0ce7d001e017643d8d24da587ea1f74f6993467d85ae8a5ef9d4f42b"}, + {file = "pydantic_core-2.41.5-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:16f80f7abe3351f8ea6858914ddc8c77e02578544a0ebc15b4c2e1a0e813b0b2"}, + {file = "pydantic_core-2.41.5-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:33cb885e759a705b426baada1fe68cbb0a2e68e34c5d0d0289a364cf01709093"}, + {file = "pydantic_core-2.41.5-pp310-pypy310_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:c8d8b4eb992936023be7dee581270af5c6e0697a8559895f527f5b7105ecd36a"}, + {file = "pydantic_core-2.41.5-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:242a206cd0318f95cd21bdacff3fcc3aab23e79bba5cac3db5a841c9ef9c6963"}, + {file = "pydantic_core-2.41.5-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d3a978c4f57a597908b7e697229d996d77a6d3c94901e9edee593adada95ce1a"}, + {file = "pydantic_core-2.41.5-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:b2379fa7ed44ddecb5bfe4e48577d752db9fc10be00a6b7446e9663ba143de26"}, + {file = "pydantic_core-2.41.5-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:266fb4cbf5e3cbd0b53669a6d1b039c45e3ce651fd5442eff4d07c2cc8d66808"}, + {file = "pydantic_core-2.41.5-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58133647260ea01e4d0500089a8c4f07bd7aa6ce109682b1426394988d8aaacc"}, + {file = "pydantic_core-2.41.5-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:287dad91cfb551c363dc62899a80e9e14da1f0e2b6ebde82c806612ca2a13ef1"}, + {file = "pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:03b77d184b9eb40240ae9fd676ca364ce1085f203e1b1256f8ab9984dca80a84"}, + {file = "pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:a668ce24de96165bb239160b3d854943128f4334822900534f2fe947930e5770"}, + {file = "pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f14f8f046c14563f8eb3f45f499cc658ab8d10072961e07225e507adb700e93f"}, + {file = "pydantic_core-2.41.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:56121965f7a4dc965bff783d70b907ddf3d57f6eba29b6d2e5dabfaf07799c51"}, + {file = "pydantic_core-2.41.5.tar.gz", hash = "sha256:08daa51ea16ad373ffd5e7606252cc32f07bc72b28284b6bc9c6df804816476e"}, ] [package.dependencies] -typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0" +typing-extensions = ">=4.14.1" [[package]] name = "pygments" @@ -4762,19 +4785,6 @@ files = [ {file = "types_s3transfer-0.16.0.tar.gz", hash = "sha256:b4636472024c5e2b62278c5b759661efeb52a81851cde5f092f24100b1ecb443"}, ] -[[package]] -name = "typing-extensions" -version = "4.15.0" -description = "Backported and Experimental Type Hints for Python 3.9+" -optional = false -python-versions = ">=3.9" -groups = ["main", "dev"] -markers = "python_version >= \"3.15\"" -files = [ - {file = "typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548"}, - {file = "typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466"}, -] - [[package]] name = "typing-extensions" version = "4.16.0" @@ -4782,7 +4792,6 @@ description = "Backported and Experimental Type Hints for Python 3.9+" optional = false python-versions = ">=3.9" groups = ["main", "dev"] -markers = "python_version < \"3.15\"" files = [ {file = "typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8"}, {file = "typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5"}, @@ -4792,10 +4801,9 @@ files = [ name = "typing-inspection" version = "0.4.4" description = "Runtime typing introspection tools" -optional = true +optional = false python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"server\"" files = [ {file = "typing_inspection-0.4.4-py3-none-any.whl", hash = "sha256:65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147"}, {file = "typing_inspection-0.4.4.tar.gz", hash = "sha256:547274fa6b0a561ccf549cc9524b999a578e737d015d8709d021f9d0d13bea47"}, @@ -5293,4 +5301,4 @@ server = ["aiocache", "alembic", "alembic-utils", "arq", "authlib", "biocommons" [metadata] lock-version = "2.1" python-versions = "^3.11" -content-hash = "bf3730983cb78a69ea69c1ecbb92bfd18d430f8843005480b4b9f19e8b7bb686" +content-hash = "764312324738b85692022685484ea6dd33b82eb7d456aeb677b7a95b52b8e230" diff --git a/pyproject.toml b/pyproject.toml index ce255b293..c99be8981 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,12 +34,13 @@ email_validator = "~2.1.1" numpy = "~1.26" httpx = "~0.26.0" pandas = ">=2.2.0,<3.0.0" -pydantic = "~2.10.0" +pydantic = "~2.12.0" python-dotenv = "^1.2" python-json-logger = "~2.0.7" SQLAlchemy = "~2.0.29" -ga4gh-va-spec = "~0.4.2" -ga4gh-cat-vrs = "~0.7.2" +ga4gh-vrs = { version = "2.4.0a6", allow-prereleases = true } +ga4gh-cat-vrs = { version = "0.8.0a6", allow-prereleases = true } +ga4gh-va-spec = { version = "0.5.0a6", allow-prereleases = true } setuptools = "~81.0.0" # Optional dependencies for running this application as a server @@ -113,10 +114,11 @@ mypy_path = "mypy_stubs" module = "variant_annotation.*" ignore_missing_imports = true -# ga4gh.cat_vrs (the Cat-VRS 1.0.0 pydantic models) ships no py.typed marker, unlike ga4gh.vrs. +# The GA4GH packages ship no py.typed marker, but their pydantic models are fully annotated. Read them +# directly rather than through local stubs, which fall out of step with each release. [[tool.mypy.overrides]] -module = "ga4gh.cat_vrs.*" -ignore_missing_imports = true +module = "ga4gh.*" +follow_untyped_imports = true [tool.pytest.ini_options] addopts = "-v --import-mode=importlib" diff --git a/src/mavedb/lib/annotation/condition.py b/src/mavedb/lib/annotation/condition.py index ae1f1b7af..b910138f9 100644 --- a/src/mavedb/lib/annotation/condition.py +++ b/src/mavedb/lib/annotation/condition.py @@ -10,4 +10,4 @@ def calibration_disease_condition(score_calibration: ScoreCalibration) -> Condit Every calibration carries a non-null MONDO disease term (often the generic "disease or disorder" (``MONDO:0000001``)) which is serialized directly to a condition. """ - return Condition(root=mondo_term_to_mappable_concept(score_calibration.disease_term)) + return Condition(**mondo_term_to_mappable_concept(score_calibration.disease_term).model_dump(exclude_none=True)) diff --git a/src/mavedb/lib/annotation/conformance.py b/src/mavedb/lib/annotation/conformance.py index 851980cb5..8766a19cb 100644 --- a/src/mavedb/lib/annotation/conformance.py +++ b/src/mavedb/lib/annotation/conformance.py @@ -11,7 +11,7 @@ """ import json -from typing import TypeVar +from typing import Any, TypeVar, Union from ga4gh.va_spec.acmg_2015 import VariantPathogenicityStatement from ga4gh.va_spec.base.core import ExperimentalVariantFunctionalImpactStudyResult, Statement @@ -21,6 +21,20 @@ ) +# VariantPathogenicityStatement is a Statement, so the union covers every emitted layer. +EmittedAnnotation = Union[ExperimentalVariantFunctionalImpactStudyResult, Statement] + + +def serialize_annotation(annotation: EmittedAnnotation, *, exclude_none: bool = True) -> dict[str, Any]: + """Dump an annotation for emission, nested evidence included in full. + + VA-Spec declares ``EvidenceLine.hasEvidenceItems`` as ``list[InformationEntity]``, and pydantic serializes + a field as its declared type: without ``serialize_as_any``, a nested study result loses ``focus``, + ``functionalImpactScore`` and ``sourceDataSet``. Every emission path goes through here for that reason. + """ + return annotation.model_dump(exclude_none=exclude_none, serialize_as_any=True) + + class AnnotationRoundTripError(Exception): """An emitted annotation did not survive serialization and re-validation.""" @@ -28,8 +42,8 @@ class AnnotationRoundTripError(Exception): def round_trip_annotation(annotation: Annotation) -> Annotation: """Serialize an annotation the way the API emits it, then read it back. - Mirrors the emission path exactly — ``model_dump(exclude_none=True)`` then ``json.dumps(default=str)`` - — because the failures worth catching are the ones those two steps introduce. + Mirrors the emission path exactly — :func:`serialize_annotation` then ``json.dumps(default=str)`` — + because the failures worth catching are the ones those two steps introduce. The assertion is that emission is a fixed point: the re-validated object dumps to the same JSON it was parsed from. Object equality would be the wrong test. VA-Spec declares container fields in terms @@ -44,14 +58,14 @@ def round_trip_annotation(annotation: Annotation) -> Annotation: that serializes differently than what was emitted. """ model = type(annotation) - emitted = json.dumps(annotation.model_dump(exclude_none=True), default=str) + emitted = json.dumps(serialize_annotation(annotation), default=str) try: reparsed = model.model_validate(json.loads(emitted)) except Exception as err: raise AnnotationRoundTripError(f"{model.__name__} did not re-validate after emission: {err}") from err - re_emitted = json.dumps(reparsed.model_dump(exclude_none=True), default=str) + re_emitted = json.dumps(serialize_annotation(reparsed), default=str) if re_emitted != emitted: raise AnnotationRoundTripError(f"{model.__name__} did not survive a round trip unchanged.") diff --git a/src/mavedb/lib/annotation/context.py b/src/mavedb/lib/annotation/context.py index 5c9255cb7..f824dc82f 100644 --- a/src/mavedb/lib/annotation/context.py +++ b/src/mavedb/lib/annotation/context.py @@ -65,7 +65,7 @@ def variant_annotation_context( # The proposition subject follows the measured as anchor rule: the categorical variant when it carries a # projection member, else the concrete measured variation. The VA subject is deliberately *narrow* # (include_convergent=False): the convergent encodings are dropped, because VA-Spec carries no per-member - # provenance to mark them as unmeasured, and StudyResult.focusVariant already pins the concrete measured + # provenance to mark them as unmeasured, and StudyResult.focus already pins the concrete measured # allele. A lone measured allele is served bare, so it skips the cross-reference fetch and the build. member_links = categorical_member_links(links, include_convergent=False) transit = None @@ -74,7 +74,7 @@ def variant_annotation_context( transit = build_categorical_variant( links, name=variant.urn or "", include_convergent=False, cross_references=cross_references ) - if transit is not None and len(transit.categorical_variant.members) > 1: + if transit is not None and len(transit.categorical_variant.members or []) > 1: subject_variant: MolecularVariation | CategoricalVariant = transit.categorical_variant else: subject_variant = vrs_object_from_mapped_variant(measured_allele.post_mapped) diff --git a/src/mavedb/lib/annotation/evidence_line.py b/src/mavedb/lib/annotation/evidence_line.py index 9ce46b181..00e5823d4 100644 --- a/src/mavedb/lib/annotation/evidence_line.py +++ b/src/mavedb/lib/annotation/evidence_line.py @@ -5,8 +5,7 @@ from ga4gh.va_spec.base.core import ( Direction, EvidenceLine, - EvidenceLineType, - StatementType, + Statement, StudyResult, VariantPathogenicityProposition, ) @@ -38,7 +37,7 @@ def acmg_evidence_line( context: VariantAnnotationContext, score_calibration: ScoreCalibration, proposition: VariantPathogenicityProposition, - evidence: list[Union[StudyResult, EvidenceLineType, StatementType, iriReference]], + evidence: list[Union[StudyResult, EvidenceLine, Statement, iriReference]], ) -> VariantPathogenicityEvidenceLine: containing_evidence_range, evidence_outcome, evidence_strength = pathogenicity_classification_of_variant( context.variant, score_calibration @@ -63,7 +62,7 @@ def acmg_evidence_line( return VariantPathogenicityEvidenceLine( description=f"Pathogenicity evidence line for {context.variant.urn}.", hasEvidenceItems=list(evidence), - specifiedBy=pathogenicity_score_calibration_as_method(score_calibration, evidence_outcome), + specifiedBy=pathogenicity_score_calibration_as_method(score_calibration), evidenceOutcome={ "primaryCoding": Coding( code=evidence_outcome_code, @@ -93,18 +92,18 @@ def acmg_evidence_line( def functional_evidence_line( context: VariantAnnotationContext, score_calibration: ScoreCalibration, - evidence: list[Union[StudyResult, EvidenceLineType, StatementType, iriReference]], + evidence: list[Union[StudyResult, EvidenceLine, Statement, iriReference]], ) -> EvidenceLine: containing_evidence_range, classification = functional_classification_of_variant(context.variant, score_calibration) return EvidenceLine( description=f"Functional evidence line for {context.variant.urn}", - hasEvidenceItems=[StudyResult(root=item) for item in evidence], + hasEvidenceItems=list(evidence), directionOfEvidenceProvided=direction_of_support_for_functional_classification(classification), evidenceOutcome=MappableConcept( primaryCoding=Coding( code=classification.value, - system="ga4gh-gks-term:experimental-var-func-impact-classification", + system="ga4gh-gkm-term:experimental-var-func-impact-classification", ), ), specifiedBy=functional_score_calibration_as_method(score_calibration), diff --git a/src/mavedb/lib/annotation/method.py b/src/mavedb/lib/annotation/method.py index 1d7f0941a..f8d3801b1 100644 --- a/src/mavedb/lib/annotation/method.py +++ b/src/mavedb/lib/annotation/method.py @@ -143,12 +143,13 @@ def pathogenicity_score_calibration_as_iri(score_calibration: ScoreCalibration) return publication_as_iri(publication) if publication else None -def pathogenicity_score_calibration_as_method( - score_calibration: ScoreCalibration, method: Optional[VariantPathogenicityEvidenceLine.Criterion] -) -> Method: +def pathogenicity_score_calibration_as_method(score_calibration: ScoreCalibration) -> Method: """ Generate a [VA Method](https://va-ga4gh.readthedocs.io/en/latest/core-information-model/entities/information-entities/method.html#method) object for a generic calibration. + + A calibration assesses functional data, so its method type is always ``functional_data_assessment``, + the ACMG 2015 type that permits the PS3 and BS3 criteria its evidence lines carry. """ # TODO#XXX - in a future software version, it will be required that the method is populated with an IRI or equivalent. # Currently, we populate the method with an IRI if a publication with the appropriate relationship is found, @@ -157,7 +158,7 @@ def pathogenicity_score_calibration_as_method( return Method( name="Calibration method", reportedIn=pathogenicity_score_calibration_as_iri(score_calibration) or "Not Provided", - methodType=method.value if method else None, + methodType=VariantPathogenicityEvidenceLine.MethodType.FUNCTIONAL_DATA_ASSESSMENT.value, ) diff --git a/src/mavedb/lib/annotation/proposition.py b/src/mavedb/lib/annotation/proposition.py index 62d2645a4..440a6f331 100644 --- a/src/mavedb/lib/annotation/proposition.py +++ b/src/mavedb/lib/annotation/proposition.py @@ -74,9 +74,9 @@ def variant_pathogenicity_proposition( return VariantPathogenicityProposition( description=f"Variant pathogenicity proposition for {context.variant.urn}.", - subjectVariant=context.subject_variant, + subject=context.subject_variant, predicate="isCausalFor", - objectCondition=calibration_disease_condition(score_calibration), + object=calibration_disease_condition(score_calibration), geneContextQualifier=sequence_feature if system == "https://www.genenames.org/" else None, # only include gene context if we have a gene identifier @@ -93,8 +93,8 @@ def variant_functional_impact_proposition( return ExperimentalVariantFunctionalImpactProposition( description=f"Variant functional impact proposition for {context.variant.urn}.", - subjectVariant=context.subject_variant, + subject=context.subject_variant, predicate="impactsFunctionOf", - objectSequenceFeature=sequence_feature, + object=sequence_feature, experimentalContextQualifier=experiment_to_document(context.variant.score_set.experiment), ) diff --git a/src/mavedb/lib/annotation/statement.py b/src/mavedb/lib/annotation/statement.py index 7db3b05d8..d897ef20a 100644 --- a/src/mavedb/lib/annotation/statement.py +++ b/src/mavedb/lib/annotation/statement.py @@ -60,7 +60,7 @@ def functional_statement( classification=MappableConcept( primaryCoding=Coding( code=functional_classification, - system="ga4gh-gks-term:experimental-var-func-impact-classification", + system="ga4gh-gkm-term:experimental-var-func-impact-classification", ), ), hasEvidenceLines=[evidence_item for evidence_item in evidence], diff --git a/src/mavedb/lib/annotation/study_result.py b/src/mavedb/lib/annotation/study_result.py index c45457468..cdf4d160f 100644 --- a/src/mavedb/lib/annotation/study_result.py +++ b/src/mavedb/lib/annotation/study_result.py @@ -20,11 +20,11 @@ def variant_impact_study_result( context: VariantAnnotationContext, ) -> ExperimentalVariantFunctionalImpactStudyResult: # The study result's focus is the concrete measured allele — never a CategoricalVariant (VA-Spec - # narrows ``focusVariant`` to MolecularVariation). The context guarantees a hydratable post_mapped. + # narrows ``focus`` to MolecularVariation). The context guarantees a hydratable post_mapped. return ExperimentalVariantFunctionalImpactStudyResult( description=f"Variant effect study result for {context.variant.urn}.", # post_mapped is guaranteed non-null by variant_annotation_context (it returns None otherwise). - focusVariant=vrs_object_from_mapped_variant(context.measured_allele.post_mapped), # type: ignore[arg-type] + focus=vrs_object_from_mapped_variant(context.measured_allele.post_mapped), # type: ignore[arg-type] functionalImpactScore=variant_score(context.variant), specifiedBy=publication_identifiers_to_method(context.variant.score_set.publication_identifier_associations), sourceDataSet=score_set_to_data_set(context.variant.score_set), diff --git a/src/mavedb/lib/cat_vrs.py b/src/mavedb/lib/cat_vrs.py index 566184819..097403ccc 100644 --- a/src/mavedb/lib/cat_vrs.py +++ b/src/mavedb/lib/cat_vrs.py @@ -9,7 +9,13 @@ from enum import Enum from typing import Mapping, Optional -from ga4gh.cat_vrs.models import CategoricalVariant, DefiningAlleleConstraint, MappableConcept, Relation +from ga4gh.cat_vrs.models import CategoricalVariant, DefiningAlleleConstraint, MappableConcept +from ga4gh.cat_vrs.relations import ( + LIFTOVER_TO_RELATION, + TRANSCRIBED_TO_RELATION, + TRANSLATION_OF_RELATION, + Relation, +) from ga4gh.core.models import Coding, ConceptMapping, iriReference from ga4gh.core.models import Relation as MappingRelation from ga4gh.vrs.models import Allele as VrsAllele @@ -23,11 +29,8 @@ from mavedb.lib.term_systems import ( CLINGEN_ALLELE_REGISTRY, CLINVAR_VARIATION, - GKS_ALLELE_RELATION, GNOMAD, MAVEDB_CAT_VRS_RELATION, - SEQUENCE_ONTOLOGY, - TermSystem, ) from mavedb.lib.term_systems import coding as term_coding from mavedb.lib.vrs import vrs_object_from_mapped_variant @@ -37,17 +40,12 @@ logger = logging.getLogger(__name__) -# Cat-VRS does not attribute its own Relation values to one uniform system: translation_of and -# transcribed_to are Sequence Ontology terms, while liftover_to has no ontology equivalent and is -# carried on Cat-VRS's own internal vocabulary instead. Per-term, not a single constant, so that -# extending `_SPEC_EQUIVALENT` can't silently mislabel a future mapping with the wrong system. -# -# Sourced from the spec's published examples/schema (ga4gh/cat-vrs recipes-source.yaml and -# examples/json/proteinSequenceConsequence-ex2.json). -_SPEC_TERM_SYSTEM: dict[Relation, TermSystem] = { - Relation.TRANSLATION_OF: SEQUENCE_ONTOLOGY, - Relation.TRANSCRIBED_TO: SEQUENCE_ONTOLOGY, - Relation.LIFTOVER_TO: GKS_ALLELE_RELATION, +# The spec's own concept for each Cat-VRS relation term. Each carries its term's system: Sequence Ontology +# for translation_of and transcribed_to, Cat-VRS's internal ga4gh-gkm-term vocabulary for liftover_to. +_SPEC_RELATION_CONCEPTS: dict[Relation, MappableConcept] = { + Relation.TRANSLATION_OF: TRANSLATION_OF_RELATION, + Relation.TRANSCRIBED_TO: TRANSCRIBED_TO_RELATION, + Relation.LIFTOVER_TO: LIFTOVER_TO_RELATION, } @@ -182,7 +180,7 @@ def _relation_concept(relation: CatVrsRelation) -> MappableConcept: mappings=( [ ConceptMapping( - coding=term_coding(_SPEC_TERM_SYSTEM[spec_term], spec_term.value), + coding=_SPEC_RELATION_CONCEPTS[spec_term].primaryCoding, relation=MappingRelation.EXACT_MATCH, ) ] diff --git a/src/mavedb/lib/term_systems.py b/src/mavedb/lib/term_systems.py index 6f8028ca3..a5f826f53 100644 --- a/src/mavedb/lib/term_systems.py +++ b/src/mavedb/lib/term_systems.py @@ -19,16 +19,6 @@ ### External Term Systems -# Sequence Ontology. Used by several Cat-VRS Relation terms (translation_of, transcribed_to); see -# ga4gh/cat-vrs examples/json/proteinSequenceConsequence-ex2.json. -SEQUENCE_ONTOLOGY: TermSystem = ("http://www.sequenceontology.org", "so") - -# Cat-VRS's own internally controlled vocabulary for the allele-relation terms that have no external -# ontology equivalent (currently just liftover_to). See ga4gh/cat-vrs recipes-source.yaml. Other -# ga4gh-gks-term categories (e.g. experimental-var-func-impact-classification) are separate term -# systems under the same vocabulary owner, not this one. -GKS_ALLELE_RELATION: TermSystem = ("ga4gh-gks-term:allele-relation", "ga4gh-gks-term") - # External variant registries. Identifiers consumers match on, so deliberately not built from the registries' # link constants: a registry moving its web pages must not rename a system in served output. CLINVAR_VARIATION: TermSystem = ("https://www.ncbi.nlm.nih.gov/clinvar/variation/", "clinvar") diff --git a/src/mavedb/lib/vrs_utils.py b/src/mavedb/lib/vrs_utils.py index f96c30203..3ad9525f8 100644 --- a/src/mavedb/lib/vrs_utils.py +++ b/src/mavedb/lib/vrs_utils.py @@ -82,10 +82,11 @@ def translate_hgvs_to_vrs(hgvs: str, translator: AlleleTranslator) -> Allele: if hgvs.startswith("NC_") and ":c." in hgvs: hgvs = hgvs.replace(":c.", ":g.") - allele: Allele = translator.translate_from(hgvs, "hgvs", do_normalize=False) + allele = translator.translate_from(hgvs, "hgvs", do_normalize=False) if ( - not isinstance(allele.location, SequenceLocation) + not isinstance(allele, Allele) + or not isinstance(allele.location, SequenceLocation) or not isinstance(allele.location.start, int) or not isinstance(allele.location.end, int) or not isinstance(allele.state, LiteralSequenceExpression) diff --git a/src/mavedb/routers/score_sets.py b/src/mavedb/routers/score_sets.py index e8e97556c..f9635d891 100644 --- a/src/mavedb/routers/score_sets.py +++ b/src/mavedb/routers/score_sets.py @@ -29,6 +29,7 @@ variant_pathogenicity_statement, variant_study_result, ) +from mavedb.lib.annotation.conformance import serialize_annotation from mavedb.lib.annotation.context import variant_annotation_context from mavedb.lib.deprecation import MAPPED_VARIANT_SUNSET, deprecation_headers, record_deprecated_usage from mavedb.lib.annotation.exceptions import EXPECTED_ABSENCE_EXCEPTIONS @@ -1449,7 +1450,7 @@ def _annotation_stream_record( try: context = variant_annotation_context(db, variant, as_of=as_of) annotation = annotation_function(context) if context is not None else None - annotation_data = annotation.model_dump(exclude_none=True) if annotation else None + annotation_data = serialize_annotation(annotation) if annotation else None except EXPECTED_ABSENCE_EXCEPTIONS: logger.debug(f"Nothing to annotate for variant {variant_urn}.") return {"variant_urn": variant_urn, "annotation": None}, "unannotated" diff --git a/src/mavedb/routers/variants.py b/src/mavedb/routers/variants.py index fe0ce2001..890e4ea43 100644 --- a/src/mavedb/routers/variants.py +++ b/src/mavedb/routers/variants.py @@ -4,7 +4,8 @@ from fastapi import APIRouter, Depends, Path, Query, Response from fastapi.exceptions import HTTPException -from fastapi.responses import StreamingResponse +from fastapi.encoders import jsonable_encoder +from fastapi.responses import JSONResponse, StreamingResponse from ga4gh.core.identifiers import GA4GH_IR_REGEXP from ga4gh.va_spec.acmg_2015 import VariantPathogenicityStatement from ga4gh.va_spec.base.core import ExperimentalVariantFunctionalImpactStudyResult, Statement @@ -19,6 +20,7 @@ variant_pathogenicity_statement, variant_study_result, ) +from mavedb.lib.annotation.conformance import EmittedAnnotation, serialize_annotation from mavedb.lib.annotation.context import variant_annotation_context from mavedb.lib.annotation.exceptions import MappingDataDoesntExistException from mavedb.lib.authentication import get_current_user @@ -60,6 +62,18 @@ } +def _annotation_response(annotation: EmittedAnnotation, as_of: Optional[datetime]) -> JSONResponse: + """Emit a VA-Spec annotation with its nested evidence intact (see :func:`serialize_annotation`). + + Returning a response bypasses FastAPI's ``response_model`` serialization, which would drop the nested + evidence's subclass fields, so the ``X-As-Of`` header is set here rather than on the injected response. + """ + return JSONResponse( + content=jsonable_encoder(serialize_annotation(annotation, exclude_none=False)), + headers={"X-As-Of": as_of.isoformat() if as_of is not None else "current"}, + ) + + def _fetch_readable_variant(db: Session, user_data: Optional[UserData], urn: str) -> Variant: """Fetch a single variant by URN and assert the caller may read its score set. @@ -206,7 +220,6 @@ def get_variant( ) def get_variant_study_result( *, - response: Response, urn: str, as_of: Optional[datetime] = Query( default=None, @@ -219,10 +232,9 @@ def get_variant_study_result( ), db: Session = Depends(deps.get_db), user_data: Optional[UserData] = Depends(get_current_user), -) -> ExperimentalVariantFunctionalImpactStudyResult: +) -> JSONResponse: """Construct a single VA-Spec StudyResult for a variant by URN, from its mapping substrate.""" save_to_logging_context({"requested_resource": urn, "as_of": as_of}) - response.headers["X-As-Of"] = as_of.isoformat() if as_of is not None else "current" variant = _fetch_readable_variant(db, user_data, urn) @@ -233,7 +245,7 @@ def get_variant_study_result( ) try: - return variant_study_result(context) + return _annotation_response(variant_study_result(context), as_of) except MappingDataDoesntExistException as e: logger.info(msg=f"Could not construct a study result for variant {urn}: {e}", extra=logging_context()) raise HTTPException(status_code=404, detail=f"No study result exists for variant {urn}: {e}") @@ -248,7 +260,6 @@ def get_variant_study_result( ) def get_variant_functional_impact_statement( *, - response: Response, urn: str, as_of: Optional[datetime] = Query( default=None, @@ -262,10 +273,9 @@ def get_variant_functional_impact_statement( db: Session = Depends(deps.get_db), user_data: Optional[UserData] = Depends(get_current_user), principal: Principal = Depends(get_principal), -) -> Statement: +) -> JSONResponse: """Construct a single VA-Spec functional-impact Statement for a variant by URN.""" save_to_logging_context({"requested_resource": urn, "as_of": as_of}) - response.headers["X-As-Of"] = as_of.isoformat() if as_of is not None else "current" variant = _fetch_readable_variant(db, user_data, urn) @@ -297,7 +307,7 @@ def get_variant_functional_impact_statement( ), ) - return functional_impact + return _annotation_response(functional_impact, as_of) @router.get( @@ -309,7 +319,6 @@ def get_variant_functional_impact_statement( ) def get_variant_pathogenicity_statement( *, - response: Response, urn: str, as_of: Optional[datetime] = Query( default=None, @@ -323,10 +332,9 @@ def get_variant_pathogenicity_statement( db: Session = Depends(deps.get_db), user_data: Optional[UserData] = Depends(get_current_user), principal: Principal = Depends(get_principal), -) -> VariantPathogenicityStatement: +) -> JSONResponse: """Construct a single VA-Spec pathogenicity Statement for a variant by URN.""" save_to_logging_context({"requested_resource": urn, "as_of": as_of}) - response.headers["X-As-Of"] = as_of.isoformat() if as_of is not None else "current" variant = _fetch_readable_variant(db, user_data, urn) @@ -358,7 +366,7 @@ def get_variant_pathogenicity_statement( ), ) - return pathogenicity_statement + return _annotation_response(pathogenicity_statement, as_of) @router.get( diff --git a/src/mavedb/scripts/export_public_data.py b/src/mavedb/scripts/export_public_data.py index b4f7d86ad..fd9195b89 100644 --- a/src/mavedb/scripts/export_public_data.py +++ b/src/mavedb/scripts/export_public_data.py @@ -30,6 +30,7 @@ from sqlalchemy.orm import Session, joinedload, lazyload from mavedb.lib.annotation.annotate import variant_highest_level_annotation +from mavedb.lib.annotation.conformance import serialize_annotation from mavedb.lib.annotation.context import variant_annotation_context from mavedb.lib.allele_annotations import get_allele_cross_references from mavedb.lib.alleles import get_live_record_allele_links @@ -213,7 +214,7 @@ def va_ndjson(db: Session, score_set: ScoreSet, principal: Principal) -> str: annotation = variant_highest_level_annotation(context, principal=principal) if context is not None else None record = { "variant_urn": variant.urn, - "annotation": annotation.model_dump(exclude_none=True) if annotation else None, + "annotation": serialize_annotation(annotation) if annotation else None, } lines.append(json.dumps(record, default=str)) diff --git a/src/mavedb/worker/jobs/variant_processing/reverse_translation.py b/src/mavedb/worker/jobs/variant_processing/reverse_translation.py index ab20f3291..d243dc037 100644 --- a/src/mavedb/worker/jobs/variant_processing/reverse_translation.py +++ b/src/mavedb/worker/jobs/variant_processing/reverse_translation.py @@ -472,6 +472,8 @@ async def reverse_translate_variants_for_score_set( variation.model_dump(exclude_none=True), hgvs, seqrepo, subject=f"variant record {rec.id}" ) + # translate_hgvs_to_variation identifies every variation it returns. + assert variation.id is not None if variation.id in seen_digests: continue diff --git a/tests/helpers/constants.py b/tests/helpers/constants.py index bb05a1f10..c630c491a 100644 --- a/tests/helpers/constants.py +++ b/tests/helpers/constants.py @@ -1653,6 +1653,7 @@ SAVED_GENERIC_DISEASE_CONCEPT = { + "type": "MappableConcept", "conceptType": "Disease", "name": "disease or disorder", "primaryCoding": { diff --git a/tests/lib/annotation/test_annotate.py b/tests/lib/annotation/test_annotate.py index fba0957af..c7e2215ad 100644 --- a/tests/lib/annotation/test_annotate.py +++ b/tests/lib/annotation/test_annotate.py @@ -88,7 +88,7 @@ def test_valid_statement_creation(self, mock_annotation_context_with_functional_ assert result.type == "Statement" assert all(evidence_item.type == "EvidenceLine" for evidence_item in result.hasEvidenceLines) assert all( - study_result.root.type == "ExperimentalVariantFunctionalImpactStudyResult" + study_result.type == "ExperimentalVariantFunctionalImpactStudyResult" for evidence_line in [evidence_line for evidence_line in result.hasEvidenceLines] for study_result in evidence_line.hasEvidenceItems ) @@ -234,7 +234,7 @@ def test_valid_pathogenicity_statement_creation( assert all(s.type == "Statement" for s in statements) assert all(ei.type == "EvidenceLine" for ei in functional_evidence_lines) assert all( - study_result.root.type == "ExperimentalVariantFunctionalImpactStudyResult" + study_result.type == "ExperimentalVariantFunctionalImpactStudyResult" for evidence_item in functional_evidence_lines for study_result in evidence_item.hasEvidenceItems ) diff --git a/tests/lib/annotation/test_condition.py b/tests/lib/annotation/test_condition.py index 2619f1abd..0362628ce 100644 --- a/tests/lib/annotation/test_condition.py +++ b/tests/lib/annotation/test_condition.py @@ -37,13 +37,13 @@ def test_builds_condition_from_the_calibration_disease_term(self): ) condition = calibration_disease_condition(_calibration_with_disease(term)) assert isinstance(condition, Condition) - assert isinstance(condition.root, MappableConcept) - assert condition.root.conceptType == "Disease" - assert condition.root.name == "Brugada syndrome" + assert isinstance(condition, MappableConcept) + assert condition.conceptType == "Disease" + assert condition.name == "Brugada syndrome" def test_primary_coding_reflects_the_stored_term(self): term = MondoTerm(code="MONDO:0015263", system=MONDO_SYSTEM, label="Brugada syndrome") - coding = calibration_disease_condition(_calibration_with_disease(term)).root.primaryCoding + coding = calibration_disease_condition(_calibration_with_disease(term)).primaryCoding assert isinstance(coding, Coding) assert coding.code.root == "MONDO:0015263" assert coding.system == MONDO_SYSTEM @@ -55,5 +55,5 @@ def test_primary_coding_reflects_the_stored_term(self): def test_generic_disease_term_yields_the_mondo_root(self): term = MondoTerm(code=MONDO_GENERIC_CODE, system=MONDO_SYSTEM, label=MONDO_GENERIC_LABEL) condition = calibration_disease_condition(_calibration_with_disease(term)) - assert condition.root.primaryCoding.code.root == MONDO_GENERIC_CODE - assert condition.root.name == MONDO_GENERIC_LABEL + assert condition.primaryCoding.code.root == MONDO_GENERIC_CODE + assert condition.name == MONDO_GENERIC_LABEL diff --git a/tests/lib/annotation/test_method.py b/tests/lib/annotation/test_method.py index 2d3954d29..cf3bfb67e 100644 --- a/tests/lib/annotation/test_method.py +++ b/tests/lib/annotation/test_method.py @@ -230,15 +230,12 @@ def test_functional_score_calibration_as_method_falls_back_to_not_provided(self) assert method.reportedIn.root == "Not Provided" def test_pathogenicity_score_calibration_as_method_sets_method_type(self): - """Test pathogenicity calibration method includes criterion-based methodType.""" + """A calibration's method is a functional data assessment, the type that permits PS3 and BS3.""" score_calibration = Mock() score_calibration.publication_identifier_associations = [] - method = pathogenicity_score_calibration_as_method( - score_calibration, - VariantPathogenicityEvidenceLine.Criterion.PS3, - ) + method = pathogenicity_score_calibration_as_method(score_calibration) assert method.name == "Calibration method" assert method.reportedIn.root == "Not Provided" - assert method.methodType == VariantPathogenicityEvidenceLine.Criterion.PS3.value + assert method.methodType == VariantPathogenicityEvidenceLine.MethodType.FUNCTIONAL_DATA_ASSESSMENT.value diff --git a/tests/lib/annotation/test_proposition.py b/tests/lib/annotation/test_proposition.py index 7d3feb79d..9fa200f93 100644 --- a/tests/lib/annotation/test_proposition.py +++ b/tests/lib/annotation/test_proposition.py @@ -66,11 +66,11 @@ def test_variant_pathogenicity_proposition(self, mock_annotation_context): assert isinstance(result, VariantPathogenicityProposition) assert result.description == f"Variant pathogenicity proposition for {mock_annotation_context.variant.urn}." - assert isinstance(result.subjectVariant, MolecularVariation) + assert isinstance(result.subject, MolecularVariation) assert result.predicate == "isCausalFor" - assert result.objectCondition.root.conceptType == "Disease" - assert result.objectCondition.root.primaryCoding.code.root == "MONDO:0000001" - assert result.objectCondition.root.primaryCoding.system == "https://purl.obolibrary.org/obo/mondo.owl" + assert result.object.conceptType == "Disease" + assert result.object.primaryCoding.code.root == "MONDO:0000001" + assert result.object.primaryCoding.system == "https://purl.obolibrary.org/obo/mondo.owl" def test_clinical_impact_proposition_reflects_the_calibration_disease(self, mock_mapped_variant): """A calibration with a specific disease term drives the proposition's condition.""" @@ -79,15 +79,15 @@ def test_clinical_impact_proposition_reflects_the_calibration_disease(self, mock ) result = variant_pathogenicity_proposition(annotation_context_for(mock_mapped_variant), calibration) - assert result.objectCondition.root.primaryCoding.code.root == "MONDO:0015263" - assert result.objectCondition.root.name == "Brugada syndrome" + assert result.object.primaryCoding.code.root == "MONDO:0015263" + assert result.object.name == "Brugada syndrome" def test_clinical_impact_proposition_carries_a_categorical_subject(self, mock_mapped_variant): """A protein assay's categorical subject (Slice 5.1) is carried onto the proposition unchanged.""" context = annotation_context_for(mock_mapped_variant, subject_variant=_protein_categorical_variant()) result = variant_pathogenicity_proposition(context, create_mock_score_calibration()) - assert isinstance(result.subjectVariant, CategoricalVariant) + assert isinstance(result.subject, CategoricalVariant) @pytest.mark.unit @@ -100,10 +100,10 @@ def test_variant_functional_impact_proposition(self, mock_annotation_context): assert isinstance(result, ExperimentalVariantFunctionalImpactProposition) assert result.description == f"Variant functional impact proposition for {mock_annotation_context.variant.urn}." - assert isinstance(result.subjectVariant, MolecularVariation) + assert isinstance(result.subject, MolecularVariation) assert result.predicate == "impactsFunctionOf" - assert result.objectSequenceFeature.primaryCoding.code.root == "BRCA1" - assert result.objectSequenceFeature.primaryCoding.system == "https://www.genenames.org/" + assert result.object.primaryCoding.code.root == "BRCA1" + assert result.object.primaryCoding.system == "https://www.genenames.org/" assert result.experimentalContextQualifier is not None def test_functional_impact_proposition_carries_a_categorical_subject(self, mock_mapped_variant): @@ -111,7 +111,7 @@ def test_functional_impact_proposition_carries_a_categorical_subject(self, mock_ context = annotation_context_for(mock_mapped_variant, subject_variant=_protein_categorical_variant()) result = variant_functional_impact_proposition(context) - assert isinstance(result.subjectVariant, CategoricalVariant) + assert isinstance(result.subject, CategoricalVariant) @pytest.mark.unit diff --git a/tests/lib/annotation/test_statement.py b/tests/lib/annotation/test_statement.py index 7cd7e41fb..7612e4038 100644 --- a/tests/lib/annotation/test_statement.py +++ b/tests/lib/annotation/test_statement.py @@ -61,7 +61,7 @@ def test_functional_statement( assert result.direction == expected_direction assert result.classification.primaryCoding.code.root == classification.value assert ( - result.classification.primaryCoding.system == "ga4gh-gks-term:experimental-var-func-impact-classification" + result.classification.primaryCoding.system == "ga4gh-gkm-term:experimental-var-func-impact-classification" ) assert result.hasEvidenceLines assert len(result.hasEvidenceLines) == 1 diff --git a/tests/lib/annotation/test_study_result.py b/tests/lib/annotation/test_study_result.py index 356eaa1e2..12b3d8366 100644 --- a/tests/lib/annotation/test_study_result.py +++ b/tests/lib/annotation/test_study_result.py @@ -28,7 +28,7 @@ def test_variant_impact_study_result(self, mock_annotation_context): assert isinstance(result, ExperimentalVariantFunctionalImpactStudyResult) assert result.description == f"Variant effect study result for {mock_annotation_context.variant.urn}." - assert isinstance(result.focusVariant, MolecularVariation) + assert isinstance(result.focus, MolecularVariation) assert result.functionalImpactScore == mock_annotation_context.variant.data["score_data"]["score"] # Verify all expected contribution types are present contribution_types = {c.name for c in result.contributions} @@ -47,7 +47,7 @@ def test_no_clingen_allele_id_is_filtered_properly(self, mock_annotation_context assert isinstance(result, ExperimentalVariantFunctionalImpactStudyResult) assert result.description == f"Variant effect study result for {mock_annotation_context.variant.urn}." - assert isinstance(result.focusVariant, MolecularVariation) + assert isinstance(result.focus, MolecularVariation) assert result.functionalImpactScore == mock_annotation_context.variant.data["score_data"]["score"] # Verify all expected contribution types are present contribution_types = {c.name for c in result.contributions} diff --git a/tests/lib/test_cat_vrs.py b/tests/lib/test_cat_vrs.py index ba905cc53..041f14dce 100644 --- a/tests/lib/test_cat_vrs.py +++ b/tests/lib/test_cat_vrs.py @@ -10,7 +10,8 @@ pytest.importorskip("psycopg2") -from ga4gh.cat_vrs.models import CategoricalVariant, DefiningAlleleConstraint, Relation +from ga4gh.cat_vrs.models import CategoricalVariant, DefiningAlleleConstraint +from ga4gh.cat_vrs.relations import Relation from ga4gh.core.models import Relation as MappingRelation from mavedb.lib.cat_vrs import ( diff --git a/tests/routers/test_variant_annotation.py b/tests/routers/test_variant_annotation.py index 7a764ec3f..77d2b3393 100644 --- a/tests/routers/test_variant_annotation.py +++ b/tests/routers/test_variant_annotation.py @@ -121,6 +121,10 @@ def test_variant_functional_impact_statement( assert response.status_code == 200 assert response_data["description"] == f"Variant functional impact statement for {score_set['urn']}#1." + # The nested study result keeps its subclass fields, which serializing as the declared base type drops. + study_result = response_data["hasEvidenceLines"][0]["hasEvidenceItems"][0] + assert study_result["type"] == "ExperimentalVariantFunctionalImpactStudyResult" + assert "focus" in study_result Statement.model_validate_json(json.dumps(response_data)) From 7af2bc8a5de5dccb63c4f4c1f1fac0a1fa697a83 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Fri, 2 Oct 2026 11:26:57 -0700 Subject: [PATCH 198/252] feat(scripts): audit the mapping and reverse-translation allele graph Adds invariants, outcomes and round-trip commands. invariants checks the structural properties the serving layer assumes, including one refget per accession, stored refgets against SeqRepo, and ClinGen ids shared on one accession. outcomes rolls up the latest RT result per score set. round-trip re-derives sampled projection pairs with RT's coordinate mapper and compares VRS digests. --- .../scripts/audit_reverse_translation.py | 652 ++++++++++++++++++ .../scripts/test_audit_reverse_translation.py | 371 ++++++++++ 2 files changed, 1023 insertions(+) create mode 100644 src/mavedb/scripts/audit_reverse_translation.py create mode 100644 tests/scripts/test_audit_reverse_translation.py diff --git a/src/mavedb/scripts/audit_reverse_translation.py b/src/mavedb/scripts/audit_reverse_translation.py new file mode 100644 index 000000000..e0ad6fd54 --- /dev/null +++ b/src/mavedb/scripts/audit_reverse_translation.py @@ -0,0 +1,652 @@ +""" +Audit the allele graph that mapping and reverse translation (RT) write. + +Usage: +``` +python3 -m mavedb.scripts.audit_reverse_translation invariants [--score-set URN ...] [--skip-seqrepo] +python3 -m mavedb.scripts.audit_reverse_translation outcomes [--score-set URN ...] [--max-failed-pct 5] +python3 -m mavedb.scripts.audit_reverse_translation round-trip [--score-set URN ...] [--per-score-set 50] +``` + +Each check guards a property the serving layer assumes but no constraint enforces. The class of bug +this exists for is the NM_007294.3 refget divergence: the mapper and RT built the same variant on two +sequences, both digests were internally consistent, every existing check passed, and convergent alleles +silently read as projections. + +`invariants` runs the structural checks in SQL, plus a comparison of every stored refget against +SeqRepo. `outcomes` rolls the latest RT result per variant up by score set, so an outlier score set +stands out. `round-trip` re-translates a sample of projection pairs with the same coordinate mapper RT +uses and checks that each pair and its protein apex identify as the same VRS alleles RT stored. + +Each finding has a severity. `error` is a broken invariant; `warning` is a shape that is suspicious but +has legitimate causes, so read the samples; `info` is a count for context. `invariants` and `round-trip` +exit with status 1 when any error-level finding is non-zero. + +Read-only. Safe to run against production, though `round-trip` makes one data-provider call per +translation and is paced by it. +""" + +import json +import logging +import sys +from collections import Counter, defaultdict +from dataclasses import dataclass, field +from typing import Any, Callable, Optional, Sequence + +import asyncclick as click +from sqlalchemy import select, text +from sqlalchemy.orm import Session + +from mavedb.lib.hgvs import strip_protein_prediction_parens +from mavedb.lib.seqrepo import AmbiguousSequenceError, SequenceNotFoundError, resolve_refget +from mavedb.lib.vrs_utils import CANONICAL_ACCESSION_PREFIXES, identify_variation, translate_hgvs_to_variation +from mavedb.models.score_set import ScoreSet +from mavedb.scripts.environment import script_environment, with_database_session + +logger = logging.getLogger(__name__) + +ERROR = "error" +WARNING = "warning" +INFO = "info" + +REFGET_PATH = "{location,sequenceReference,refgetAccession}" + + +@dataclass +class Finding: + """The result of one check: how many rows violate it, and a few of them to read.""" + + name: str + severity: str + description: str + count: int + samples: list[dict[str, Any]] = field(default_factory=list) + + +@dataclass(frozen=True) +class SqlCheck: + """A check expressed as a query whose rows are the violations. + + ``sql`` may contain ``{allele_filter}``, ``{record_filter}`` or ``{link_filter}``, which expand to a + score-set restriction when the audit is scoped and to nothing otherwise. ``scoped=False`` marks a + check that has no meaning inside one score set (orphaned alleles belong to none) and is skipped then. + """ + + name: str + severity: str + description: str + sql: str + scoped: bool = True + + +SQL_CHECKS: tuple[SqlCheck, ...] = ( + SqlCheck( + name="accession_with_multiple_refgets", + severity=ERROR, + description="Alleles on one accession carry more than one refget, so the same variant can mint two digests.", + sql=f""" + SELECT split_part(coalesce(a.hgvs_g, a.hgvs_c, a.hgvs_p), ':', 1) AS accession, + array_agg(DISTINCT a.post_mapped #>> '{REFGET_PATH}') AS refgets, + count(*) AS alleles + FROM alleles a + WHERE a.post_mapped #>> '{REFGET_PATH}' IS NOT NULL {{allele_filter}} + GROUP BY 1 + HAVING count(DISTINCT a.post_mapped #>> '{REFGET_PATH}') > 1 + """, + ), + SqlCheck( + name="level_disagrees_with_hgvs", + severity=ERROR, + description="An allele's level disagrees with the HGVS column it fills or that expression's type.", + sql=""" + SELECT a.id AS allele_id, a.level, a.hgvs_g, a.hgvs_c, a.hgvs_p + FROM alleles a + WHERE (num_nonnulls(a.hgvs_g, a.hgvs_c, a.hgvs_p) <> 1 + OR NOT ((a.level = 'genomic' AND split_part(coalesce(a.hgvs_g, ''), ':', 2) LIKE 'g.%') + OR (a.level = 'cdna' AND split_part(coalesce(a.hgvs_c, ''), ':', 2) ~ '^[cn][.]') + OR (a.level = 'protein' AND split_part(coalesce(a.hgvs_p, ''), ':', 2) LIKE 'p.%'))) + {allele_filter} + """, + ), + SqlCheck( + name="accession_disagrees_with_level", + severity=WARNING, + description="An allele's accession is not the kind its level implies (NC_ genomic, NM_/ENST cdna, NP_/ENSP protein).", + sql=""" + SELECT a.id AS allele_id, a.level, coalesce(a.hgvs_g, a.hgvs_c, a.hgvs_p) AS hgvs + FROM alleles a + WHERE NOT ((a.level = 'genomic' AND coalesce(a.hgvs_g, '') ~ '^(NC|NG|NT|NW)_') + OR (a.level = 'cdna' AND coalesce(a.hgvs_c, '') ~ '^(NM_|NR_|XM_|XR_|ENST)') + OR (a.level = 'protein' AND coalesce(a.hgvs_p, '') ~ '^(NP_|XP_|ENSP)')) + {allele_filter} + """, + ), + SqlCheck( + name="clingen_allele_id_on_twin_alleles", + severity=ERROR, + description=( + "One ClinGen allele id sits on more than one allele on the same accession, so two writers built one " + "change differently. A CA/PA id spans reference sequences, so the same id across accessions is expected." + ), + sql=""" + SELECT a.clingen_allele_id, split_part(coalesce(a.hgvs_g, a.hgvs_c, a.hgvs_p), ':', 1) AS accession, + count(*) AS alleles, array_agg(coalesce(a.hgvs_g, a.hgvs_c, a.hgvs_p) ORDER BY a.id) AS hgvs + FROM alleles a + WHERE a.clingen_allele_id IS NOT NULL {allele_filter} + GROUP BY 1, 2 + HAVING count(*) > 1 + """, + ), + SqlCheck( + name="mapped_record_without_authoritative_link", + severity=ERROR, + description="A live mapping record whose latest mapping succeeded has no live authoritative allele.", + sql=""" + SELECT mr.id AS mapping_record_id, mr.variant_id, mr.score_set_id + FROM mapping_records mr + WHERE mr.valid_to IS NULL {record_filter} + AND NOT EXISTS (SELECT 1 FROM mapping_record_alleles l + WHERE l.mapping_record_id = mr.id AND l.is_authoritative AND l.valid_to IS NULL) + AND (SELECT e.disposition FROM annotation_event e + WHERE e.variant_id = mr.variant_id AND e.annotation_type = 'vrs_mapping' + ORDER BY e.id DESC LIMIT 1) = 'present' + """, + ), + SqlCheck( + name="live_link_on_retired_record", + severity=ERROR, + description="An allele link is live but its mapping record was retired; retiring a record retires its links.", + sql=""" + SELECT l.id AS link_id, l.mapping_record_id, l.allele_id, mr.valid_to AS record_retired_at + FROM mapping_record_alleles l + JOIN mapping_records mr ON mr.id = l.mapping_record_id + WHERE l.valid_to IS NULL AND mr.valid_to IS NOT NULL {link_filter} + """, + ), + SqlCheck( + name="measured_allele_outside_projection_group", + severity=ERROR, + description=( + "RT attached derived nucleotide alleles to a record, but the measured nucleotide allele is in no " + "projection group: the fold-in missed it, so its convergent siblings read as projections." + ), + sql=""" + SELECT auth.mapping_record_id, auth.allele_id, a.level, coalesce(a.hgvs_g, a.hgvs_c) AS hgvs + FROM mapping_record_alleles auth + JOIN alleles a ON a.id = auth.allele_id + WHERE auth.is_authoritative AND auth.valid_to IS NULL AND auth.projection_group IS NULL + AND a.level IN ('cdna', 'genomic') + {auth_filter} + AND EXISTS (SELECT 1 FROM mapping_record_alleles d + JOIN alleles da ON da.id = d.allele_id + WHERE d.mapping_record_id = auth.mapping_record_id + AND NOT d.is_authoritative AND d.valid_to IS NULL + AND da.level IN ('cdna', 'genomic')) + """, + ), + SqlCheck( + name="malformed_projection_group", + severity=WARNING, + description=( + "A live projection group is not exactly one cdna allele and one genomic allele. A member that " + "failed to translate leaves a group of one; read the samples." + ), + sql=""" + SELECT l.mapping_record_id, l.projection_group, count(*) AS members, + string_agg(a.level, ',' ORDER BY a.level) AS levels + FROM mapping_record_alleles l + JOIN alleles a ON a.id = l.allele_id + WHERE l.valid_to IS NULL AND l.projection_group IS NOT NULL {link_filter} + GROUP BY 1, 2 + HAVING string_agg(a.level, ',' ORDER BY a.level) <> 'cdna,genomic' + """, + ), + SqlCheck( + name="record_with_multiple_protein_alleles", + severity=WARNING, + description="A live mapping record links more than one protein allele; RT shares a single protein apex per record.", + sql=""" + SELECT l.mapping_record_id, count(*) AS protein_alleles, + array_agg(coalesce(a.hgvs_p, a.vrs_digest) ORDER BY a.id) AS hgvs + FROM mapping_record_alleles l + JOIN alleles a ON a.id = l.allele_id + WHERE l.valid_to IS NULL AND a.level = 'protein' {link_filter} + GROUP BY 1 + HAVING count(*) > 1 + """, + ), + SqlCheck( + name="mapped_record_never_reverse_translated", + severity=INFO, + description="A live record with an authoritative allele has no RT event; expected until RT has run on its score set.", + sql=""" + SELECT mr.id AS mapping_record_id, mr.variant_id, mr.score_set_id + FROM mapping_records mr + WHERE mr.valid_to IS NULL {record_filter} + AND EXISTS (SELECT 1 FROM mapping_record_alleles l + WHERE l.mapping_record_id = mr.id AND l.is_authoritative AND l.valid_to IS NULL) + AND NOT EXISTS (SELECT 1 FROM annotation_event e + WHERE e.variant_id = mr.variant_id AND e.annotation_type = 'cross_level_translation') + """, + ), + SqlCheck( + name="allele_without_live_link", + severity=INFO, + description="An allele no live mapping record links to. Remaps leave some behind; a large count means links were retired without replacement.", + sql=""" + SELECT a.id AS allele_id, a.level, coalesce(a.hgvs_g, a.hgvs_c, a.hgvs_p) AS hgvs + FROM alleles a + WHERE NOT EXISTS (SELECT 1 FROM mapping_record_alleles l WHERE l.allele_id = a.id AND l.valid_to IS NULL) + """, + scoped=False, + ), +) + + +def _filters(scoped: bool) -> dict[str, str]: + """Score-set restrictions for each alias a check's SQL can use, or empty strings when unscoped.""" + if not scoped: + return {"allele_filter": "", "record_filter": "", "link_filter": "", "auth_filter": ""} + + return { + "allele_filter": ( + "AND EXISTS (SELECT 1 FROM mapping_record_alleles sl " + "WHERE sl.allele_id = a.id AND sl.score_set_id = ANY(:score_set_ids))" + ), + "record_filter": "AND mr.score_set_id = ANY(:score_set_ids)", + "link_filter": "AND l.score_set_id = ANY(:score_set_ids)", + "auth_filter": "AND auth.score_set_id = ANY(:score_set_ids)", + } + + +def run_sql_check(db: Session, check: SqlCheck, score_set_ids: Optional[list[int]], samples: int) -> Finding: + """Count a check's violations and fetch the first few, optionally within some score sets.""" + # str.replace rather than str.format: the SQL holds JSON paths whose braces format would consume. + sql = check.sql + for placeholder, fragment in _filters(score_set_ids is not None).items(): + sql = sql.replace(f"{{{placeholder}}}", fragment) + params: dict[str, Any] = {"samples": samples} + if score_set_ids is not None: + params["score_set_ids"] = score_set_ids + + count = db.execute(text(f"SELECT count(*) FROM ({sql}) violations"), params).scalar_one() + rows = ( + db.execute(text(f"SELECT * FROM ({sql}) violations LIMIT :samples"), params).mappings().all() if count else [] + ) + return Finding(check.name, check.severity, check.description, count, [dict(row) for row in rows]) + + +def check_refgets_against_seqrepo( + db: Session, + resolve: Callable[[str], str], + score_set_ids: Optional[list[int]], + samples: int, +) -> list[Finding]: + """Compare every stored (accession, refget) pair with the refget SeqRepo holds for the accession. + + ``accession_with_multiple_refgets`` misses an accession whose alleles all agree on the wrong + sequence; this catches it. Only accessions with a canonical prefix are checked, matching the + insert-time guard in :func:`mavedb.lib.vrs_utils.verify_allele_refget`. + + ``resolve`` is :func:`mavedb.lib.seqrepo.resolve_refget` bound to a SeqRepo; it is a parameter so tests + need not open one. + """ + pairs_sql = f""" + SELECT split_part(coalesce(a.hgvs_g, a.hgvs_c, a.hgvs_p), ':', 1) AS accession, + a.post_mapped #>> '{REFGET_PATH}' AS refget, + count(*) AS alleles + FROM alleles a + WHERE a.post_mapped #>> '{REFGET_PATH}' IS NOT NULL {_filters(score_set_ids is not None)["allele_filter"]} + GROUP BY 1, 2 + """ + params = {"score_set_ids": score_set_ids} if score_set_ids is not None else {} + + mismatched: list[dict[str, Any]] = [] + unresolved: list[dict[str, Any]] = [] + expected_by_accession: dict[str, Optional[str]] = {} + + for accession, refget, alleles in db.execute(text(pairs_sql), params).all(): + if not accession.startswith(CANONICAL_ACCESSION_PREFIXES): + continue + + if accession not in expected_by_accession: + try: + expected_by_accession[accession] = resolve(accession) + except (SequenceNotFoundError, AmbiguousSequenceError) as e: + expected_by_accession[accession] = None + unresolved.append({"accession": accession, "error": str(e)}) + + expected = expected_by_accession[accession] + if expected is not None and refget != expected: + mismatched.append({"accession": accession, "stored": refget, "seqrepo": expected, "alleles": alleles}) + + return [ + Finding( + "refget_disagrees_with_seqrepo", + ERROR, + "Alleles sit on a refget other than the one SeqRepo holds for their accession.", + sum(row["alleles"] for row in mismatched), + mismatched[:samples], + ), + Finding( + "accession_unresolved_in_seqrepo", + WARNING, + "SeqRepo has no sequence, or more than one, for an accession alleles use.", + len(unresolved), + unresolved[:samples], + ), + ] + + +def report(findings: Sequence[Finding]) -> int: + """Log each finding with its samples and return how many error-level findings are non-zero.""" + for finding in findings: + status = "ok" if finding.count == 0 else finding.severity.upper() + logger.info("%-8s %10d %s", status, finding.count, finding.name) + if finding.count: + logger.info(" %s", finding.description) + for sample in finding.samples: + logger.info(" %s", json.dumps(sample, default=str)) + + return sum(1 for finding in findings if finding.severity == ERROR and finding.count) + + +def _score_set_ids(db: Session, urns: Sequence[str]) -> Optional[list[int]]: + """Resolve ``--score-set`` URNs to ids, or ``None`` when the audit is unscoped.""" + if not urns: + return None + + found: dict[str, int] = { + urn: id for urn, id in db.execute(select(ScoreSet.urn, ScoreSet.id).where(ScoreSet.urn.in_(urns))).all() if urn + } + missing = sorted(set(urns) - set(found)) + if missing: + raise click.BadParameter(f"No score set with URN: {', '.join(missing)}", param_hint="--score-set") + + return list(found.values()) + + +score_set_option = click.option( + "--score-set", "urns", multiple=True, help="Restrict the audit to this score set. Repeatable." +) + + +@script_environment.command() +@score_set_option +@click.option("--samples", type=int, default=5, show_default=True, help="Violating rows to print per check.") +@click.option("--skip-seqrepo", is_flag=True, help="Skip the refget comparison, which needs the SeqRepo mount.") +@with_database_session +def invariants(db: Session, urns: tuple[str, ...], samples: int, skip_seqrepo: bool) -> None: + """Check the structural invariants of the mapping and reverse-translation allele graph.""" + score_set_ids = _score_set_ids(db, urns) + + findings: list[Finding] = [] + for check in SQL_CHECKS: + if score_set_ids is not None and not check.scoped: + continue + logger.info("running %s", check.name) + findings.append(run_sql_check(db, check, score_set_ids, samples)) + + if not skip_seqrepo: + from mavedb.data_providers.services import seqrepo + + sr = seqrepo() + logger.info("running refget_disagrees_with_seqrepo") + findings.extend(check_refgets_against_seqrepo(db, lambda acc: resolve_refget(sr, acc), score_set_ids, samples)) + + if errors := report(findings): + logger.error("%d error-level check(s) failed.", errors) + sys.exit(1) + + +RT_OUTCOMES_SQL = """ + WITH latest AS ( + SELECT DISTINCT ON (e.variant_id) e.variant_id, e.score_set_id, e.disposition, e.reason + FROM annotation_event e + WHERE e.annotation_type = 'cross_level_translation' {event_filter} + ORDER BY e.variant_id, e.id DESC + ) + SELECT s.urn, latest.disposition, latest.reason, count(*) AS variants + FROM latest + JOIN scoresets s ON s.id = latest.score_set_id + GROUP BY 1, 2, 3 +""" + + +@dataclass +class ScoreSetOutcome: + """The latest RT result of every variant in one score set, by disposition and by failure reason.""" + + urn: str + by_disposition: Counter = field(default_factory=Counter) + failure_reasons: Counter = field(default_factory=Counter) + + @property + def total(self) -> int: + return sum(self.by_disposition.values()) + + @property + def failed_pct(self) -> float: + return 100.0 * self.by_disposition["failed"] / self.total if self.total else 0.0 + + +def collect_outcomes(rows: Sequence[tuple[str, str, str, int]]) -> list[ScoreSetOutcome]: + """Fold ``(urn, disposition, reason, variants)`` rows into one outcome per score set, worst first.""" + outcomes: dict[str, ScoreSetOutcome] = {} + for urn, disposition, reason, variants in rows: + outcome = outcomes.setdefault(urn, ScoreSetOutcome(urn)) + outcome.by_disposition[disposition] += variants + if disposition == "failed": + outcome.failure_reasons[reason] += variants + + return sorted(outcomes.values(), key=lambda o: (-o.failed_pct, o.urn)) + + +@script_environment.command() +@score_set_option +@click.option( + "--max-failed-pct", + type=float, + default=5.0, + show_default=True, + help="Flag score sets where more than this share of variants failed RT.", +) +@with_database_session +def outcomes(db: Session, urns: tuple[str, ...], max_failed_pct: float) -> None: + """Summarize the latest reverse-translation result per variant, by score set.""" + score_set_ids = _score_set_ids(db, urns) + event_filter = "AND e.score_set_id = ANY(:score_set_ids)" if score_set_ids is not None else "" + params = {"score_set_ids": score_set_ids} if score_set_ids is not None else {} + + rows = db.execute(text(RT_OUTCOMES_SQL.format(event_filter=event_filter)), params).all() + results = collect_outcomes([tuple(row) for row in rows]) # type: ignore[misc] + + logger.info("%-28s %9s %9s %9s %9s %s", "score set", "variants", "present", "n/a", "failed %", "failure reasons") + for outcome in results: + flag = " <-- above threshold" if outcome.failed_pct > max_failed_pct else "" + reasons = ", ".join(f"{reason} {n}" for reason, n in outcome.failure_reasons.most_common(3)) + logger.info( + "%-28s %9d %9d %9d %8.1f%% %s%s", + outcome.urn, + outcome.total, + outcome.by_disposition["present"], + outcome.by_disposition["not_applicable"], + outcome.failed_pct, + reasons, + flag, + ) + + flagged = [o.urn for o in results if o.failed_pct > max_failed_pct] + logger.info( + "%d score set(s) reverse translated; %d above %.1f%% failed.", len(results), len(flagged), max_failed_pct + ) + + +ROUND_TRIP_SQL = """ + WITH pairs AS ( + SELECT l.score_set_id, l.mapping_record_id, l.projection_group, + max(a.hgvs_c) FILTER (WHERE a.level = 'cdna') AS hgvs_c, + max(a.vrs_digest) FILTER (WHERE a.level = 'cdna') AS cdna_digest, + max(a.hgvs_g) FILTER (WHERE a.level = 'genomic') AS hgvs_g, + max(a.vrs_digest) FILTER (WHERE a.level = 'genomic') AS genomic_digest + FROM mapping_record_alleles l + JOIN alleles a ON a.id = l.allele_id + WHERE l.valid_to IS NULL AND l.projection_group IS NOT NULL {link_filter} + GROUP BY 1, 2, 3 + HAVING count(*) FILTER (WHERE a.level = 'cdna') = 1 AND count(*) FILTER (WHERE a.level = 'genomic') = 1 + ), + sampled AS ( + SELECT pairs.*, + row_number() OVER (PARTITION BY score_set_id ORDER BY mapping_record_id, projection_group) AS n + FROM pairs + ) + SELECT sampled.mapping_record_id, sampled.projection_group, sampled.hgvs_c, sampled.cdna_digest, + sampled.hgvs_g, sampled.genomic_digest, + apex.digests AS protein_digests, apex.hgvs AS protein_hgvs + FROM sampled + LEFT JOIN LATERAL ( + SELECT array_agg(pa.vrs_digest ORDER BY pa.id) AS digests, array_agg(pa.hgvs_p ORDER BY pa.id) AS hgvs + FROM mapping_record_alleles pl + JOIN alleles pa ON pa.id = pl.allele_id + WHERE pl.mapping_record_id = sampled.mapping_record_id AND pl.valid_to IS NULL AND pa.level = 'protein' + ) apex ON true + WHERE sampled.n <= :per_score_set + ORDER BY sampled.mapping_record_id, sampled.projection_group +""" + + +@dataclass(frozen=True) +class ProjectionPair: + """One live projection group as stored, plus the protein alleles live on its record.""" + + mapping_record_id: int + projection_group: int + hgvs_c: str + cdna_digest: str + hgvs_g: str + genomic_digest: str + protein_digests: Sequence[str] + protein_hgvs: Sequence[Optional[str]] + + +def round_trip_pair( + pair: ProjectionPair, + c_to_g: Callable[[str], str], + c_to_p: Callable[[str], str], + digest_of: Callable[[str], str], +) -> list[tuple[str, dict[str, Any]]]: + """Re-derive a pair's genomic member and protein apex from its cdna member and compare digests. + + Comparing digests rather than strings makes the check indifferent to how an expression is written + (3' shifting, prediction parentheses): two expressions for the same change identify to one VRS + allele. Returns ``(outcome, detail)`` tuples, one for the genomic side and one for the protein side. + + The callables are RT's coordinate mapper and HGVS-to-digest translation; parameters so tests can + supply fakes instead of a data provider and SeqRepo. + """ + results: list[tuple[str, dict[str, Any]]] = [] + base = { + "mapping_record_id": pair.mapping_record_id, + "projection_group": pair.projection_group, + "hgvs_c": pair.hgvs_c, + } + + try: + derived_g = c_to_g(pair.hgvs_c) + derived_g_digest = digest_of(derived_g) + except Exception as e: + results.append(("genomic: could not re-derive", {**base, "error": str(e)})) + else: + if derived_g_digest == pair.genomic_digest: + results.append(("genomic: matches", base)) + else: + results.append( + ( + "GENOMIC: DISAGREES", + {**base, "stored": pair.hgvs_g, "derived": derived_g, "derived_digest": derived_g_digest}, + ) + ) + + digests = [d for d in pair.protein_digests if d is not None] + if not digests: + results.append(("protein: no apex on record", base)) + return results + + try: + derived_p = strip_protein_prediction_parens(c_to_p(pair.hgvs_c)) + derived_p_digest = digest_of(derived_p) + except Exception as e: + results.append(("protein: could not re-derive", {**base, "error": str(e)})) + return results + + if derived_p_digest in digests: + results.append(("protein: matches", base)) + else: + results.append( + ( + "PROTEIN: DISAGREES", + {**base, "stored": list(pair.protein_hgvs), "derived": derived_p, "derived_digest": derived_p_digest}, + ) + ) + + return results + + +@script_environment.command() +@score_set_option +@click.option( + "--per-score-set", + type=int, + default=50, + show_default=True, + help="Projection pairs to re-translate in each score set.", +) +@click.option("--samples", type=int, default=10, show_default=True, help="Rows to print per disagreeing outcome.") +@with_database_session +def round_trip(db: Session, urns: tuple[str, ...], per_score_set: int, samples: int) -> None: + """Re-translate sampled projection pairs and check they identify as the alleles RT stored.""" + from ga4gh.vrs.extras.translator import AlleleTranslator + + from mavedb.data_providers.services import cdot_rest, seqrepo_data_proxy + from mavedb.worker.lib.translation_ports import WorkerCoordinateTranslator + + score_set_ids = _score_set_ids(db, urns) + link_filter = "AND l.score_set_id = ANY(:score_set_ids)" if score_set_ids is not None else "" + params: dict[str, Any] = {"per_score_set": per_score_set} + if score_set_ids is not None: + params["score_set_ids"] = score_set_ids + + coordinates = WorkerCoordinateTranslator(cdot_rest()) + translator = AlleleTranslator(seqrepo_data_proxy()) + + def digest_of(hgvs: str) -> str: + return identify_variation(translate_hgvs_to_variation(hgvs, translator)) + + counts: Counter[str] = Counter() + examples: dict[str, list[dict[str, Any]]] = defaultdict(list) + rows = db.execute(text(ROUND_TRIP_SQL.format(link_filter=link_filter)), params).mappings().all() + logger.info("re-translating %d projection pair(s)", len(rows)) + + for row in rows: + pair = ProjectionPair( + **{**row, "protein_digests": row["protein_digests"] or [], "protein_hgvs": row["protein_hgvs"] or []} + ) + for outcome, detail in round_trip_pair(pair, coordinates.c_to_g, coordinates.c_to_p, digest_of): + counts[outcome] += 1 + if len(examples[outcome]) < samples: + examples[outcome].append(detail) + + for outcome, count in sorted(counts.items()): + logger.info("%10d %s", count, outcome) + if not outcome.endswith("matches"): + for detail in examples[outcome]: + logger.info(" %s", json.dumps(detail, default=str)) + + disagreements = counts["GENOMIC: DISAGREES"] + counts["PROTEIN: DISAGREES"] + if disagreements: + logger.error("%d re-translation(s) disagree with the stored alleles.", disagreements) + sys.exit(1) + + +if __name__ == "__main__": + script_environment() diff --git a/tests/scripts/test_audit_reverse_translation.py b/tests/scripts/test_audit_reverse_translation.py new file mode 100644 index 000000000..9c85aebe5 --- /dev/null +++ b/tests/scripts/test_audit_reverse_translation.py @@ -0,0 +1,371 @@ +# ruff: noqa: E402 + +from datetime import datetime, timezone + +import pytest + +pytest.importorskip("ga4gh.vrs") + +from mavedb.lib.seqrepo import SequenceNotFoundError +from mavedb.models.annotation_event import AnnotationEvent +from mavedb.models.variant import Variant +from mavedb.scripts.audit_reverse_translation import ( + ERROR, + SQL_CHECKS, + ProjectionPair, + check_refgets_against_seqrepo, + collect_outcomes, + round_trip_pair, + run_sql_check, +) +from tests.helpers.util.annotation import AlleleSpec, seed_mapping_record + +CHECKS = {check.name: check for check in SQL_CHECKS} + + +def _post_mapped(refget: str) -> dict: + return { + "type": "Allele", + "location": {"type": "SequenceLocation", "sequenceReference": {"refgetAccession": refget}}, + } + + +def _score_set_with_variants(session, make_score_set, urn, n): + score_set = make_score_set() + score_set.urn = urn + session.commit() + + variants = [ + Variant(urn=f"{urn}#{i}", score_set_id=score_set.id, hgvs_nt=f"c.{i}A>G", data={}) for i in range(1, n + 1) + ] + session.add_all(variants) + session.commit() + return score_set, variants + + +def _healthy_record(session, variant, n): + """A measured cdna allele folded into its projection pair, with the genomic twin and a protein apex.""" + return seed_mapping_record( + session, + variant, + alleles=[ + AlleleSpec( + digest=f"c-{n}", + level="cdna", + is_authoritative=True, + hgvs_c=f"NM_000001.1:c.{n}A>G", + clingen_allele_id=f"CA{n}", + projection_group=1, + post_mapped=_post_mapped("SQ.cdna"), + ), + AlleleSpec( + digest=f"g-{n}", + level="genomic", + hgvs_g=f"NC_000001.11:g.{1000 + n}A>G", + clingen_allele_id=f"CA{n}", + projection_group=1, + post_mapped=_post_mapped("SQ.genome"), + ), + AlleleSpec( + digest=f"p-{n}", + level="protein", + hgvs_p=f"NP_000001.1:p.Lys{n}Glu", + post_mapped=_post_mapped("SQ.protein"), + ), + ], + ) + + +def _count(session, name, score_set_ids): + return run_sql_check(session, CHECKS[name], score_set_ids, samples=5) + + +@pytest.mark.integration +class TestSqlChecks: + def test_a_healthy_graph_passes_every_error_check(self, session, make_score_set): + score_set, variants = _score_set_with_variants(session, make_score_set, "urn:mavedb:00000001-a-1", 2) + for n, variant in enumerate(variants, start=1): + _healthy_record(session, variant, n) + + failing = { + check.name: finding.count + for check in SQL_CHECKS + if check.severity == ERROR and (finding := run_sql_check(session, check, [score_set.id], 5)).count + } + + assert failing == {} + + def test_flags_a_measured_allele_left_outside_its_projection_group(self, session, make_score_set): + score_set, (variant,) = _score_set_with_variants(session, make_score_set, "urn:mavedb:00000002-a-1", 1) + seed_mapping_record( + session, + variant, + alleles=[ + AlleleSpec(digest="measured", level="cdna", is_authoritative=True, hgvs_c="NM_000002.1:c.5A>G"), + AlleleSpec(digest="rt-c", level="cdna", hgvs_c="NM_000002.2:c.5A>G", projection_group=1), + AlleleSpec(digest="rt-g", level="genomic", hgvs_g="NC_000002.12:g.50A>G", projection_group=1), + ], + ) + + finding = _count(session, "measured_allele_outside_projection_group", [score_set.id]) + + assert finding.count == 1 + assert finding.samples[0]["hgvs"] == "NM_000002.1:c.5A>G" + + def test_flags_an_accession_whose_alleles_carry_two_refgets(self, session, make_score_set): + score_set, variants = _score_set_with_variants(session, make_score_set, "urn:mavedb:00000003-a-1", 2) + for n, (variant, refget) in enumerate(zip(variants, ("SQ.ncbi", "SQ.cdot")), start=1): + seed_mapping_record( + session, + variant, + alleles=[ + AlleleSpec( + digest=f"twin-{n}", + level="cdna", + is_authoritative=True, + hgvs_c=f"NM_007294.3:c.{n}A>G", + post_mapped=_post_mapped(refget), + ) + ], + ) + + finding = _count(session, "accession_with_multiple_refgets", [score_set.id]) + + assert finding.count == 1 + assert finding.samples[0]["accession"] == "NM_007294.3" + assert sorted(finding.samples[0]["refgets"]) == ["SQ.cdot", "SQ.ncbi"] + + @pytest.mark.parametrize( + "hgvs, flagged", + [ + (("NP_000004.1:p.Lys1Glu", "NP_000004.1:p.Lys1Glu="), 1), + (("NP_000004.1:p.Lys1Glu", "NP_000004.2:p.Lys1Glu"), 0), + ], + ids=["same accession", "different accessions"], + ) + def test_flags_one_clingen_allele_id_only_on_twins_on_one_accession(self, session, make_score_set, hgvs, flagged): + score_set, variants = _score_set_with_variants(session, make_score_set, "urn:mavedb:00000004-a-1", 2) + for n, (variant, expression) in enumerate(zip(variants, hgvs), start=1): + seed_mapping_record( + session, + variant, + alleles=[ + AlleleSpec( + digest=f"caid-{n}", + level="protein", + is_authoritative=True, + hgvs_p=expression, + clingen_allele_id="PA999", + ) + ], + ) + + assert _count(session, "clingen_allele_id_on_twin_alleles", [score_set.id]).count == flagged + + def test_flags_a_projection_group_with_one_member(self, session, make_score_set): + score_set, (variant,) = _score_set_with_variants(session, make_score_set, "urn:mavedb:00000005-a-1", 1) + seed_mapping_record( + session, + variant, + alleles=[ + AlleleSpec( + digest="lonely", + level="cdna", + is_authoritative=True, + hgvs_c="NM_000005.1:c.1A>G", + projection_group=1, + ) + ], + ) + + finding = _count(session, "malformed_projection_group", [score_set.id]) + + assert finding.count == 1 + assert finding.samples[0]["levels"] == "cdna" + + def test_flags_a_level_that_disagrees_with_the_hgvs(self, session, make_score_set): + score_set, (variant,) = _score_set_with_variants(session, make_score_set, "urn:mavedb:00000006-a-1", 1) + seed_mapping_record( + session, + variant, + alleles=[AlleleSpec(digest="mislevel", level="cdna", is_authoritative=True, hgvs_g="NC_000006.12:g.1A>G")], + ) + + assert _count(session, "level_disagrees_with_hgvs", [score_set.id]).count == 1 + + def test_flags_a_live_link_on_a_retired_record(self, session, make_score_set): + score_set, (variant,) = _score_set_with_variants(session, make_score_set, "urn:mavedb:00000007-a-1", 1) + record = _healthy_record(session, variant, 7) + record.valid_to = datetime.now(timezone.utc) + session.commit() + + assert _count(session, "live_link_on_retired_record", [score_set.id]).count == 3 + + def test_flags_a_mapped_record_with_no_authoritative_allele(self, session, make_score_set): + score_set, (variant,) = _score_set_with_variants(session, make_score_set, "urn:mavedb:00000008-a-1", 1) + seed_mapping_record(session, variant, alleles=[]) + session.add( + AnnotationEvent( + annotation_type="vrs_mapping", + variant_id=variant.id, + score_set_id=score_set.id, + disposition="present", + reason="mapped", + ) + ) + session.commit() + + assert _count(session, "mapped_record_without_authoritative_link", [score_set.id]).count == 1 + + def test_does_not_flag_a_record_whose_mapping_failed(self, session, make_score_set): + score_set, (variant,) = _score_set_with_variants(session, make_score_set, "urn:mavedb:00000009-a-1", 1) + seed_mapping_record(session, variant, alleles=[]) + session.add( + AnnotationEvent( + annotation_type="vrs_mapping", + variant_id=variant.id, + score_set_id=score_set.id, + disposition="failed", + reason="failed", + ) + ) + session.commit() + + assert _count(session, "mapped_record_without_authoritative_link", [score_set.id]).count == 0 + + def test_scoping_excludes_other_score_sets(self, session, make_score_set): + clean, (clean_variant,) = _score_set_with_variants(session, make_score_set, "urn:mavedb:00000010-a-1", 1) + _, (broken_variant,) = _score_set_with_variants(session, make_score_set, "urn:mavedb:00000011-a-1", 1) + _healthy_record(session, clean_variant, 10) + seed_mapping_record( + session, + broken_variant, + alleles=[ + AlleleSpec( + digest="b-lonely", + level="cdna", + is_authoritative=True, + hgvs_c="NM_000011.1:c.1A>G", + projection_group=1, + ) + ], + ) + + assert _count(session, "malformed_projection_group", [clean.id]).count == 0 + assert _count(session, "malformed_projection_group", None).count == 1 + + +@pytest.mark.integration +class TestRefgetsAgainstSeqRepo: + def test_counts_alleles_off_the_canonical_refget_and_lists_unresolved_accessions(self, session, make_score_set): + score_set, variants = _score_set_with_variants(session, make_score_set, "urn:mavedb:00000012-a-1", 3) + specs = [ + ("NM_000012.1:c.1A>G", "SQ.right"), + ("NM_000012.1:c.2A>G", "SQ.wrong"), + ("NM_999999.1:c.1A>G", "SQ.unknown"), + ] + for n, (variant, (hgvs, refget)) in enumerate(zip(variants, specs), start=1): + seed_mapping_record( + session, + variant, + alleles=[ + AlleleSpec( + digest=f"sr-{n}", + level="cdna", + is_authoritative=True, + hgvs_c=hgvs, + post_mapped=_post_mapped(refget), + ) + ], + ) + + def resolve(accession): + if accession == "NM_000012.1": + return "SQ.right" + raise SequenceNotFoundError(accession) + + mismatch, unresolved = check_refgets_against_seqrepo(session, resolve, [score_set.id], samples=5) + + assert mismatch.count == 1 + assert mismatch.samples == [ + {"accession": "NM_000012.1", "stored": "SQ.wrong", "seqrepo": "SQ.right", "alleles": 1} + ] + assert unresolved.count == 1 + assert unresolved.samples[0]["accession"] == "NM_999999.1" + + +def _pair(**overrides): + values = dict( + mapping_record_id=1, + projection_group=1, + hgvs_c="NM_1.1:c.1A>G", + cdna_digest="c", + hgvs_g="NC_1.11:g.100A>G", + genomic_digest="g", + protein_digests=["p"], + protein_hgvs=["NP_1.1:p.Lys1Glu"], + ) + return ProjectionPair(**{**values, **overrides}) + + +DIGESTS = {"NC_1.11:g.100A>G": "g", "NP_1.1:p.Lys1Glu": "p", "NC_1.11:g.200A>G": "g-other"} + + +class TestRoundTripPair: + def test_reports_matches_when_both_sides_identify_as_the_stored_alleles(self): + outcomes = round_trip_pair( + _pair(), lambda c: "NC_1.11:g.100A>G", lambda c: "NP_1.1:p.(Lys1Glu)", DIGESTS.__getitem__ + ) + + assert [outcome for outcome, _ in outcomes] == ["genomic: matches", "protein: matches"] + + def test_reports_a_genomic_projection_that_identifies_differently(self): + outcomes = round_trip_pair( + _pair(), lambda c: "NC_1.11:g.200A>G", lambda c: "NP_1.1:p.Lys1Glu", DIGESTS.__getitem__ + ) + + assert outcomes[0][0] == "GENOMIC: DISAGREES" + assert outcomes[0][1]["derived"] == "NC_1.11:g.200A>G" + + def test_reports_a_protein_consequence_that_identifies_differently(self): + outcomes = round_trip_pair( + _pair(protein_digests=["p-other"]), + lambda c: "NC_1.11:g.100A>G", + lambda c: "NP_1.1:p.Lys1Glu", + DIGESTS.__getitem__, + ) + + assert outcomes[1][0] == "PROTEIN: DISAGREES" + + def test_separates_translation_failures_from_disagreements(self): + def fail(_): + raise ValueError("intronic") + + outcomes = round_trip_pair(_pair(), fail, fail, DIGESTS.__getitem__) + + assert [outcome for outcome, _ in outcomes] == ["genomic: could not re-derive", "protein: could not re-derive"] + + def test_reports_a_record_with_no_protein_apex(self): + outcomes = round_trip_pair( + _pair(protein_digests=[], protein_hgvs=[]), lambda c: "NC_1.11:g.100A>G", lambda c: "", DIGESTS.__getitem__ + ) + + assert outcomes[1][0] == "protein: no apex on record" + + +class TestCollectOutcomes: + def test_orders_score_sets_by_failure_share_and_tallies_failure_reasons(self): + outcomes = collect_outcomes( + [ + ("urn:a", "present", "translated", 90), + ("urn:a", "failed", "translation_error", 10), + ("urn:b", "present", "translated", 50), + ("urn:b", "failed", "translation_failed", 30), + ("urn:b", "failed", "translation_error", 20), + ] + ) + + assert [o.urn for o in outcomes] == ["urn:b", "urn:a"] + assert outcomes[0].failed_pct == 50.0 + assert outcomes[0].failure_reasons.most_common(1) == [("translation_failed", 30)] + assert outcomes[1].total == 100 From cf63681e683a5c648acb20dc1b62c8db4546de57 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Fri, 2 Oct 2026 11:26:58 -0700 Subject: [PATCH 199/252] fix(clingen): replace stored CAIDs CAR assigns differently on forced re-registration Legacy CAIDs from ClinGen LDH linkage were assigned by variant URN, so after a remap some name another allele, and the allele backfill copied them onto alleles. With force_reregister, a CAID CAR returns for the allele's own HGVS now replaces a different stored one, recorded as corrected, and the allele's gnomAD and ClinVar links are retired so the pipeline links it again. A CAR response that does not list the submitted HGVS is rejected rather than trusted by position. Job definitions gain optional_params so a pipeline can set force_reregister, exposed as run_score_set_pipelines --force-reregister. --- src/mavedb/lib/types/workflow.py | 1 + src/mavedb/lib/workflow/definitions.py | 1 + src/mavedb/lib/workflow/job_factory.py | 5 + src/mavedb/models/enums/event_reason.py | 2 +- src/mavedb/scripts/run_score_set_pipelines.py | 15 +- .../worker/jobs/external_services/clingen.py | 84 +++++++-- tests/lib/workflow/test_job_factory.py | 19 ++ .../jobs/external_services/test_clingen.py | 164 ++++++++++++++---- 8 files changed, 244 insertions(+), 47 deletions(-) diff --git a/src/mavedb/lib/types/workflow.py b/src/mavedb/lib/types/workflow.py index 459d4337d..215adbf14 100644 --- a/src/mavedb/lib/types/workflow.py +++ b/src/mavedb/lib/types/workflow.py @@ -75,6 +75,7 @@ class JobDefinition(TypedDict): type: str function: str params: dict[str, Any] + optional_params: NotRequired[dict[str, Any]] dependencies: list[tuple[str, DependencyType]] retry_delay_seconds: NotRequired[int] diff --git a/src/mavedb/lib/workflow/definitions.py b/src/mavedb/lib/workflow/definitions.py index 4b7b1e178..e4cf5139e 100644 --- a/src/mavedb/lib/workflow/definitions.py +++ b/src/mavedb/lib/workflow/definitions.py @@ -39,6 +39,7 @@ def annotation_pipeline_job_definitions( "score_set_id": None, # Required param to be filled in at runtime "updater_id": None, # Required param to be filled in at runtime }, + "optional_params": {"force_reregister": False}, # Resubmit alleles that already hold a CAID "dependencies": [("reverse_translate_variants_for_score_set", DependencyType.SUCCESS_REQUIRED)], }, { diff --git a/src/mavedb/lib/workflow/job_factory.py b/src/mavedb/lib/workflow/job_factory.py index 151cc3b57..4d6c4b243 100644 --- a/src/mavedb/lib/workflow/job_factory.py +++ b/src/mavedb/lib/workflow/job_factory.py @@ -51,6 +51,11 @@ def create_job_run( raise ValueError(f"Missing required param: {key}") job_params[key] = pipeline_params[key] + # Optional params take the pipeline's value when given and their declared default otherwise. + # Preset params above are fixed by the definition and never overridden. + for key, default in job_def.get("optional_params", {}).items(): + job_params[key] = pipeline_params.get(key, default) + job_run = JobRun( job_type=job_def["type"], job_function=job_def["function"], diff --git a/src/mavedb/models/enums/event_reason.py b/src/mavedb/models/enums/event_reason.py index 3bcb5ee5c..503bc6b11 100644 --- a/src/mavedb/models/enums/event_reason.py +++ b/src/mavedb/models/enums/event_reason.py @@ -25,6 +25,7 @@ class EventReason(str, Enum): CREATED = "created" # linked/registered this run (gnomAD, ClinVar, CAR) PREEXISTING = "preexisting" # already held before this run (gnomAD, ClinVar, CAR) RECONFIRMED = "reconfirmed" # re-verified unchanged (gnomAD, CAR force) + CORRECTED = "corrected" # replaced a stored value the source now assigns differently (CAR force) SKIPPED = "skipped" # version-skip: already current at this source version (gnomAD, VEP) SUPERSEDED = "superseded" # re-resolved within a release, newest wins (ClinVar) SUBMITTED = "submitted" # LDH @@ -52,7 +53,6 @@ class EventReason(str, Enum): API_ERROR = "api_error" # network/timeout/upstream error (ClinVar, VEP, LDH, CAR no-response, RT UTA outage) SERVICE_REJECTED = "service_rejected" # external service refused the input (CAR) MALFORMED_RESPONSE = "malformed_response" # unparseable/contractless response (CAR) - CAID_CONFLICT = "caid_conflict" # returned identifier conflicts with the stored one (CAR) TRANSLATION_FAILED = "translation_failed" # all candidate HGVS failed translation (RT) TRANSLATION_ERROR = "translation_error" # the translation engine errored (RT) TRANSCRIPT_UNRESOLVED = "transcript_unresolved" # protein-coding target with no resolvable transcript (RT) diff --git a/src/mavedb/scripts/run_score_set_pipelines.py b/src/mavedb/scripts/run_score_set_pipelines.py index 221ee675f..c5872dbf9 100644 --- a/src/mavedb/scripts/run_score_set_pipelines.py +++ b/src/mavedb/scripts/run_score_set_pipelines.py @@ -34,6 +34,11 @@ poetry run python -m mavedb.scripts.run_score_set_pipelines map_annotate_score_set \\ --phase caid --collection-urn urn:mavedb:collection-0000001 + # Re-register every allele with CAR while mapping and annotating; a CAID CAR now assigns + # differently replaces the stored one, and its gnomAD/ClinVar links are rebuilt in the same pipeline. + poetry run python -m mavedb.scripts.run_score_set_pipelines map_annotate_score_set \\ + --published-only --concurrency 4 --force-reregister + # Run one gene you already know, no cohort files involved. poetry run python -m mavedb.scripts.run_score_set_pipelines map_annotate_score_set \\ --gene BRCA1 --concurrency 4 @@ -459,7 +464,7 @@ async def enqueue_pipeline( custom_pipeline: Optional[tuple[str, PipelineDefinition]], score_set: ScoreSet, user: User, - extra_params: tuple[tuple[str, str], ...], + extra_params: tuple[tuple[str, object], ...], ) -> EnqueueOutcome: """run_pipeline.py's enqueue body, generalized to accept either a base pipeline name or a resolved (name, job-subset) pair. A redis.enqueue_job failure triggers @@ -709,6 +714,11 @@ def render_report( help="Write one URN per line for every cohort score set whose latest pipeline is FAILED/PARTIAL.", ) @click.option("--updater-id", type=int, default=None, help="ID of the user to attribute pipeline actions to.") +@click.option( + "--force-reregister", + is_flag=True, + help="Resubmit alleles that already hold a CAID to CAR, replacing any CAID CAR now assigns differently.", +) @click.option( "--extra-param", "extra_params", @@ -733,6 +743,7 @@ async def main( cohort_out: Optional[str], failure_out: Optional[str], updater_id: Optional[int], + force_reregister: bool, extra_params: tuple[tuple[str, str], ...], ) -> None: """Bulk-drive PIPELINE_NAME across a cohort of score sets. Use --list to see available pipelines.""" @@ -891,7 +902,7 @@ async def main( custom_pipeline=custom_pipeline, score_set=score_set, user=user, - extra_params=extra_params, + extra_params=extra_params + ((("force_reregister", True),) if force_reregister else ()), ) prefix = " [enqueued]" if outcome.ok else " [failed] " diff --git a/src/mavedb/worker/jobs/external_services/clingen.py b/src/mavedb/worker/jobs/external_services/clingen.py index d563e658c..1efd1a2ad 100644 --- a/src/mavedb/worker/jobs/external_services/clingen.py +++ b/src/mavedb/worker/jobs/external_services/clingen.py @@ -15,6 +15,7 @@ from dataclasses import dataclass from sqlalchemy import select +from sqlalchemy.orm import Session from mavedb.lib.annotation_status_manager import AnnotationStatusManager from mavedb.lib.clingen.alleles import get_alleles_for_score_set @@ -30,14 +31,17 @@ ClinGenAlleleRegistryService, ClinGenLdhService, ) +from mavedb.lib.types.clingen import ClinGenAllele from mavedb.lib.types.workflow import JobExecutionOutcome from mavedb.lib.utils import batched from mavedb.lib.variants import get_hgvs_from_post_mapped from mavedb.models.allele import Allele as AlleleModel +from mavedb.models.clinvar_allele_link import ClinvarAlleleLink from mavedb.models.enums.annotation_type import AnnotationType from mavedb.models.enums.disposition import Disposition from mavedb.models.enums.event_reason import EventReason from mavedb.models.enums.job_pipeline import FailureCategory +from mavedb.models.gnomad_allele_link import GnomadAlleleLink from mavedb.models.mapping_record import MappingRecord from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.score_set import ScoreSet @@ -55,6 +59,27 @@ class _AlleleEntry: existing_caid: str | None +def _response_hgvs(response: ClinGenAllele) -> set[str]: + """Every HGVS expression a CAR allele response lists, across genomic, transcript and protein alleles.""" + definitions = [ + *response.get("genomicAlleles", []), + *response.get("transcriptAlleles", []), + *response.get("aminoAcidAlleles", []), + ] + return {hgvs for definition in definitions for hgvs in definition.get("hgvs", [])} + + +def _retire_caid_keyed_links(db: Session, allele_id: int) -> tuple[int, int]: + """Retire an allele's live gnomAD and ClinVar links, which were matched through its CAID. + + Both linkers skip an allele only while it holds a live link, so retiring them makes the next + gnomAD and ClinVar runs link the allele again under its corrected CAID. + """ + gnomad = db.execute(GnomadAlleleLink.retire_live_where(GnomadAlleleLink.allele_id == allele_id)) + clinvar = db.execute(ClinvarAlleleLink.retire_live_where(ClinvarAlleleLink.allele_id == allele_id)) + return gnomad.rowcount, clinvar.rowcount # type: ignore[attr-defined] + + def _annotate_caid( annotation_manager: AnnotationStatusManager, allele_id: int, @@ -94,6 +119,11 @@ async def submit_score_set_mappings_to_car(ctx: dict, job_id: int, job_manager: - score_set_id (int): ID of the ScoreSet to process - correlation_id (str): Correlation ID for tracking + Optional job_params: + - force_reregister (bool): resubmit alleles that already hold a CAID. Where CAR assigns a different + one, it replaces the stored CAID (recorded as ``corrected``) and the allele's gnomAD and ClinVar + links are retired. + Args: ctx (dict): Worker context containing DB and Redis connections job_manager (JobManager): Manager for job lifecycle and DB operations @@ -101,6 +131,7 @@ async def submit_score_set_mappings_to_car(ctx: dict, job_id: int, job_manager: Side Effects: - Updates Allele records with ClinGen Allele IDs - Submits data to ClinGen Allele Registry + - Retires the gnomAD and ClinVar links of an allele whose CAID is corrected Returns: JobExecutionOutcome: outcome with per-allele counts (submitted/registered/already-registered/failed). @@ -113,7 +144,9 @@ async def submit_score_set_mappings_to_car(ctx: dict, job_id: int, job_manager: score_set = job_manager.db.scalars(select(ScoreSet).where(ScoreSet.id == job.job_params["score_set_id"])).one() # type: ignore correlation_id = job.job_params["correlation_id"] # type: ignore - force_reregister = bool(job.job_params.get("force_reregister", False)) # type: ignore[union-attr] + # A pipeline param may arrive as a string from --extra-param, where bool("false") would be True. + raw_force = job.job_params.get("force_reregister", False) # type: ignore[union-attr] + force_reregister = raw_force is True or str(raw_force).lower() == "true" job_manager.save_to_context( { @@ -368,33 +401,56 @@ def _outcome_data() -> dict[str, int]: continue caid = caid_iri.split("/")[-1] + + # The positional zip is CAR's contract, but nothing else ties a response to its submission. + # An allele that does not list the submitted expression cannot be trusted for these alleles. + if hgvs_string not in _response_hgvs(response): + logger.error( + msg=f"CAR returned {caid!r} for HGVS '{hgvs_string}', but that allele does not list it.", + extra=job_manager.logging_context(), + ) + for allele_id in allele_ids_for_hgvs: + failed_allele_ids.add(allele_id) + _annotate_caid( + annotation_manager, + allele_id, + Disposition.FAILED, + EventReason.MALFORMED_RESPONSE, + error_message="ClinGen Allele Registry returned an allele that does not include the submitted HGVS.", + metadata={"submitted_hgvs": hgvs_string, "returned_caid": caid}, + ) + + continue + for allele_id in allele_ids_for_hgvs: entry = allele_data[allele_id] prior_caid = entry.existing_caid - # TODO#780 - CAID is immutable — a different value returned by CAR is a hard invariant - # violation. Do not overwrite; record a failure with full audit context. + # Only force_reregister resubmits an allele that already holds a CAID, and CAR's answer for the + # allele's own HGVS wins. Its gnomAD and ClinVar links were matched through the old CAID, so retire + # them and the later steps link the allele again. if prior_caid and prior_caid != caid: - logger.error( + linked_allele_ids.add(allele_id) + alleles_by_id[allele_id].clingen_allele_id = caid + retired_gnomad, retired_clinvar = _retire_caid_keyed_links(job_manager.db, allele_id) + logger.warning( msg=( - f"CAR returned a different CAID for allele {allele_id}: " - f"stored={prior_caid!r}, returned={caid!r}. " - "Not overwriting. Investigate immediately." + f"Replaced CAID {prior_caid!r} with {caid!r} for allele {allele_id}; retired " + f"{retired_gnomad} gnomAD and {retired_clinvar} ClinVar link(s)." ), extra=job_manager.logging_context(), ) - - failed_allele_ids.add(allele_id) _annotate_caid( annotation_manager, allele_id, - Disposition.FAILED, - EventReason.CAID_CONFLICT, - error_message="CAR returned a CAID that conflicts with the stored value.", + Disposition.PRESENT, + EventReason.CORRECTED, metadata={ - "clingen_allele_id": prior_caid, - "conflicting_caid": caid, + "clingen_allele_id": caid, + "replaced_caid": prior_caid, "submitted_hgvs": hgvs_string, + "retired_gnomad_links": retired_gnomad, + "retired_clinvar_links": retired_clinvar, }, ) diff --git a/tests/lib/workflow/test_job_factory.py b/tests/lib/workflow/test_job_factory.py index 4ea1b5d0b..a00360697 100644 --- a/tests/lib/workflow/test_job_factory.py +++ b/tests/lib/workflow/test_job_factory.py @@ -26,6 +26,25 @@ def test_create_job_run_persists_preset_params_from_definition(self, job_factory assert job_run.job_params["param1"] == "value1" assert job_run.job_params["param2"] == "value2" + @pytest.mark.parametrize( + "pipeline_value, expected", + [({}, False), ({"force": True}, True)], + ids=["default", "set by pipeline"], + ) + def test_create_job_run_takes_optional_params_from_the_pipeline_or_their_default( + self, job_factory, sample_job_definition, pipeline_value, expected + ): + job_def = {**sample_job_definition, "optional_params": {"force": False}} + job_run = job_factory.create_job_run( + job_def=job_def, + correlation_id="test-correlation-id", + pipeline_params={"required_param": "required_value", "param1": "override", **pipeline_value}, + pipeline_id=1, + ) + + assert job_run.job_params["force"] is expected + assert job_run.job_params["param1"] == "value1" + def test_create_job_run_raises_error_for_missing_params(self, job_factory, sample_job_definition): incomplete_params = {"param1": "new_value1"} # Missing param2 diff --git a/tests/worker/jobs/external_services/test_clingen.py b/tests/worker/jobs/external_services/test_clingen.py index 0281545ae..52205a325 100644 --- a/tests/worker/jobs/external_services/test_clingen.py +++ b/tests/worker/jobs/external_services/test_clingen.py @@ -13,6 +13,10 @@ from mavedb.lib.types.workflow import JobExecutionOutcome from mavedb.lib.variants import get_hgvs_from_post_mapped from mavedb.models.allele import Allele +from mavedb.models.clinical_control import ClinvarControl +from mavedb.models.clinvar_allele_link import ClinvarAlleleLink +from mavedb.models.gnomad_allele_link import GnomadAlleleLink +from mavedb.models.gnomad_variant import GnomADVariant from mavedb.models.enums.job_pipeline import JobStatus, PipelineStatus from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.variant import Variant @@ -25,6 +29,16 @@ from tests.helpers.constants import TEST_CLINGEN_LDH_LINKING_RESPONSE_BAD_REQUEST from tests.helpers.util.setup.worker import create_mappings_in_score_set + +def _car_response(caid, allele): + """A CAR registration response for ``allele``, listing its submitted HGVS as CAR does.""" + return { + "@id": caid, + "type": "nucleotide", + "genomicAlleles": [{"hgvs": [get_hgvs_from_post_mapped(allele.post_mapped)]}], + } + + pytestmark = pytest.mark.usefixtures("patch_db_session_ctxmgr") @@ -283,7 +297,10 @@ async def test_submit_score_set_mappings_to_car_event_per_allele( session.flush() def fake_dispatch(hgvs_list): - return [{"@id": f"CA{idx}", "type": "nucleotide", "genomicAlleles": []} for idx, _ in enumerate(hgvs_list)] + return [ + {"@id": f"CA{idx}", "type": "nucleotide", "genomicAlleles": [{"hgvs": [hgvs]}]} + for idx, hgvs in enumerate(hgvs_list) + ] with ( patch( @@ -465,7 +482,7 @@ async def test_submit_score_set_mappings_to_car_repeated_hgvs( { "@id": "CA_DUPLICATE", "type": "nucleotide", - "genomicAlleles": [{"hgvs": get_hgvs_from_post_mapped(alleles[0].post_mapped)}], + "genomicAlleles": [{"hgvs": [get_hgvs_from_post_mapped(alleles[0].post_mapped)]}], } ] @@ -532,7 +549,7 @@ async def test_submit_score_set_mappings_to_car_partial_failure( { "@id": f"CA{alleles[0].id}", "type": "nucleotide", - "genomicAlleles": [{"hgvs": get_hgvs_from_post_mapped(alleles[0].post_mapped)}], + "genomicAlleles": [{"hgvs": [get_hgvs_from_post_mapped(alleles[0].post_mapped)]}], } ] @@ -696,7 +713,7 @@ async def test_submit_score_set_mappings_to_car_success( { "@id": f"CA{alleles[0].id}", "type": "nucleotide", - "genomicAlleles": [{"hgvs": get_hgvs_from_post_mapped(alleles[0].post_mapped)}], + "genomicAlleles": [{"hgvs": [get_hgvs_from_post_mapped(alleles[0].post_mapped)]}], } ] @@ -822,7 +839,13 @@ async def test_submit_score_set_mappings_to_car_force_reregister_same_caid( } session.flush() - registered_alleles_mock = [{"@id": "CA_CONFIRMED", "type": "nucleotide", "genomicAlleles": []}] + registered_alleles_mock = [ + { + "@id": "CA_CONFIRMED", + "type": "nucleotide", + "genomicAlleles": [{"hgvs": [get_hgvs_from_post_mapped(allele.post_mapped)]}], + } + ] with ( patch( @@ -850,7 +873,7 @@ async def test_submit_score_set_mappings_to_car_force_reregister_same_caid( assert event.disposition == "present" assert event.reason == "reconfirmed" - async def test_submit_score_set_mappings_to_car_force_reregister_caid_conflict( + async def test_submit_score_set_mappings_to_car_force_reregister_replaces_a_caid_car_assigns_differently( self, mock_worker_ctx, session, @@ -863,7 +886,7 @@ async def test_submit_score_set_mappings_to_car_force_reregister_caid_conflict( dummy_variant_creation_job_run, dummy_variant_mapping_job_run, ): - """Force re-registration returning a different CAID fails without overwriting the stored CAID.""" + """Under force, CAR's CAID replaces a different stored one, and the links the old one keyed are retired.""" await create_mappings_in_score_set( session, mock_s3_client, @@ -875,7 +898,31 @@ async def test_submit_score_set_mappings_to_car_force_reregister_caid_conflict( ) allele = session.scalars(select(Allele)).first() - allele.clingen_allele_id = "CA_STORED" + allele.clingen_allele_id = "CA_BACKFILLED" + gnomad_variant = GnomADVariant( + db_name="gnomAD", + db_identifier="1-12345-G-A", + db_version="v4.1", + allele_count=1, + allele_number=2, + allele_frequency=0.5, + ) + clinvar_control = ClinvarControl( + gene_symbol="BRCA1", + clinical_significance="Pathogenic", + clinical_review_status="reviewed by expert panel", + db_name="ClinVar", + db_identifier="12345", + db_version="01_2026", + ) + session.add_all([gnomad_variant, clinvar_control]) + session.flush() + session.add_all( + [ + GnomadAlleleLink(allele_id=allele.id, gnomad_variant_id=gnomad_variant.id), + ClinvarAlleleLink(allele_id=allele.id, clinvar_control_id=clinvar_control.id), + ] + ) session.flush() submit_score_set_mappings_to_car_sample_job_run.job_params = { @@ -884,13 +931,73 @@ async def test_submit_score_set_mappings_to_car_force_reregister_caid_conflict( } session.flush() - # CAR returns a DIFFERENT CAID — this is an invariant violation - registered_alleles_mock = [{"@id": "CA_DIFFERENT", "type": "nucleotide", "genomicAlleles": []}] + with ( + patch( + "mavedb.worker.jobs.external_services.clingen.ClinGenAlleleRegistryService.dispatch_submissions", + return_value=[_car_response("CA_FROM_CAR", allele)], + ), + patch("mavedb.worker.jobs.external_services.clingen.CAR_SUBMISSION_ENDPOINT", "http://fake-endpoint"), + patch("mavedb.worker.jobs.external_services.clingen.CLIN_GEN_SUBMISSION_ENABLED", True), + ): + result = await submit_score_set_mappings_to_car( + mock_worker_ctx, + submit_score_set_mappings_to_car_sample_job_run.id, + JobManager(session, mock_worker_ctx["redis"], submit_score_set_mappings_to_car_sample_job_run.id), + ) + + assert result.status == JobStatus.SUCCEEDED + session.refresh(allele) + assert allele.clingen_allele_id == "CA_FROM_CAR" + + live_gnomad = session.scalars( + select(GnomadAlleleLink).where(GnomadAlleleLink.allele_id == allele.id, GnomadAlleleLink.current) + ).all() + live_clinvar = session.scalars( + select(ClinvarAlleleLink).where(ClinvarAlleleLink.allele_id == allele.id, ClinvarAlleleLink.current) + ).all() + assert live_gnomad == [] and live_clinvar == [] + + corrected = session.scalars( + select(AnnotationEvent).where( + AnnotationEvent.annotation_type == "clingen_allele_id", AnnotationEvent.reason == "corrected" + ) + ).one() + assert corrected.disposition == "present" + assert corrected.event_metadata["replaced_caid"] == "CA_BACKFILLED" + assert corrected.event_metadata["clingen_allele_id"] == "CA_FROM_CAR" + assert corrected.event_metadata["retired_gnomad_links"] == 1 + assert corrected.event_metadata["retired_clinvar_links"] == 1 + + async def test_submit_score_set_mappings_to_car_rejects_a_response_without_the_submitted_hgvs( + self, + mock_worker_ctx, + session, + with_submit_score_set_mappings_to_car_job, + submit_score_set_mappings_to_car_sample_job_run, + mock_s3_client, + sample_score_dataframe, + sample_count_dataframe, + with_dummy_setup_jobs, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ): + """A response for some other allele is not written to the submitted one, however the counts line up.""" + await create_mappings_in_score_set( + session, + mock_s3_client, + mock_worker_ctx, + sample_score_dataframe, + sample_count_dataframe, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ) + allele = session.scalars(select(Allele)).first() + other_allele = {"@id": "CA_OTHER", "type": "nucleotide", "genomicAlleles": [{"hgvs": ["NC_000001.11:g.1A>T"]}]} with ( patch( "mavedb.worker.jobs.external_services.clingen.ClinGenAlleleRegistryService.dispatch_submissions", - return_value=registered_alleles_mock, + return_value=[other_allele], ), patch("mavedb.worker.jobs.external_services.clingen.CAR_SUBMISSION_ENDPOINT", "http://fake-endpoint"), patch("mavedb.worker.jobs.external_services.clingen.CLIN_GEN_SUBMISSION_ENABLED", True), @@ -901,22 +1008,15 @@ async def test_submit_score_set_mappings_to_car_force_reregister_caid_conflict( JobManager(session, mock_worker_ctx["redis"], submit_score_set_mappings_to_car_sample_job_run.id), ) - # Job fails because all CAID-returning submissions had a conflict assert result.status == JobStatus.FAILED - - # CAID must NOT have been overwritten session.refresh(allele) - assert allele.clingen_allele_id == "CA_STORED" + assert allele.clingen_allele_id is None - events = session.scalars( + event = session.scalars( select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") - ).all() - assert len(events) == 1 - for event in events: - assert event.disposition == "failed" - assert event.reason == "caid_conflict" - assert event.event_metadata["clingen_allele_id"] == "CA_STORED" - assert event.event_metadata["conflicting_caid"] == "CA_DIFFERENT" + ).one() + assert (event.disposition, event.reason) == ("failed", "malformed_response") + assert event.event_metadata["returned_caid"] == "CA_OTHER" async def _add_derived_alleles_with_distinct_hgvs(self, session, count): """Attach ``count`` extra alleles to the score set, each carrying a distinct HGVS so the CAR @@ -980,7 +1080,9 @@ async def test_submit_score_set_mappings_to_car_batches_submissions( def fake_dispatch(hgvs_list): dispatched_batches.append(list(hgvs_list)) - return [{"@id": f"CA/{hgvs}", "type": "nucleotide", "genomicAlleles": []} for hgvs in hgvs_list] + return [ + {"@id": f"CA/{hgvs}", "type": "nucleotide", "genomicAlleles": [{"hgvs": [hgvs]}]} for hgvs in hgvs_list + ] with ( patch( @@ -1050,7 +1152,9 @@ def fake_dispatch(hgvs_list): # register normally. if call_count["n"] == 1: return [] - return [{"@id": f"CA/{hgvs}", "type": "nucleotide", "genomicAlleles": []} for hgvs in hgvs_list] + return [ + {"@id": f"CA/{hgvs}", "type": "nucleotide", "genomicAlleles": [{"hgvs": [hgvs]}]} for hgvs in hgvs_list + ] with ( patch( @@ -1120,7 +1224,7 @@ async def test_submit_score_set_mappings_to_car_independent_ctx( { "@id": f"CA{alleles[0].id}", "type": "nucleotide", - "genomicAlleles": [{"hgvs": get_hgvs_from_post_mapped(alleles[0].post_mapped)}], + "genomicAlleles": [{"hgvs": [get_hgvs_from_post_mapped(alleles[0].post_mapped)]}], } ] @@ -1187,7 +1291,7 @@ async def test_submit_score_set_mappings_to_car_pipeline_ctx( { "@id": f"CA{alleles[0].id}", "type": "nucleotide", - "genomicAlleles": [{"hgvs": get_hgvs_from_post_mapped(alleles[0].post_mapped)}], + "genomicAlleles": [{"hgvs": [get_hgvs_from_post_mapped(alleles[0].post_mapped)]}], } ] @@ -1481,7 +1585,7 @@ async def test_submit_score_set_mappings_to_car_partial_failure( { "@id": f"CA{alleles[0].id}", "type": "nucleotide", - "genomicAlleles": [{"hgvs": get_hgvs_from_post_mapped(alleles[0].post_mapped)}], + "genomicAlleles": [{"hgvs": [get_hgvs_from_post_mapped(alleles[0].post_mapped)]}], } ] @@ -1675,7 +1779,7 @@ async def test_submit_score_set_mappings_to_car_with_arq_context_independent( { "@id": f"CA{alleles[0].id}", "type": "nucleotide", - "genomicAlleles": [{"hgvs": get_hgvs_from_post_mapped(alleles[0].post_mapped)}], + "genomicAlleles": [{"hgvs": [get_hgvs_from_post_mapped(alleles[0].post_mapped)]}], } ] @@ -1743,7 +1847,7 @@ async def test_submit_score_set_mappings_to_car_with_arq_context_pipeline( { "@id": f"CA{alleles[0].id}", "type": "nucleotide", - "genomicAlleles": [{"hgvs": get_hgvs_from_post_mapped(alleles[0].post_mapped)}], + "genomicAlleles": [{"hgvs": [get_hgvs_from_post_mapped(alleles[0].post_mapped)]}], } ] From efa802b4ea55bcc127a4b88961b221cc16bb81c9 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Fri, 2 Oct 2026 13:30:22 -0700 Subject: [PATCH 200/252] chore(deps): update variant-annotation to v0.1.0-dev.3 Reverse translation now projects a coding delins to a single genomic delins instead of a cis-phased set, so the measured allele of a codon-level score set folds into its projection group and the genomic candidates can be registered with ClinGen. --- poetry.lock | 8 ++++---- pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index 027b54a21..77a6c9bd1 100644 --- a/poetry.lock +++ b/poetry.lock @@ -4937,7 +4937,7 @@ test = ["aiohttp (>=3.10.5)", "flake8 (>=6.1,<7.0)", "mypy (>=0.800)", "psutil", [[package]] name = "variant-annotation" -version = "0.1.0-dev.2" +version = "0.1.0-dev.3" description = "Genetic variant mapping and annotation pipeline" optional = true python-versions = ">=3.11" @@ -4967,8 +4967,8 @@ tests = ["pytest"] [package.source] type = "git" url = "https://github.com/bbi-lab/variant-annotation.git" -reference = "v0.1.0-dev.2" -resolved_reference = "073b7aa07c44fad1059725733ff8b11ac0dd2d4b" +reference = "v0.1.0-dev.3" +resolved_reference = "fd1e2c300189762ac87d39df23739aff935965ff" [[package]] name = "variant-translation" @@ -5301,4 +5301,4 @@ server = ["aiocache", "alembic", "alembic-utils", "arq", "authlib", "biocommons" [metadata] lock-version = "2.1" python-versions = "^3.11" -content-hash = "764312324738b85692022685484ea6dd33b82eb7d456aeb677b7a95b52b8e230" +content-hash = "a6c52d53587f9c6592bd7738cd45281d6c68378d25a1b926f74d2eae9671a2c8" diff --git a/pyproject.toml b/pyproject.toml index c99be8981..c4058bb9b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,7 +65,7 @@ starlette = { version = "^1.7.0", optional = true } starlette-context = { version = "^0.3.6", optional = true } slack-sdk = { version = "~3.21.3", optional = true } uvicorn = { extras = ["standard"], version = "*", optional = true } -variant-annotation = { git = "https://github.com/bbi-lab/variant-annotation.git", tag = "v0.1.0-dev.2", optional = true } +variant-annotation = { git = "https://github.com/bbi-lab/variant-annotation.git", tag = "v0.1.0-dev.3", optional = true } watchtower = { version = "~3.2.0", optional = true } asyncclick = "^8.3.0.7" filelock = "^3.29.0" From 04055f5a178992c601ce149260785bfb0f40e020 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Fri, 2 Oct 2026 14:04:53 -0700 Subject: [PATCH 201/252] fix(worker): rewrite reverse-translation inversions as delins Implements variant-annotation v0.1.0-dev.4's c_to_g_literal on the worker's coordinate translator. HGVS normalization writes a delins whose inserted bases are the reverse complement of the reference as inv, which VRS cannot translate, so reverse translation dropped those genomic candidates. The library now recovers the delins from an unnormalized projection. --- poetry.lock | 8 ++++---- pyproject.toml | 2 +- src/mavedb/worker/lib/translation_ports.py | 13 +++++++++++++ 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index 77a6c9bd1..70caf8f0e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -4937,7 +4937,7 @@ test = ["aiohttp (>=3.10.5)", "flake8 (>=6.1,<7.0)", "mypy (>=0.800)", "psutil", [[package]] name = "variant-annotation" -version = "0.1.0-dev.3" +version = "0.1.0-dev.4" description = "Genetic variant mapping and annotation pipeline" optional = true python-versions = ">=3.11" @@ -4967,8 +4967,8 @@ tests = ["pytest"] [package.source] type = "git" url = "https://github.com/bbi-lab/variant-annotation.git" -reference = "v0.1.0-dev.3" -resolved_reference = "fd1e2c300189762ac87d39df23739aff935965ff" +reference = "v0.1.0-dev.4" +resolved_reference = "1a7bc7e4daaeb7cb602c538bbccf9b9cf4043c63" [[package]] name = "variant-translation" @@ -5301,4 +5301,4 @@ server = ["aiocache", "alembic", "alembic-utils", "arq", "authlib", "biocommons" [metadata] lock-version = "2.1" python-versions = "^3.11" -content-hash = "a6c52d53587f9c6592bd7738cd45281d6c68378d25a1b926f74d2eae9671a2c8" +content-hash = "6488da5d58e664eb98f1c93f7118865e2dda724662f7be6b7ca715dc623632db" diff --git a/pyproject.toml b/pyproject.toml index c4058bb9b..17d4e0fc5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,7 +65,7 @@ starlette = { version = "^1.7.0", optional = true } starlette-context = { version = "^0.3.6", optional = true } slack-sdk = { version = "~3.21.3", optional = true } uvicorn = { extras = ["standard"], version = "*", optional = true } -variant-annotation = { git = "https://github.com/bbi-lab/variant-annotation.git", tag = "v0.1.0-dev.3", optional = true } +variant-annotation = { git = "https://github.com/bbi-lab/variant-annotation.git", tag = "v0.1.0-dev.4", optional = true } watchtower = { version = "~3.2.0", optional = true } asyncclick = "^8.3.0.7" filelock = "^3.29.0" diff --git a/src/mavedb/worker/lib/translation_ports.py b/src/mavedb/worker/lib/translation_ports.py index 1fd34809f..c1b1ba0da 100644 --- a/src/mavedb/worker/lib/translation_ports.py +++ b/src/mavedb/worker/lib/translation_ports.py @@ -43,6 +43,7 @@ def __init__(self, hdp: Any) -> None: self._hdp = hdp self._parser: Any = None self._mapper: Any = None + self._literal_mapper: Any = None def _ensure_initialized(self) -> None: if self._mapper is None: @@ -55,6 +56,13 @@ def _ensure_initialized(self) -> None: assembly_name="GRCh38", alt_aln_method="splign", ) + # Unnormalized, so a delins stays a delins rather than becoming ``inv`` (see c_to_g_literal). + self._literal_mapper = hgvs.assemblymapper.AssemblyMapper( + self._hdp, + assembly_name="GRCh38", + alt_aln_method="splign", + normalize=False, + ) def c_to_p(self, c_hgvs: str) -> str: self._ensure_initialized() @@ -67,3 +75,8 @@ def g_to_c(self, g_hgvs: str, transcript: str) -> str: def c_to_g(self, c_hgvs: str) -> str: self._ensure_initialized() return str(self._mapper.c_to_g(self._parser.parse(c_hgvs))) + + def c_to_g_literal(self, c_hgvs: str) -> str: + """Project to genomic without normalization; variant-annotation uses it to rewrite an ``inv`` as delins.""" + self._ensure_initialized() + return str(self._literal_mapper.c_to_g(self._parser.parse(c_hgvs))) From 09bbcf66d07e6385628cdce2ce384c78f2a59a7a Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Fri, 2 Oct 2026 14:04:54 -0700 Subject: [PATCH 202/252] docs(scripts): explain projection-group and orphaned-allele audit findings --- src/mavedb/scripts/audit_reverse_translation.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/mavedb/scripts/audit_reverse_translation.py b/src/mavedb/scripts/audit_reverse_translation.py index e0ad6fd54..62fbfd341 100644 --- a/src/mavedb/scripts/audit_reverse_translation.py +++ b/src/mavedb/scripts/audit_reverse_translation.py @@ -188,8 +188,9 @@ class SqlCheck: name="malformed_projection_group", severity=WARNING, description=( - "A live projection group is not exactly one cdna allele and one genomic allele. A member that " - "failed to translate leaves a group of one; read the samples." + "A live projection group is not exactly one cdna allele and one genomic allele. A member RT could not " + "translate leaves a group of one; the variant's cross_level_translation event lists it under " + "failed_candidates with the error." ), sql=""" SELECT l.mapping_record_id, l.projection_group, count(*) AS members, @@ -232,7 +233,10 @@ class SqlCheck: SqlCheck( name="allele_without_live_link", severity=INFO, - description="An allele no live mapping record links to. Remaps leave some behind; a large count means links were retired without replacement.", + description=( + "An allele no live mapping record links to. Superseded mappings, including every non-current legacy " + "row the backfill imported, leave alleles with retired links only; one with no link at all was never linked." + ), sql=""" SELECT a.id AS allele_id, a.level, coalesce(a.hgvs_g, a.hgvs_c, a.hgvs_p) AS hgvs FROM alleles a From 993d1ff1dc80c88416af136078f8c74d84c20fb2 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Fri, 2 Oct 2026 14:12:57 -0700 Subject: [PATCH 203/252] chore(deps): update variant-annotation to v0.1.0-dev.5 Reverse translation no longer keeps a genomic projection that deletes an intron for a coding insertion at an exon junction. --- poetry.lock | 8 ++++---- pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index 70caf8f0e..3a451d1a7 100644 --- a/poetry.lock +++ b/poetry.lock @@ -4937,7 +4937,7 @@ test = ["aiohttp (>=3.10.5)", "flake8 (>=6.1,<7.0)", "mypy (>=0.800)", "psutil", [[package]] name = "variant-annotation" -version = "0.1.0-dev.4" +version = "0.1.0-dev.5" description = "Genetic variant mapping and annotation pipeline" optional = true python-versions = ">=3.11" @@ -4967,8 +4967,8 @@ tests = ["pytest"] [package.source] type = "git" url = "https://github.com/bbi-lab/variant-annotation.git" -reference = "v0.1.0-dev.4" -resolved_reference = "1a7bc7e4daaeb7cb602c538bbccf9b9cf4043c63" +reference = "v0.1.0-dev.5" +resolved_reference = "b0952affeb3677fdc5b927cf8fc46dcfe7d54cdd" [[package]] name = "variant-translation" @@ -5301,4 +5301,4 @@ server = ["aiocache", "alembic", "alembic-utils", "arq", "authlib", "biocommons" [metadata] lock-version = "2.1" python-versions = "^3.11" -content-hash = "6488da5d58e664eb98f1c93f7118865e2dda724662f7be6b7ca715dc623632db" +content-hash = "ab765d9795413ebcf330c5040e2e8dddd9eb267bcf918c906aa8c9472f6a2fd7" diff --git a/pyproject.toml b/pyproject.toml index 17d4e0fc5..f49a1ae46 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,7 +65,7 @@ starlette = { version = "^1.7.0", optional = true } starlette-context = { version = "^0.3.6", optional = true } slack-sdk = { version = "~3.21.3", optional = true } uvicorn = { extras = ["standard"], version = "*", optional = true } -variant-annotation = { git = "https://github.com/bbi-lab/variant-annotation.git", tag = "v0.1.0-dev.4", optional = true } +variant-annotation = { git = "https://github.com/bbi-lab/variant-annotation.git", tag = "v0.1.0-dev.5", optional = true } watchtower = { version = "~3.2.0", optional = true } asyncclick = "^8.3.0.7" filelock = "^3.29.0" From 208d6fab30c8362bed86b152feadac01eb42efb7 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Fri, 2 Oct 2026 14:30:03 -0700 Subject: [PATCH 204/252] fix(scripts): stop round-trip reporting unchanged and start-codon proteins as disagreements A start-codon change has no predictable protein effect (p.?), which VRS cannot express, so report it separately instead of as a failed re-derivation. Two notations for an unchanged protein, such as hgvs's p.Ter330Ter for an insertion that creates a stop right after the last residue and RT's p.Thr328=, now count as a match. --- src/mavedb/scripts/audit_reverse_translation.py | 17 +++++++++++++++++ tests/scripts/test_audit_reverse_translation.py | 16 ++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/src/mavedb/scripts/audit_reverse_translation.py b/src/mavedb/scripts/audit_reverse_translation.py index 62fbfd341..3f8155e60 100644 --- a/src/mavedb/scripts/audit_reverse_translation.py +++ b/src/mavedb/scripts/audit_reverse_translation.py @@ -28,6 +28,7 @@ import json import logging +import re import sys from collections import Counter, defaultdict from dataclasses import dataclass, field @@ -519,6 +520,14 @@ def outcomes(db: Session, urns: tuple[str, ...], max_failed_pct: float) -> None: """ +_UNCHANGED_PROTEIN_RE = re.compile(r":p\.(=|[A-Z][a-z]{2}\d+=|Ter\d+Ter|\*\d+\*)$") + + +def _is_unchanged_protein(hgvs_p: str) -> bool: + """Whether a protein HGVS expression says the protein sequence is unchanged.""" + return bool(_UNCHANGED_PROTEIN_RE.search(hgvs_p)) + + @dataclass(frozen=True) class ProjectionPair: """One live projection group as stored, plus the protein alleles live on its record.""" @@ -578,6 +587,10 @@ def round_trip_pair( try: derived_p = strip_protein_prediction_parens(c_to_p(pair.hgvs_c)) + # A start-codon change has no predictable protein effect (p.? or p.Met1?), and VRS cannot express one. + if derived_p.endswith("?"): + results.append(("protein: unknown effect (start codon)", {**base, "derived": derived_p})) + return results derived_p_digest = digest_of(derived_p) except Exception as e: results.append(("protein: could not re-derive", {**base, "error": str(e)})) @@ -585,6 +598,10 @@ def round_trip_pair( if derived_p_digest in digests: results.append(("protein: matches", base)) + elif _is_unchanged_protein(derived_p) and all(_is_unchanged_protein(h) for h in pair.protein_hgvs if h): + # Different notations for the same unchanged protein: hgvs writes an insertion that creates a stop right + # after the last residue as p.Ter330Ter, while RT's apex is p.Thr328=. Their digests differ. + results.append(("protein: matches (unchanged)", base)) else: results.append( ( diff --git a/tests/scripts/test_audit_reverse_translation.py b/tests/scripts/test_audit_reverse_translation.py index 9c85aebe5..71b5ae5dd 100644 --- a/tests/scripts/test_audit_reverse_translation.py +++ b/tests/scripts/test_audit_reverse_translation.py @@ -345,6 +345,22 @@ def fail(_): assert [outcome for outcome, _ in outcomes] == ["genomic: could not re-derive", "protein: could not re-derive"] + @pytest.mark.parametrize("derived", ["NP_1.1:p.?", "NP_1.1:p.(Met1?)"]) + def test_reports_a_start_codon_change_as_unknown_effect_not_a_failure(self, derived): + outcomes = round_trip_pair(_pair(), lambda c: "NC_1.11:g.100A>G", lambda c: derived, DIGESTS.__getitem__) + + assert outcomes[1][0] == "protein: unknown effect (start codon)" + + def test_two_notations_for_an_unchanged_protein_match(self): + pair = _pair(protein_digests=["p-synonymous"], protein_hgvs=["NP_1.1:p.Thr328="]) + digests = {**DIGESTS, "NP_1.1:p.Ter330Ter": "p-stop"} + + outcomes = round_trip_pair( + pair, lambda c: "NC_1.11:g.100A>G", lambda c: "NP_1.1:p.Ter330Ter", digests.__getitem__ + ) + + assert outcomes[1][0] == "protein: matches (unchanged)" + def test_reports_a_record_with_no_protein_apex(self): outcomes = round_trip_pair( _pair(protein_digests=[], protein_hgvs=[]), lambda c: "NC_1.11:g.100A>G", lambda c: "", DIGESTS.__getitem__ From 30af659d0a11a127559f884bd24d5260eefea350 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Mon, 5 Oct 2026 11:08:33 -0700 Subject: [PATCH 205/252] fix(worker): wait out UTA outages longer before failing reverse translation The public UTA server refuses connections for minutes at a time. The library defaults gave up after about 6 s (reverse-translate subprocess) and 3 s (the job's own UTA lookups), so one outage failed every score set in flight. Retry for about 2.5 min and 1 min instead. --- .../variant_processing/reverse_translation.py | 4 +++ src/mavedb/worker/lib/translation_ports.py | 7 +++- .../test_reverse_translation.py | 5 +++ tests/worker/lib/test_translation_ports.py | 32 +++++++++++++++++++ 4 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 tests/worker/lib/test_translation_ports.py diff --git a/src/mavedb/worker/jobs/variant_processing/reverse_translation.py b/src/mavedb/worker/jobs/variant_processing/reverse_translation.py index d243dc037..e3988920a 100644 --- a/src/mavedb/worker/jobs/variant_processing/reverse_translation.py +++ b/src/mavedb/worker/jobs/variant_processing/reverse_translation.py @@ -67,6 +67,10 @@ _DEFAULT_TRANSLATION_CONFIG: dict[str, Any] = { "include_indels": True, "wt_codon_mode": WtCodonMode.ALL, + # The public UTA server refuses connections for minutes at a time; the library default (about 6 s + # in total) fails the whole batch on a blip. Waits 5, 10, 20, 40 and 80 s before giving up. + "upstream_max_attempts": 6, + "upstream_retry_backoff_seconds": 5.0, } diff --git a/src/mavedb/worker/lib/translation_ports.py b/src/mavedb/worker/lib/translation_ports.py index c1b1ba0da..b3a2ea210 100644 --- a/src/mavedb/worker/lib/translation_ports.py +++ b/src/mavedb/worker/lib/translation_ports.py @@ -19,6 +19,11 @@ from variant_annotation.lib.clients.uta import UtaClient +# The public UTA server refuses connections for minutes at a time; the client default (about 3 s in +# total) fails the job on a blip. Waits 2, 4, 8, 16 and 32 s before giving up. +UTA_MAX_ATTEMPTS = 6 +UTA_BACKOFF_SECONDS = 2.0 + @contextlib.contextmanager def uta_transcript_source() -> Generator[UtaClient]: @@ -32,7 +37,7 @@ def uta_transcript_source() -> Generator[UtaClient]: uta_db_url = (os.environ.get("UTA_DB_URL") or "").strip() if not uta_db_url: raise RuntimeError("UTA_DB_URL must be set to resolve transcript facts (NP_→NM_ associations and WT codons).") - with UtaClient.from_url(uta_db_url) as client: + with UtaClient.from_url(uta_db_url, max_attempts=UTA_MAX_ATTEMPTS, backoff_seconds=UTA_BACKOFF_SECONDS) as client: yield client diff --git a/tests/worker/jobs/variant_processing/test_reverse_translation.py b/tests/worker/jobs/variant_processing/test_reverse_translation.py index e8f9a0060..152133ff0 100644 --- a/tests/worker/jobs/variant_processing/test_reverse_translation.py +++ b/tests/worker/jobs/variant_processing/test_reverse_translation.py @@ -251,6 +251,11 @@ def test_none_uses_job_defaults(self): assert config.include_indels is True assert config.wt_codon_mode == WtCodonMode.ALL + def test_defaults_ride_out_a_minutes_long_uta_outage(self): + config = _build_translation_config(None) + assert config.upstream_max_attempts == 6 + assert config.upstream_retry_backoff_seconds == 5.0 + def test_empty_dict_uses_job_defaults(self): config = _build_translation_config({}) assert config.include_indels is True diff --git a/tests/worker/lib/test_translation_ports.py b/tests/worker/lib/test_translation_ports.py new file mode 100644 index 000000000..ad874a009 --- /dev/null +++ b/tests/worker/lib/test_translation_ports.py @@ -0,0 +1,32 @@ +# ruff: noqa: E402 +import pytest + +pytest.importorskip("arq") + +from unittest.mock import MagicMock, patch + +from mavedb.worker.lib import translation_ports +from mavedb.worker.lib.translation_ports import uta_transcript_source + + +@pytest.mark.unit +class TestUtaTranscriptSource: + def test_retries_long_enough_to_ride_out_a_uta_outage(self, monkeypatch): + monkeypatch.setenv("UTA_DB_URL", "postgresql://anonymous@uta.example.org:5432/uta/uta_20241220") + client = MagicMock() + client.__enter__.return_value = client + + with patch.object(translation_ports.UtaClient, "from_url", return_value=client) as from_url: + with uta_transcript_source() as source: + assert source is client + + from_url.assert_called_once_with( + "postgresql://anonymous@uta.example.org:5432/uta/uta_20241220", max_attempts=6, backoff_seconds=2.0 + ) + + def test_requires_uta_db_url(self, monkeypatch): + monkeypatch.delenv("UTA_DB_URL", raising=False) + + with pytest.raises(RuntimeError, match="UTA_DB_URL must be set"): + with uta_transcript_source(): + pass From 97d8b4da132b09dfc4a019af5214d2530882ec04 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Mon, 5 Oct 2026 11:08:34 -0700 Subject: [PATCH 206/252] fix(backfill): skip alleles not on SeqRepo's sequence for their accession Legacy rows can put one accession on two sequences (NP_001346945.1). With one allele per HGVS enforced, whichever the backfill reached first claimed each HGVS, and the rest failed and rolled back whole variants. Check each allele against SeqRepo, as the mapping and reverse translation jobs already do; a mismatch keeps the mapping record without an allele for the enrich re-map to rebuild, and is counted as alleles_off_sequence. --- ...ate_mapped_variants_to_allele_substrate.py | 65 +++++++++++++++++-- 1 file changed, 60 insertions(+), 5 deletions(-) diff --git a/alembic/manual_migrations/migrate_mapped_variants_to_allele_substrate.py b/alembic/manual_migrations/migrate_mapped_variants_to_allele_substrate.py index ea0e239bc..71f4aeda2 100644 --- a/alembic/manual_migrations/migrate_mapped_variants_to_allele_substrate.py +++ b/alembic/manual_migrations/migrate_mapped_variants_to_allele_substrate.py @@ -125,6 +125,7 @@ from typing import Optional import sqlalchemy as sa +from biocommons.seqrepo import SeqRepo from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Session, configure_mappers @@ -133,8 +134,11 @@ from mavedb.lib.annotation_status_manager import AnnotationStatusManager from pydantic import ValidationError +from mavedb.data_providers.services import seqrepo +from mavedb.lib.hgvs import extract_accession +from mavedb.lib.seqrepo import AmbiguousSequenceError, SequenceNotFoundError, resolve_refget from mavedb.lib.variants import get_hgvs_from_post_mapped -from mavedb.lib.vrs_utils import canonical_variation_document +from mavedb.lib.vrs_utils import CANONICAL_ACCESSION_PREFIXES, canonical_variation_document, location_refgets from mavedb.models.allele import Allele from mavedb.models.annotation_event import AnnotationEvent from mavedb.models.clinvar_allele_link import ClinvarAlleleLink @@ -381,13 +385,55 @@ def _fill_missing_allele_fields(allele: Allele, mv: MappedVariant, level: Sequen setattr(allele, hgvs_col, hgvs_value) +class _SequenceGuard: + """Checks that an allele sits on the sequence SeqRepo holds for its HGVS accession. + + The backfill counterpart of :func:`mavedb.lib.vrs_utils.verify_allele_refget`, which the mapping and + reverse translation jobs run at insert time. Legacy rows can put one accession on two sequences + (``NP_001346945.1`` on two refgets), and ``uq_alleles_hgvs`` lets only one of + them hold each HGVS. Without this check, whichever the backfill reaches first wins. SeqRepo's answer + is cached per accession for the run. + """ + + def __init__(self, sr: SeqRepo) -> None: + self._sr = sr + self._expected: dict[str, Optional[str]] = {} + self.mismatches = 0 + + def agrees(self, variation: dict, hgvs: Optional[str]) -> bool: + """Return False if the variation's refget differs from SeqRepo's for the accession. + + An accession SeqRepo can't resolve passes, as it does in ``verify_allele_refget``: there is nothing + to compare against. + """ + accession = extract_accession(hgvs or "") + if not accession.startswith(CANONICAL_ACCESSION_PREFIXES): + return True + if accession not in self._expected: + try: + self._expected[accession] = resolve_refget(self._sr, accession) + except (SequenceNotFoundError, AmbiguousSequenceError) as e: + logger.error(f"Cannot verify refgets on {accession}: {e}") + self._expected[accession] = None + expected = self._expected[accession] + if expected is None or all(refget == expected for refget in location_refgets(variation)): + return True + self.mismatches += 1 + return False + + def _authoritative_allele( - db: Session, mv: MappedVariant, level: SequenceLevel, allele_cache: dict[str, Allele] + db: Session, + mv: MappedVariant, + level: SequenceLevel, + allele_cache: dict[str, Allele], + sequence_guard: _SequenceGuard, ) -> Optional[Allele]: """Get-or-create the authoritative allele from ``mv.post_mapped``, via an in-memory dedup cache. Returns ``None`` for versions with no post-mapped representation (failed or benign-absent variants - get a record but no linked allele, matching the mapping job). + get a record but no linked allele, matching the mapping job), and for versions on a sequence other + than SeqRepo's for their accession (see :class:`_SequenceGuard`); the enrich re-map rebuilds those. Unlike :func:`mavedb.lib.variant_translations.get_or_create_allele` (used by the live mapping job), this does **not** query the database or flush per call. ``allele_cache`` is pre-seeded by the @@ -419,6 +465,11 @@ def _authoritative_allele( logger.exception(f"Could not canonicalize post_mapped for mapped_variant {mv.id}; skipping its allele.") return None + hgvs_value = _resolve_hgvs_assay_level(mv, level) + if not sequence_guard.agrees(post_mapped, hgvs_value): + logger.warning(f"mapped_variant {mv.id} ({hgvs_value}) is not on SeqRepo's sequence; skipping its allele.") + return None + existing = allele_cache.get(digest) if existing is None: # `allele_cache` is seeded by a per-chunk prefetch keyed on the STORED `post_mapped` id, but @@ -435,7 +486,6 @@ def _authoritative_allele( _fill_missing_allele_fields(existing, mv, level) return existing - hgvs_value = _resolve_hgvs_assay_level(mv, level) allele = Allele( vrs_digest=digest, level=level, @@ -807,6 +857,7 @@ def _migrate_variant( annotation_manager: AnnotationStatusManager, existing_records: set[tuple[int, date]], allele_cache: dict[str, Allele], + sequence_guard: _SequenceGuard, ) -> set[int]: """Reconstruct as much of a variant's record history as its data supports, returning the ids of every allele this variant authoritatively touched. @@ -929,7 +980,7 @@ def _migrate_variant( db.add(record) stats["records"] += 1 - allele = _authoritative_allele(db, mv, level, allele_cache) + allele = _authoritative_allele(db, mv, level, allele_cache, sequence_guard) if allele is not None: db.add( MappingRecordAllele( @@ -1084,6 +1135,7 @@ def do_migration( print(" DRY RUN — no changes will be committed.") run_at = run_at or datetime.now(timezone.utc) annotation_manager = AnnotationStatusManager(db) + sequence_guard = _SequenceGuard(seqrepo()) variant_ids = _variant_ids_to_migrate(db, score_set_urn) print(f"Found {len(variant_ids)} variants to migrate.") @@ -1155,6 +1207,7 @@ def do_migration( annotation_manager, existing_records, allele_cache, + sequence_guard, ) except Exception as exc: # noqa: BLE001 — one bad variant must not abort the batch # Discard any events this variant buffered but never got to flush before raising — they @@ -1191,10 +1244,12 @@ def do_migration( phase_seconds["annotation_timelines"] += perf_counter() - mark print("\nMigration completed:") + stats["alleles_off_sequence"] = sequence_guard.mismatches for key in ( "variants", "records", "record_alleles", + "alleles_off_sequence", "hgvs_recovered_from_level_column", "hgvs_recovered_from_vrs_expression", "annotation_events_written", From 6d9b0b64da5ede8fa55cd3047d596e949cb3c649 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Mon, 5 Oct 2026 11:08:34 -0700 Subject: [PATCH 207/252] fix(scripts): run audit_allele_identifiers' CLI when invoked as a script It had no __main__ entry point, so it exited silently and looked like a clean pass. --- src/mavedb/scripts/audit_allele_identifiers.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/mavedb/scripts/audit_allele_identifiers.py b/src/mavedb/scripts/audit_allele_identifiers.py index 5b8fb1380..cbd0292e7 100644 --- a/src/mavedb/scripts/audit_allele_identifiers.py +++ b/src/mavedb/scripts/audit_allele_identifiers.py @@ -267,3 +267,7 @@ def _recomputed_location_identifier(location: dict[str, Any]) -> Optional[str]: except Exception: logger.debug("Could not hydrate a location for identification", exc_info=True) return None + + +if __name__ == "__main__": + script_environment() From abb36e4cd4b02bbbb936e84d5bf34a7b6e9c36f4 Mon Sep 17 00:00:00 2001 From: Estelle Da Date: Tue, 6 Oct 2026 18:32:33 +1100 Subject: [PATCH 208/252] Add more except cases and some related tests. --- src/mavedb/lib/identifiers.py | 17 ++++--- tests/lib/test_identifiers.py | 95 +++++++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+), 7 deletions(-) create mode 100644 tests/lib/test_identifiers.py diff --git a/src/mavedb/lib/identifiers.py b/src/mavedb/lib/identifiers.py index c8d5d6acf..dcc9c6a14 100644 --- a/src/mavedb/lib/identifiers.py +++ b/src/mavedb/lib/identifiers.py @@ -4,6 +4,7 @@ from typing import Mapping, Optional, Union import eutils # type: ignore +import requests from eutils import EutilsNCBIError # type: ignore from eutils._internal.xmlfacades.pubmedarticle import PubmedArticle # type: ignore from eutils._internal.xmlfacades.pubmedarticleset import PubmedArticleSet # type: ignore @@ -13,6 +14,7 @@ from mavedb.lib.exceptions import AmbiguousIdentifierError, NonexistentIdentifierError from mavedb.lib.external_publications import Crossref, CrossrefWork, PublicationAuthors, Rxiv, RxivContentDetail +from mavedb.lib.logging.context import format_raised_exception_info_as_dict, logging_context, save_to_logging_context from mavedb.lib.validation.publication import identifier_valid_for, infer_identifier_from_url, validate_db_name from mavedb.models.doi_identifier import DoiIdentifier from mavedb.models.ensembl_identifier import EnsemblIdentifier @@ -317,6 +319,7 @@ async def find_generic_article( # When we are not provided a db name, we must try to match the provided identifier to a # publication from each one of our accepted databases. + last_fetch_error: Optional[Exception] = None for publication_db, identifier_valid in identifier_valid_for(identifier).items(): if identifier_valid: existing_publication = db.execute( @@ -328,14 +331,11 @@ async def find_generic_article( if not existing_publication: try: external_publication = await db_specific_fetches[publication_db](identifier) - except json.JSONDecodeError: - logger.warning( - "Failed to fetch identifier %r from %s while fanning out over candidate databases.", - identifier, - publication_db, - exc_info=True, - ) + except (json.JSONDecodeError, requests.exceptions.RequestException, requests.exceptions.HTTPError) as exc: + save_to_logging_context(format_raised_exception_info_as_dict(exc)) + logger.error(msg=f"Failed to fetch identifier {identifier} from candidate databases.", exc_info=exc, extra=logging_context()) external_publication = None + last_fetch_error = exc found_articles[publication_db] = ( ExternalPublication(identifier, publication_db, external_publication) @@ -345,6 +345,9 @@ async def find_generic_article( else: found_articles[publication_db] = existing_publication + if last_fetch_error is not None and not any(found_articles.values()): + raise last_fetch_error + return found_articles diff --git a/tests/lib/test_identifiers.py b/tests/lib/test_identifiers.py new file mode 100644 index 000000000..06c900b8f --- /dev/null +++ b/tests/lib/test_identifiers.py @@ -0,0 +1,95 @@ +# ruff: noqa: E402 + +import pytest +import requests + +from mavedb.lib.identifiers import find_generic_article +from tests.helpers.constants import TEST_BIORXIV_IDENTIFIER, TEST_PUBMED_IDENTIFIER + +PUBMED_EFETCH_URL = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi" + + +def mock_pubmed_success(requests_mock, identifier): + requests_mock.post( + PUBMED_EFETCH_URL, + text=f""" + + + + {identifier} +
+ + test + + + 1999 + + + + + test + +
+ + + + test + + + + + """, + ) + + +def medrxiv_url(identifier): + return f"https://api.biorxiv.org/details/medrxiv/10.1101/{identifier}/na/json" + + +def biorxiv_url(identifier): + return f"https://api.biorxiv.org/details/biorxiv/10.1101/{identifier}/na/json" + + +@pytest.mark.asyncio +async def test_find_generic_article_drops_candidate_with_malformed_response(session, requests_mock): + """ + A PMID like TEST_PUBMED_IDENTIFIER is also a plausible (8 digit) legacy medRxiv + identifier, so find_generic_article fans out to both databases. If medRxiv responds + with a body that can't be parsed as JSON, that candidate should be dropped (and logged) + rather than failing the whole lookup when PubMed already has a match. + """ + mock_pubmed_success(requests_mock, TEST_PUBMED_IDENTIFIER) + requests_mock.get(medrxiv_url(TEST_PUBMED_IDENTIFIER), text="not json") + + matches = await find_generic_article(session, TEST_PUBMED_IDENTIFIER) + + assert matches["PubMed"] is not None + assert matches.get("medRxiv") is None + + +@pytest.mark.asyncio +async def test_find_generic_article_drops_candidate_on_request_exception(session, requests_mock): + """ + The same fan-out should also tolerate a candidate database being entirely unreachable + (timeouts, connection errors, bad status codes), not just malformed JSON bodies. + """ + mock_pubmed_success(requests_mock, TEST_PUBMED_IDENTIFIER) + requests_mock.get(medrxiv_url(TEST_PUBMED_IDENTIFIER), exc=requests.exceptions.ConnectTimeout) + + matches = await find_generic_article(session, TEST_PUBMED_IDENTIFIER) + + assert matches["PubMed"] is not None + assert matches.get("medRxiv") is None + + +@pytest.mark.asyncio +async def test_find_generic_article_reraises_error_when_no_candidate_succeeds(session, requests_mock): + """ + When every candidate database fails and none produces a match, the underlying error + should propagate instead of being silently reported as "not found", so callers can + still surface it as a gateway error (eg. a 504 on a Rxiv timeout). + """ + requests_mock.get(biorxiv_url(TEST_BIORXIV_IDENTIFIER), exc=requests.exceptions.ConnectTimeout) + + with pytest.raises(requests.exceptions.ConnectTimeout): + await find_generic_article(session, TEST_BIORXIV_IDENTIFIER) From aaffeb1e68c429ab7ef6e6fa4c6ceda61c833f09 Mon Sep 17 00:00:00 2001 From: Estelle Da Date: Wed, 7 Oct 2026 14:07:00 +1100 Subject: [PATCH 209/252] Fixed no module fastapi problem in some tests. --- src/mavedb/lib/identifiers.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/mavedb/lib/identifiers.py b/src/mavedb/lib/identifiers.py index dcc9c6a14..7855b6af7 100644 --- a/src/mavedb/lib/identifiers.py +++ b/src/mavedb/lib/identifiers.py @@ -14,7 +14,6 @@ from mavedb.lib.exceptions import AmbiguousIdentifierError, NonexistentIdentifierError from mavedb.lib.external_publications import Crossref, CrossrefWork, PublicationAuthors, Rxiv, RxivContentDetail -from mavedb.lib.logging.context import format_raised_exception_info_as_dict, logging_context, save_to_logging_context from mavedb.lib.validation.publication import identifier_valid_for, infer_identifier_from_url, validate_db_name from mavedb.models.doi_identifier import DoiIdentifier from mavedb.models.ensembl_identifier import EnsemblIdentifier @@ -331,9 +330,14 @@ async def find_generic_article( if not existing_publication: try: external_publication = await db_specific_fetches[publication_db](identifier) - except (json.JSONDecodeError, requests.exceptions.RequestException, requests.exceptions.HTTPError) as exc: - save_to_logging_context(format_raised_exception_info_as_dict(exc)) - logger.error(msg=f"Failed to fetch identifier {identifier} from candidate databases.", exc_info=exc, extra=logging_context()) + except (json.JSONDecodeError, requests.exceptions.RequestException) as exc: + logger.warning( + "Failed to fetch identifier %r from %s while fanning out over candidate databases.", + identifier, + publication_db, + exc_info=True, + ) + external_publication = None last_fetch_error = exc From 0e1be4cf9572e5de384ee7685787bb16396f8413 Mon Sep 17 00:00:00 2001 From: Estelle Da Date: Wed, 7 Oct 2026 15:03:54 +1100 Subject: [PATCH 210/252] Tried to solve the No module named 'biocommons' problem in some tests. --- tests/scripts/test_audit_reverse_translation.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/scripts/test_audit_reverse_translation.py b/tests/scripts/test_audit_reverse_translation.py index 71b5ae5dd..b626a88e9 100644 --- a/tests/scripts/test_audit_reverse_translation.py +++ b/tests/scripts/test_audit_reverse_translation.py @@ -4,6 +4,11 @@ import pytest +arq = pytest.importorskip("arq") +cdot = pytest.importorskip("cdot") +fastapi = pytest.importorskip("fastapi") +biocommons = pytest.importorskip("biocommons") +bioutils = pytest.importorskip("bioutils") pytest.importorskip("ga4gh.vrs") from mavedb.lib.seqrepo import SequenceNotFoundError From 675cc25771fbfb7ea7c2365007b010f62291f00b Mon Sep 17 00:00:00 2001 From: Estelle Da Date: Wed, 7 Oct 2026 15:38:05 +1100 Subject: [PATCH 211/252] Tried to solve the No module named 'psycopg2' in test_identifiers file. --- tests/lib/test_identifiers.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/lib/test_identifiers.py b/tests/lib/test_identifiers.py index 06c900b8f..cc378223b 100644 --- a/tests/lib/test_identifiers.py +++ b/tests/lib/test_identifiers.py @@ -1,6 +1,9 @@ # ruff: noqa: E402 import pytest + +pytest.importorskip("psycopg2") + import requests from mavedb.lib.identifiers import find_generic_article From a8fa96040c14346b5930c7f26747ff55795119a1 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 7 Oct 2026 09:38:20 -0700 Subject: [PATCH 212/252] fix(publish): serialize URN assignment and return 409 on collision The generate_*_urn functions read the highest assigned URN and add one, so two concurrent publications under READ COMMITTED could mint the same URN. Publishing now takes a transaction-scoped advisory lock before loading the score set, and refresh_variant_urns no longer commits mid-publish. A collision that still reaches the database rolls back and returns 409 instead of a 500. --- src/mavedb/lib/score_sets.py | 2 - src/mavedb/lib/urns.py | 27 ++++++++--- src/mavedb/routers/score_sets.py | 80 +++++++++++++++++++------------- tests/lib/test_urns.py | 34 ++++++++++++-- tests/routers/test_score_set.py | 31 +++++++++++++ 5 files changed, 130 insertions(+), 44 deletions(-) diff --git a/src/mavedb/lib/score_sets.py b/src/mavedb/lib/score_sets.py index e38b5a90f..6d599980f 100644 --- a/src/mavedb/lib/score_sets.py +++ b/src/mavedb/lib/score_sets.py @@ -800,8 +800,6 @@ def refresh_variant_urns(db: Session, score_set: ScoreSet): variant.urn = refreshed_urn db.add(variant) - db.commit() - def bulk_create_urns(n, score_set, reset_counter=False) -> list[str]: start_value = 0 if reset_counter else score_set.num_variants diff --git a/src/mavedb/lib/urns.py b/src/mavedb/lib/urns.py index 46be37a10..e67ad26d1 100644 --- a/src/mavedb/lib/urns.py +++ b/src/mavedb/lib/urns.py @@ -4,7 +4,7 @@ from typing import Optional from uuid import uuid4 -from sqlalchemy import func +from sqlalchemy import func, text from sqlalchemy.orm import Session from mavedb.lib.validation.urn_re import ( @@ -18,13 +18,26 @@ logger = logging.getLogger(__name__) +def lock_urn_assignment(db: Session) -> None: + """ + Serialize permanent URN assignment for the rest of the current transaction. + + The ``generate_*_urn`` functions read the highest assigned URN and add one, so under READ COMMITTED two concurrent + publications would mint the same URN. Take this before loading the records being published; the lock is released + when the transaction commits or rolls back. + + :param db: An active database session + """ + db.execute(text("SELECT pg_advisory_xact_lock(hashtext('mavedb.urn_assignment'))")) + + def generate_experiment_set_urn(db: Session): """ Generate a new URN for an experiment set. Experiment set URNs include an 8-digit, zero-padded, sequentially-assigned numeric part. This function finds the - maximum value in the database and adds one to form the new URN. To ensure atomicity, it should be called in the - context of a database transaction. + maximum value in the database and adds one to form the new URN. Callers must hold ``lock_urn_assignment`` so that + concurrent publications cannot read the same maximum. :param db: An active database session :return: The next available experiment set URN @@ -54,8 +67,8 @@ def generate_experiment_urn(db: Session, experiment_set: ExperimentSet, experime a, b, ..., z, aa, ab, ..., az, ba, ... bz, ... zz, aaa, ..., zzz, aaaa, ... ``` This function looks at the database records for other experiments in the set and finds the maximum value of the - alphabetic part, then increments it to form the new URN. To ensure atomicity, it should be called in the context of - a database transaction to ensure atomicity. + alphabetic part, then increments it to form the new URN. Callers must hold ``lock_urn_assignment`` so that + concurrent publications cannot read the same maximum. For meta-analyses, the suffix is always 0. There can only be one meta-analysis per experiment set. @@ -110,8 +123,8 @@ def generate_score_set_urn(db: Session, experiment: Experiment): zero-padded to a fixed width. This function looks at the database records for other scoresets belonging to the experiment and finds the maximum - value of the numeric part, then increments it to form the new URN. To ensure atomicity, it should be called in the - context of a database transaction. + value of the numeric part, then increments it to form the new URN. Callers must hold ``lock_urn_assignment`` so that + concurrent publications cannot read the same maximum. :param db: An active database session :param experiment: The experiment to which this score set belongs diff --git a/src/mavedb/routers/score_sets.py b/src/mavedb/routers/score_sets.py index f9635d891..1f352232f 100644 --- a/src/mavedb/routers/score_sets.py +++ b/src/mavedb/routers/score_sets.py @@ -87,6 +87,7 @@ generate_experiment_set_urn, generate_experiment_urn, generate_score_set_urn, + lock_urn_assignment, ) from mavedb.lib.variant_detail import get_variant_detail from mavedb.lib.workflow.kickoff import enqueue_pipeline_for_score_set @@ -2779,6 +2780,9 @@ async def publish_score_set( """ save_to_logging_context({"requested_resource": urn}) + # Taken before loading the score set so a concurrent publish of the same record waits, then sees it published. + lock_urn_assignment(db) + item: Optional[ScoreSet] = db.query(ScoreSet).filter(ScoreSet.urn == urn).one_or_none() if not item: logger.info(msg="Failed to publish score set; The requested score set does not exist.", extra=logging_context()) @@ -2823,44 +2827,58 @@ async def publish_score_set( detail="cannot publish score set without variant scores", ) - published_date = date.today() + # Autoflush can write the new URNs at any query below, so a collision may surface before the commit. + try: + published_date = date.today() - if item.experiment.experiment_set.private or not item.experiment.experiment_set.published_date: - retired_experiment_set_urn = item.experiment.experiment_set.urn - item.experiment.experiment_set.urn = generate_experiment_set_urn(db) - record_urn_redirect(db, retired_experiment_set_urn, item.experiment.experiment_set.urn) - item.experiment.experiment_set.private = False - item.experiment.experiment_set.published_date = published_date - db.add(item.experiment.experiment_set) + if item.experiment.experiment_set.private or not item.experiment.experiment_set.published_date: + retired_experiment_set_urn = item.experiment.experiment_set.urn + item.experiment.experiment_set.urn = generate_experiment_set_urn(db) + record_urn_redirect(db, retired_experiment_set_urn, item.experiment.experiment_set.urn) + item.experiment.experiment_set.private = False + item.experiment.experiment_set.published_date = published_date + db.add(item.experiment.experiment_set) - save_to_logging_context({"experiment_set": item.experiment.experiment_set.urn}) + save_to_logging_context({"experiment_set": item.experiment.experiment_set.urn}) - if item.experiment.private or not item.experiment.published_date: - retired_experiment_urn = item.experiment.urn - item.experiment.urn = generate_experiment_urn( - db, - item.experiment.experiment_set, - experiment_is_meta_analysis=len(item.meta_analyzes_score_sets) > 0, - ) - record_urn_redirect(db, retired_experiment_urn, item.experiment.urn) - item.experiment.private = False - item.experiment.published_date = published_date - db.add(item.experiment) + if item.experiment.private or not item.experiment.published_date: + retired_experiment_urn = item.experiment.urn + item.experiment.urn = generate_experiment_urn( + db, + item.experiment.experiment_set, + experiment_is_meta_analysis=len(item.meta_analyzes_score_sets) > 0, + ) + record_urn_redirect(db, retired_experiment_urn, item.experiment.urn) + item.experiment.private = False + item.experiment.published_date = published_date + db.add(item.experiment) - save_to_logging_context({"experiment": item.experiment.urn}) + save_to_logging_context({"experiment": item.experiment.urn}) - retired_score_set_urn = item.urn - item.urn = generate_score_set_urn(db, item.experiment) - # Variant URNs are rewritten below from the score set's, so this one redirect forwards them too. - record_urn_redirect(db, retired_score_set_urn, item.urn) - item.private = False - item.published_date = published_date - refresh_variant_urns(db, item) + retired_score_set_urn = item.urn + item.urn = generate_score_set_urn(db, item.experiment) + # Variant URNs are rewritten below from the score set's, so this one redirect forwards them too. + record_urn_redirect(db, retired_score_set_urn, item.urn) + item.private = False + item.published_date = published_date + refresh_variant_urns(db, item) - save_to_logging_context({"score_set": item.urn}) + save_to_logging_context({"score_set": item.urn}) - db.add(item) - db.commit() + db.add(item) + db.commit() + except IntegrityError: + db.rollback() + logger.warning( + msg="Failed to publish score set; A newly assigned URN collided with an existing record.", + extra=logging_context(), + exc_info=True, + ) + raise HTTPException( + status_code=409, + detail="This score set could not be published because its new URN is already in use. " + "Please reload the page to check whether it was published, and try again if it was not.", + ) db.refresh(item) try: diff --git a/tests/lib/test_urns.py b/tests/lib/test_urns.py index 380bf0fc9..600c723b5 100644 --- a/tests/lib/test_urns.py +++ b/tests/lib/test_urns.py @@ -1,9 +1,8 @@ import pytest +from sqlalchemy import text +from sqlalchemy.orm import sessionmaker -from mavedb.lib.urns import score_set_urn_sort_key, variant_urn_sort_key - -pytestmark = pytest.mark.unit - +from mavedb.lib.urns import lock_urn_assignment, score_set_urn_sort_key, variant_urn_sort_key # --------------------------------------------------------------------------- # Each of these pairs is one a lexical sort gets wrong. They are the reason the keys exist, so they are @@ -12,6 +11,7 @@ # --------------------------------------------------------------------------- +@pytest.mark.unit class TestScoreSetUrnSortKey: def test_unpadded_score_set_number_orders_numerically(self): urns = ["urn:mavedb:00000001-a-10", "urn:mavedb:00000001-a-2"] @@ -66,6 +66,7 @@ def test_undecomposable_urns_still_order_deterministically_among_themselves(self assert sorted(urns, key=score_set_urn_sort_key) == ["tmp:a", "tmp:b", "tmp:c"] +@pytest.mark.unit class TestVariantUrnSortKey: def test_unpadded_variant_number_orders_numerically(self): urns = [f"urn:mavedb:00000001-a-1#{n}" for n in (2, 10, 1)] @@ -97,3 +98,28 @@ def test_urns_without_a_variant_suffix_sort_last_without_raising(self, urn): numbered = "urn:mavedb:00000001-a-1#1" assert sorted([urn, numbered], key=variant_urn_sort_key) == [numbered, urn] + + +@pytest.mark.integration +class TestLockUrnAssignment: + @staticmethod + def _other_session_can_take_lock(session) -> bool: + other = sessionmaker(bind=session.get_bind())() + try: + return other.execute(text("SELECT pg_try_advisory_xact_lock(hashtext('mavedb.urn_assignment'))")).scalar() + finally: + other.rollback() + other.close() + + def test_lock_excludes_other_sessions_until_commit(self, session): + lock_urn_assignment(session) + assert self._other_session_can_take_lock(session) is False + + session.commit() + assert self._other_session_can_take_lock(session) is True + + def test_lock_is_released_on_rollback(self, session): + lock_urn_assignment(session) + session.rollback() + + assert self._other_session_can_take_lock(session) is True diff --git a/tests/routers/test_score_set.py b/tests/routers/test_score_set.py index 06d13a518..137ca757c 100644 --- a/tests/routers/test_score_set.py +++ b/tests/routers/test_score_set.py @@ -1912,6 +1912,37 @@ def test_cannot_publish_an_already_published_score_set(session, data_provider, c assert unchanged.json()["urn"] == published_score_set["urn"] +def test_publish_score_set_returns_conflict_when_assigned_urn_is_taken( + session, data_provider, client, setup_router_db, data_files +): + experiment = create_experiment(client) + score_set_1 = create_seq_score_set(client, experiment["urn"]) + score_set_1 = mock_worker_variant_insertion(client, session, data_provider, score_set_1, data_files / "scores.csv") + score_set_2 = create_seq_score_set(client, experiment["urn"]) + score_set_2 = mock_worker_variant_insertion(client, session, data_provider, score_set_2, data_files / "scores.csv") + + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published_score_set_1 = publish_score_set(client, score_set_1["urn"]) + + # Simulates a concurrent publication that minted the same URN between this request's read and its write. + with ( + patch( + "mavedb.routers.score_sets.generate_score_set_urn", + return_value=published_score_set_1["urn"], + ), + patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as worker_queue, + ): + response = client.post(f"/api/v1/score-sets/{score_set_2['urn']}/publish") + worker_queue.assert_not_called() + + assert response.status_code == 409 + assert "already in use" in response.json()["detail"] + + unpublished = client.get(f"/api/v1/score-sets/{score_set_2['urn']}") + assert unpublished.status_code == 200 + assert unpublished.json()["private"] is True + + def test_publish_score_set_discards_pipeline_when_entrypoint_enqueue_fails( session, data_provider, client, setup_router_db, data_files ): From 561a362d8f38106e19e2ce1036a8a959673fee3b Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 7 Oct 2026 09:38:21 -0700 Subject: [PATCH 213/252] perf(score-sets): keep lean variant lookups as per-variant index probes As plain joins, Postgres can switch to a hash join that scans all of mapping_record_alleles once its row estimate for a score set crosses a cost threshold, which is orders of magnitude slower when the table is not cached. Each per-variant lookup is now a LATERAL ... LIMIT 1 subquery, which the planner cannot flatten, and the protein HGVS is folded into the same query. A test pins the plan shape with nested loops and sequential scans disabled. --- src/mavedb/lib/score_set_variants.py | 161 +++++++++++++++++---------- tests/helpers/util/query_plan.py | 57 ++++++++++ tests/lib/test_score_set_variants.py | 33 +++++- 3 files changed, 190 insertions(+), 61 deletions(-) create mode 100644 tests/helpers/util/query_plan.py diff --git a/src/mavedb/lib/score_set_variants.py b/src/mavedb/lib/score_set_variants.py index 5e95ac27e..319e3928a 100644 --- a/src/mavedb/lib/score_set_variants.py +++ b/src/mavedb/lib/score_set_variants.py @@ -1,8 +1,7 @@ """Lean whole-set view backing the score-set page (table, heatmap, score/effect histograms). -Assembles one record per variant in two O(N) bulk queries: a base per-variant projection, plus a -separate protein-HGVS query stitched back in Python (see :func:`get_protein_hgvs_by_record` for why -it's split out). +Assembles one record per variant in a single query: the score set's variants, with each per-variant +annotation fetched by a ``LATERAL ... LIMIT 1`` lookup (see :func:`get_lean_score_set_variants` for why). Each record carries the **submitted** HGVS (``hgvs_nt``/``hgvs_pro``/``hgvs_splice``, the depositor's target frame) and the **mapped** (reference-frame) HGVS as a ``MappedTriple`` — one canonical HGVS per @@ -23,7 +22,7 @@ from datetime import datetime from typing import Optional, Sequence -from sqlalchemy import and_, func, select +from sqlalchemy import and_, func, select, true from sqlalchemy.orm import Session, aliased from mavedb.lib.hgvs import parse_simple_substitution @@ -172,7 +171,7 @@ def get_protein_hgvs_by_record( ) -> dict[int, str]: """The canonical protein-level mapped HGVS for a set of live mapping records, keyed by ``mapping_record.id``. Records with no protein projection (UTR/intronic) are absent from the map. - Shared by the lean whole-set view and the CSV export. + Used by the CSV export. Exactly one of *score_set_id* (every variant in one score set) or *variant_ids* (an explicit set, which may span several score sets — the variant-level CSV's cross-set equivalent measurements) is @@ -225,77 +224,119 @@ def get_lean_score_set_variants( db: Session, score_set: ScoreSet, *, as_of: Optional[datetime] = None ) -> list[LeanVariantRecord]: """Assemble the lean whole-set view for ``score_set``, one record per variant, ordered by variant - number. Unmapped variants are retained (left joins, null mapped fields) so the table and score - histogram still see them. ``as_of`` defaults to currently-live rows. + number. Unmapped variants are retained (null mapped fields) so the table and score histogram still + see them. ``as_of`` defaults to currently-live rows. + + Every per-variant lookup is a ``LATERAL ... LIMIT 1`` subquery. As plain joins, Postgres may choose a + hash join that scans the whole of ``mapping_record_alleles`` once its row estimate for the score set + crosses a threshold, which is orders of magnitude slower when that table is not cached. A lateral subquery with a + ``LIMIT`` cannot be flattened into the join tree, so each lookup stays a per-variant index probe + whatever the planner's statistics say. Each lookup yields at most one row by invariant (one live + record per variant, one live authoritative link per record, at most one projection partner, one + protein allele); the ``ORDER BY`` only makes a violated invariant deterministic. """ - # The authoritative link's *other* projection_group member — same record, same group, not - # authoritative — carries the canonical HGVS at the unmeasured nucleotide level. See join below. - projection_link = aliased(MappingRecordAllele) - projection_allele = aliased(Allele) + # The variant's live mapping record. + record = ( + select(MappingRecord.id, MappingRecord.assay_level, MappingRecord.hgvs_assay_level) + .where(MappingRecord.variant_id == Variant.id, MappingRecord.live_at(as_of)) + .order_by(MappingRecord.id) + .limit(1) + .lateral("record") + ) - # 1:1 annotation joins, so this is a stream of O(N) index-scan lookups. MappingRecord.id rides - # along as the key the protein projection (separate query, below) is stitched back on. - base_statement = ( + # The authoritative (measured) allele, with its VEP consequence. ``is_authoritative`` is tested as a + # bare boolean: ``IS TRUE`` would not match the partial unique index's ``WHERE is_authoritative``. + authoritative_link = aliased(MappingRecordAllele) + authoritative = ( select( - Variant.urn, - Variant.data, - Variant.hgvs_nt, - Variant.hgvs_pro, - Variant.hgvs_splice, - MappingRecord.id.label("mapping_record_id"), - MappingRecord.assay_level, - MappingRecord.hgvs_assay_level, + authoritative_link.projection_group, Allele.vrs_digest, Allele.clingen_allele_id, VepAlleleConsequence.functional_consequence, + ) + .select_from(authoritative_link) + .join(Allele, Allele.id == authoritative_link.allele_id) + .outerjoin( + VepAlleleConsequence, and_(VepAlleleConsequence.allele_id == Allele.id, VepAlleleConsequence.live_at(as_of)) + ) + .where( + authoritative_link.mapping_record_id == record.c.id, + authoritative_link.is_authoritative, + authoritative_link.live_at(as_of), + ) + .order_by(authoritative_link.id) + .limit(1) + .lateral("authoritative") + ) + + # The authoritative link's projection_group partner: the canonical HGVS at the unmeasured nucleotide + # level. A NULL group (protein assay, or pre-reverse-translation data) matches nothing, so the slot + # degrades to None. + projection_link = aliased(MappingRecordAllele) + projection_allele = aliased(Allele) + projection = ( + select( projection_allele.level.label("projection_level"), # Exactly one of hgvs_g/hgvs_c is populated (it's a nucleotide allele); coalesce picks it. func.coalesce(projection_allele.hgvs_g, projection_allele.hgvs_c).label("projection_hgvs"), ) - # LEFT join so unmapped variants stay (null mapped fields) for the table + score histogram. - # live_at rides in the ON clause, not WHERE. a WHERE would reject the outer join's null row, - # silently turning this into an INNER join. Same pattern on every join below. - .outerjoin(MappingRecord, and_(MappingRecord.variant_id == Variant.id, MappingRecord.live_at(as_of))) - # The one authoritative (measured) allele; is_authoritative keeps this 1:1 (one authoritative - # link per live record, an invariant the mapping job upholds). - .outerjoin( - MappingRecordAllele, - and_( - MappingRecordAllele.mapping_record_id == MappingRecord.id, - MappingRecordAllele.is_authoritative.is_(True), - MappingRecordAllele.live_at(as_of), - ), + .select_from(projection_link) + .join(projection_allele, projection_allele.id == projection_link.allele_id) + .where( + projection_link.mapping_record_id == record.c.id, + projection_link.projection_group == authoritative.c.projection_group, + ~projection_link.is_authoritative, + projection_link.live_at(as_of), ) - # The allele row itself (digest, ClinGen id). No live_at: alleles are immutable/deduplicated, - # not ValidTime — the live link already establishes what applies. - .outerjoin(Allele, Allele.id == MappingRecordAllele.allele_id) - # Its VEP consequence. - .outerjoin( - VepAlleleConsequence, and_(VepAlleleConsequence.allele_id == Allele.id, VepAlleleConsequence.live_at(as_of)) + .order_by(projection_link.id) + .limit(1) + .lateral("projection") + ) + + # The record's protein-level allele (the protein apex for a nucleotide assay). Absent for UTR/intronic. + protein_link = aliased(MappingRecordAllele) + protein_allele = aliased(Allele) + protein = ( + select(protein_allele.hgvs_p.label("protein_hgvs")) + .select_from(protein_link) + .join(protein_allele, protein_allele.id == protein_link.allele_id) + .where( + protein_link.mapping_record_id == record.c.id, + protein_link.live_at(as_of), + protein_allele.level == SequenceLevel.protein.value, ) - # The authoritative link's projection_group partner (other nucleotide level). Unlike the protein - # subquery, this is safe as a direct join: a group has ≤2 members, so at most one match, and the - # mapping_record_id predicate uses an existing index — no O(N^2) rescan. A NULL group (protein - # assay, or pre-reverse-translation data) matches nothing, so the slot degrades to None. - .outerjoin( - projection_link, - and_( - projection_link.mapping_record_id == MappingRecord.id, - projection_link.projection_group == MappingRecordAllele.projection_group, - projection_link.is_authoritative.is_(False), - projection_link.live_at(as_of), - ), + .order_by(protein_allele.id) + .limit(1) + .lateral("protein") + ) + + statement = ( + select( + Variant.urn, + Variant.data, + Variant.hgvs_nt, + Variant.hgvs_pro, + Variant.hgvs_splice, + record.c.assay_level, + record.c.hgvs_assay_level, + authoritative.c.vrs_digest, + authoritative.c.clingen_allele_id, + authoritative.c.functional_consequence, + projection.c.projection_level, + projection.c.projection_hgvs, + protein.c.protein_hgvs, ) - .outerjoin(projection_allele, projection_allele.id == projection_link.allele_id) - # The anchor: just this score set's variants. + .select_from(Variant) + # LEFT joins so unmapped variants stay (null mapped fields) for the table + score histogram. + .outerjoin(record, true()) + .outerjoin(authoritative, true()) + .outerjoin(projection, true()) + .outerjoin(protein, true()) .where(Variant.score_set_id == score_set.id) # Natural table order: variant number (the integer after '#'), id breaks ties stably. .order_by(Variant.variant_number, Variant.id) ) - # Runs as its own query, not a join (see get_protein_hgvs_by_record), stitched back below. - protein_hgvs_by_record = get_protein_hgvs_by_record(db, score_set.id, as_of=as_of) - return [ LeanVariantRecord( variant_urn=row.urn, @@ -312,8 +353,8 @@ def get_lean_score_set_variants( assay_level_hgvs=row.hgvs_assay_level, projection_level=row.projection_level, projection_hgvs=row.projection_hgvs, - protein_hgvs=protein_hgvs_by_record.get(row.mapping_record_id), + protein_hgvs=row.protein_hgvs, ), ) - for row in db.execute(base_statement) + for row in db.execute(statement) ] diff --git a/tests/helpers/util/query_plan.py b/tests/helpers/util/query_plan.py new file mode 100644 index 000000000..ea9cd9afc --- /dev/null +++ b/tests/helpers/util/query_plan.py @@ -0,0 +1,57 @@ +"""Query-plan assertions for reads that must stay O(variants) regardless of planner statistics. + +A score-set read that joins a large per-variant table can flip from per-row index lookups to a hash join +over the whole table once the planner's row estimate crosses its cost threshold. These helpers EXPLAIN a +captured statement with nested loops and sequential scans disabled, so any join the planner *could* turn +into a whole-table read does, and report scans of the given tables that are not keyed on the outer row. +""" + +from contextlib import contextmanager +from typing import Iterator, Mapping + +from sqlalchemy import event, text + + +@contextmanager +def captured_statements(session) -> Iterator[list[tuple[str, object]]]: + """Collect every ``(statement, parameters)`` the session's engine sends while the block runs.""" + captured: list[tuple[str, object]] = [] + engine = session.get_bind() + + def capture(_conn, _cursor, statement, parameters, _context, _executemany): + captured.append((statement, parameters)) + + event.listen(engine, "before_cursor_execute", capture) + try: + yield captured + finally: + event.remove(engine, "before_cursor_execute", capture) + + +def unkeyed_scans(session, statement: str, parameters, keys: Mapping[str, str]) -> list[str]: + """Scans of the tables in ``keys`` whose condition does not constrain the mapped key column. + + ``keys`` maps a table name to the column that must appear as `` =`` in the scan's index or + recheck condition, i.e. the scan is a lookup driven by the outer row rather than a read of the table. + Runs inside the caller's transaction; the ``SET LOCAL``s end with it. + """ + session.execute(text("SET LOCAL enable_nestloop = off")) + session.execute(text("SET LOCAL enable_seqscan = off")) + cursor = session.connection().connection.cursor() + cursor.execute("EXPLAIN (FORMAT JSON) " + statement, parameters) + plan = cursor.fetchone()[0][0]["Plan"] + + def walk(node): + yield node + for child in node.get("Plans", []): + yield from walk(child) + + bad = [] + for node in walk(plan): + table = node.get("Relation Name") + if table not in keys: + continue + condition = node.get("Index Cond", "") + node.get("Recheck Cond", "") + if f"{keys[table]} =" not in condition: + bad.append(f"{node['Node Type']} on {table} ({condition or 'no condition'})") + return bad diff --git a/tests/lib/test_score_set_variants.py b/tests/lib/test_score_set_variants.py index 934322954..130e17747 100644 --- a/tests/lib/test_score_set_variants.py +++ b/tests/lib/test_score_set_variants.py @@ -6,7 +6,7 @@ authoritative allele; each HGVS string always rides even when it is not a placeable simple substitution; unmapped variants are retained with null mapped fields; ``as_of`` reconstructs the annotation layer while the immutable submitted HGVS and score stay put; and the set is ordered by -variant number. +variant number. They also pin the query's plan shape: per-variant lookups stay keyed index probes. """ from datetime import datetime, timedelta, timezone @@ -22,6 +22,7 @@ from mavedb.models.variant import Variant from mavedb.models.vep_allele_consequence import VepAlleleConsequence from tests.helpers.constants import TEST_MINIMAL_VARIANT +from tests.helpers.util.query_plan import captured_statements, unkeyed_scans # Deterministic windows far from the transaction clock (mirrors tests/lib/test_alleles.py). T0 = datetime(2020, 1, 1, tzinfo=timezone.utc) @@ -309,3 +310,33 @@ def test_as_of_reconstructs_the_historical_assay_level(session, setup_lib_db_wit assert pre_mapping.assay_level_digest is None assert pre_mapping.assay_level is None assert pre_mapping.mapped == MappedTriple() + + +@pytest.mark.integration +@pytest.mark.parametrize("as_of", [None, T2], ids=["current", "as_of"]) +def test_per_variant_lookups_cannot_become_whole_table_reads(session, setup_lib_db_with_score_set, as_of): + """With nested loops and sequential scans disabled, every read of the per-variant tables must still be + keyed on the outer row: the query must not be able to turn into a hash join over the whole table.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, score=-2.3, hgvs_nt="c.1A>T") + record = _record(session, variant, assay_level="cdna", hgvs_assay_level="NM_000546.6:c.1216G>A", valid_from=T0) + measured = _allele(session, "cdna-digest", level="cdna") + genomic = _allele(session, "gen-digest", level="genomic", hgvs_g="NC_000017.11:g.7676154C>T") + protein = _allele(session, "prot-digest", level="protein", hgvs_p="NP_000537.3:p.Ala406Thr") + _link(session, record, measured, is_authoritative=True, projection_group=0, valid_from=T0) + _link(session, record, genomic, is_authoritative=False, projection_group=0, valid_from=T0) + _link(session, record, protein, is_authoritative=False, valid_from=T0) + + with captured_statements(session) as captured: + get_lean_score_set_variants(session, score_set, as_of=as_of) + [(statement, parameters)] = [(s, p) for s, p in captured if "FROM variants" in s] + + assert ( + unkeyed_scans( + session, + statement, + parameters, + {"mapping_records": "variant_id", "mapping_record_alleles": "mapping_record_id"}, + ) + == [] + ) From 567071af09bb44f90b517290d32fbb620bbde770 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 7 Oct 2026 09:38:21 -0700 Subject: [PATCH 214/252] feat(variant): add a research-use-only classification fallback to allele measurements preferredClassification still never comes from an RUO calibration, so ranking is unchanged. When it is absent, researchUseOnlyClassification carries the RUO pick for display. Measurements tied on evidence now rank those whose experiment provides more assay-fact keywords first. --- src/mavedb/lib/allele_measurements.py | 59 +++++++++++---- src/mavedb/view_models/allele_measurement.py | 14 +++- tests/lib/test_allele_measurements.py | 77 +++++++++++++++----- tests/view_models/test_allele_measurement.py | 38 ++++++++++ 4 files changed, 157 insertions(+), 31 deletions(-) create mode 100644 tests/view_models/test_allele_measurement.py diff --git a/src/mavedb/lib/allele_measurements.py b/src/mavedb/lib/allele_measurements.py index 23ca174d6..ebe0f879d 100644 --- a/src/mavedb/lib/allele_measurements.py +++ b/src/mavedb/lib/allele_measurements.py @@ -46,6 +46,8 @@ from mavedb.lib.variants import variant_score from mavedb.models.allele import Allele from mavedb.models.enums.sequence_level import SequenceLevel +from mavedb.models.experiment import Experiment +from mavedb.models.experiment_controlled_keyword import ExperimentControlledKeywordAssociation from mavedb.models.mapping_record import MappingRecord from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.score_calibration import ScoreCalibration @@ -53,6 +55,7 @@ from mavedb.models.score_calibration_functional_classification_variant_association import ( score_calibration_functional_classification_variants_association_table as classification_variants, ) +from mavedb.models.score_set import ScoreSet from mavedb.models.variant import Variant logger = logging.getLogger(__name__) @@ -71,7 +74,9 @@ class AlleleMeasurement: """One measurement in the query's equivalence class. ``assay_level`` is the sequence level at which this measurement was assayed. ``preferred_classification`` - is the readable classification the UI will default to. + is the readable classification the UI will default to; it never comes from a research-use-only calibration, + so ranking and any consumer treating it as clinical evidence can rely on that. ``research_use_only_classification`` + is a display-only fallback: the RUO pick, set only when ``preferred_classification`` is ``None``. """ variant_urn: str @@ -83,6 +88,7 @@ class AlleleMeasurement: score_set_urn: str score_set_title: str preferred_classification: Optional[ScoreCalibrationFunctionalClassification] + research_use_only_classification: Optional[ScoreCalibrationFunctionalClassification] is_current: bool superseded_by_score_set: Optional[str] @@ -105,13 +111,16 @@ class _ProteinApex: def _preferred_classification( - db: Session, variant: Variant, *, user_data: Optional[UserData] + db: Session, variant: Variant, *, user_data: Optional[UserData], research_use_only: bool = False ) -> Optional[ScoreCalibrationFunctionalClassification]: """The variant's preferred *readable* functional classification, or ``None``. Mirrors the UI's calibration cascade — ``primary`` then ``investigator_provided``, strongest evidence - within a tier, then ``id`` for determinism. RUO calibrations are excluded outright, and calibrations - the caller can't read are skipped rather than blanking a readable call. + within a tier, then ``id`` for determinism. Calibrations the caller can't read are skipped rather than + blanking a readable call. + + Draws from one side of the RUO line only: non-RUO calibrations by default (the call MaveDB asserts, matching + the annotation layer's default), or RUO calibrations alone when ``research_use_only`` is set, for display. """ candidates = [ (classification, calibration) @@ -123,7 +132,7 @@ def _preferred_classification( ) .join(ScoreCalibration, ScoreCalibration.id == ScoreCalibrationFunctionalClassification.calibration_id) .where(classification_variants.c.variant_id == variant.id) - .where(ScoreCalibration.research_use_only.is_(False)) + .where(ScoreCalibration.research_use_only.is_(research_use_only)) ).all() if has_permission(user_data, calibration, Action.READ).permitted ] @@ -142,9 +151,21 @@ def preference(pair: tuple[ScoreCalibrationFunctionalClassification, ScoreCalibr return min(candidates, key=preference)[0] -def _ordering_key(measurement: AlleleMeasurement, published_date: Optional[date]) -> tuple: +# Experiment keywords shown on the variant page's measurement table; how many are filled in breaks evidence ties. +ASSAY_FACT_KEYWORD_KEYS = frozenset( + {"Phenotypic Assay Method", "Molecular Mechanism Assessed", "Phenotypic Assay Model System"} +) + + +def _assay_fact_count(score_set: ScoreSet) -> int: + """How many of :data:`ASSAY_FACT_KEYWORD_KEYS` the score set's experiment provides.""" + keyword_objs = score_set.experiment.keyword_objs if score_set.experiment is not None else [] + return len({obj.controlled_keyword.key for obj in keyword_objs} & ASSAY_FACT_KEYWORD_KEYS) + + +def _ordering_key(measurement: AlleleMeasurement, published_date: Optional[date], assay_fact_count: int) -> tuple: """Sort: current before superseded; direct before related; strongest evidence (pathogenic wins ties) - first; newest-published first; then URN for a stable tiebreak.""" + first; most assay facts provided first; newest-published first; then URN for a stable tiebreak.""" magnitude, direction = classification_evidence_strength(measurement.preferred_classification) return ( 0 if measurement.is_current else 1, @@ -152,6 +173,7 @@ def _ordering_key(measurement: AlleleMeasurement, published_date: Optional[date] 0 if measurement.preferred_classification is not None else 1, -magnitude, direction, + -assay_fact_count, -(published_date.toordinal()) if published_date is not None else 0, measurement.variant_urn, ) @@ -263,13 +285,18 @@ def get_allele_measurements( .join(Allele, Allele.id == authoritative.allele_id) .where(MappingRecord.id.in_(record_ids)) .where(MappingRecord.live_at(as_of)) - .options(joinedload(Variant.score_set)) + .options( + joinedload(Variant.score_set) + .joinedload(ScoreSet.experiment) + .selectinload(Experiment.keyword_objs) + .joinedload(ExperimentControlledKeywordAssociation.controlled_keyword) + ) ) .tuples() .all() ) - measurements: list[tuple[AlleleMeasurement, Optional[date]]] = [] + measurements: list[tuple[AlleleMeasurement, Optional[date], int]] = [] protein_consequence_count = 0 apex_only_unresolved_count = 0 for record, variant, measured_allele in rows: @@ -302,6 +329,7 @@ def get_allele_measurements( apex_only_unresolved_count += 1 assay_level = SequenceLevel(record.assay_level) if record.assay_level else None + preferred = _preferred_classification(db, variant, user_data=user_data) measurement = AlleleMeasurement( variant_urn=variant.urn or "", score=variant_score(variant), @@ -311,11 +339,16 @@ def get_allele_measurements( submitted_hgvs=variant.hgvs_pro if assay_level == SequenceLevel.protein.value else variant.hgvs_nt, score_set_urn=score_set.urn or "", score_set_title=score_set.title or "", - preferred_classification=_preferred_classification(db, variant, user_data=user_data), + preferred_classification=preferred, + research_use_only_classification=( + _preferred_classification(db, variant, user_data=user_data, research_use_only=True) + if preferred is None + else None + ), is_current=is_current, superseded_by_score_set=superseding.urn if superseding is not None else None, ) - measurements.append((measurement, score_set.published_date)) + measurements.append((measurement, score_set.published_date, _assay_fact_count(score_set))) # Publish the apex provenance to the request log so the ambiguous-apex and apex-only-unresolved rates are observable. if apex is not None: @@ -328,5 +361,5 @@ def get_allele_measurements( } ) - measurements.sort(key=lambda pair: _ordering_key(pair[0], pair[1])) - return [measurement for measurement, _ in measurements] + measurements.sort(key=lambda entry: _ordering_key(*entry)) + return [measurement for measurement, _, _ in measurements] diff --git a/src/mavedb/view_models/allele_measurement.py b/src/mavedb/view_models/allele_measurement.py index 0baebaddd..33353df4e 100644 --- a/src/mavedb/view_models/allele_measurement.py +++ b/src/mavedb/view_models/allele_measurement.py @@ -7,6 +7,8 @@ from typing import Optional +from pydantic import model_validator + from mavedb.lib.allele_measurements import MeasurementRelationship from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.view_models.base.base import BaseModel @@ -22,7 +24,9 @@ class AlleleMeasurement(BaseModel): this allele), ``protein_consequence`` (a protein measurement of a nt query's consequence), or ``nucleotide_encoding`` (a nt measurement encoding a protein query). ``preferredClassification`` is the readable functional classification the UI defaults to (primary-first cascade, RUO excluded), omitted - when absent or gated. ``isCurrent`` / + when absent or gated; ranking uses it alone. ``researchUseOnlyClassification`` is a display-only + fallback from research-use-only calibrations, present only when ``preferredClassification`` is absent; + it must not be treated as clinical evidence. ``isCurrent`` / ``supersededByScoreSet`` let a superseded measurement (surfaced only under ``include_superseded``) self-describe; ``supersededByScoreSet`` is the superseding *score set*'s URN. """ @@ -36,8 +40,16 @@ class AlleleMeasurement(BaseModel): score_set_urn: str score_set_title: str preferred_classification: Optional[SavedFunctionalClassification] = None + research_use_only_classification: Optional[SavedFunctionalClassification] = None is_current: bool superseded_by_score_set: Optional[str] = None + @model_validator(mode="after") + def classifications_are_mutually_exclusive(self) -> "AlleleMeasurement": + """The RUO fallback exists only where no preferred call does; both set means the builder broke that.""" + if self.preferred_classification is not None and self.research_use_only_classification is not None: + raise ValueError("researchUseOnlyClassification may only be set when preferredClassification is absent.") + return self + class Config: from_attributes = True diff --git a/tests/lib/test_allele_measurements.py b/tests/lib/test_allele_measurements.py index 04b363e19..64eeb9348 100644 --- a/tests/lib/test_allele_measurements.py +++ b/tests/lib/test_allele_measurements.py @@ -467,6 +467,46 @@ def test_research_use_only_calibrations_are_excluded(session, setup_lib_db_with_ result = get_allele_measurements(session, "CA123", user_data=_user_data(session)) assert result[0].preferred_classification is None + # The RUO pick is still surfaced for display, by the same cascade (strongest evidence within the tier). + assert result[0].research_use_only_classification.oddspaths_ratio == 1000.0 + + +@pytest.mark.integration +def test_research_use_only_classification_absent_when_a_preferred_call_exists(session, setup_lib_db_with_score_set): + """The RUO fallback is display-only: with any non-RUO call present it stays empty, even when the RUO call + is stronger.""" + score_set = setup_lib_db_with_score_set + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + variant = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}) + _link(session, _record(session, variant, assay_level="cdna"), nt, is_authoritative=True) + + _calibration(session, score_set, variants=[variant], primary=False, oddspaths_ratio=1000.0, research_use_only=True) + _calibration(session, score_set, variants=[variant], primary=False, oddspaths_ratio=2.0) + + result = get_allele_measurements(session, "CA123", user_data=_user_data(session)) + + assert result[0].preferred_classification.oddspaths_ratio == 2.0 + assert result[0].research_use_only_classification is None + + +@pytest.mark.integration +def test_research_use_only_classification_does_not_rank(session, setup_lib_db_with_score_set): + """A measurement carrying only an RUO call ranks as unclassified: below one with a non-RUO call, however + strong the RUO evidence.""" + score_set = setup_lib_db_with_score_set + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + ruo_only = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}) + classified = _variant(session, score_set, 2, data={"score_data": {"score": 1.0}}) + _link(session, _record(session, ruo_only, assay_level="cdna"), nt, is_authoritative=True) + _link(session, _record(session, classified, assay_level="cdna"), nt, is_authoritative=True) + + _calibration(session, score_set, variants=[ruo_only], primary=False, oddspaths_ratio=1000.0, research_use_only=True) + _calibration(session, score_set, variants=[classified], primary=False, oddspaths_ratio=2.0) + + result = get_allele_measurements(session, "CA123", user_data=_user_data(session)) + + assert [m.variant_urn for m in result] == [classified.urn, ruo_only.urn] + assert result[1].research_use_only_classification is not None @pytest.mark.integration @@ -563,6 +603,7 @@ def _measurement(urn, *, relationship=MeasurementRelationship.direct, classifica score_set_urn="", score_set_title="", preferred_classification=classification, + research_use_only_classification=None, is_current=is_current, superseded_by_score_set=None, ) @@ -570,37 +611,39 @@ def _measurement(urn, *, relationship=MeasurementRelationship.direct, classifica def test_ordering_key_ranks_every_dimension(): """Pin the full precedence chain of ``_ordering_key`` as a pure unit (no DB, no calibration fixtures): - current → direct → classified → strongest magnitude → pathogenic direction → newest published → urn. + current → direct → classified → strongest magnitude → pathogenic direction → most assay facts → newest + published → urn. Each adjacent pair below differs by exactly the next-lower key, so sorting a shuffled copy must reproduce this exact order.""" strong_path = _oddspaths_classification(100.0) # magnitude |ln 100|, pathogenic strong_benign = _oddspaths_classification(0.01) # equal magnitude, benign weak_path = _oddspaths_classification(2.0) # smaller magnitude, pathogenic - # (measurement, published_date), already in the expected ranked order. + # (measurement, published_date, assay_fact_count), already in the expected ranked order. ranked = [ - (_measurement("u#1", classification=strong_path), date(2024, 1, 1)), # strongest evidence, pathogenic - (_measurement("u#2", classification=strong_benign), date(2024, 1, 1)), # equal magnitude, benign loses - (_measurement("u#3", classification=weak_path), date(2024, 1, 1)), # weaker magnitude - (_measurement("u#4a", classification=None), date(2024, 6, 1)), # unclassified; newer published leads its tie - (_measurement("u#4b", classification=None), date(2024, 1, 1)), # unclassified, older published - (_measurement("u#4c", classification=None), date(2024, 1, 1)), # unclassified, same date → urn tiebreak - (_measurement("u#5", relationship=MeasurementRelationship.protein_consequence), date(2024, 6, 1)), # related - (_measurement("u#6", is_current=False), date(2024, 6, 1)), # superseded sorts last + (_measurement("u#1", classification=strong_path), date(2024, 1, 1), 0), # strongest evidence, pathogenic + (_measurement("u#2", classification=strong_benign), date(2024, 1, 1), 0), # equal magnitude, benign loses + (_measurement("u#3", classification=weak_path), date(2024, 1, 1), 0), # weaker magnitude + (_measurement("u#4", classification=None), date(2024, 1, 1), 3), # unclassified; most assay facts leads + (_measurement("u#5a", classification=None), date(2024, 6, 1), 0), # fewer facts; newer published leads + (_measurement("u#5b", classification=None), date(2024, 1, 1), 0), # unclassified, older published + (_measurement("u#5c", classification=None), date(2024, 1, 1), 0), # same date → urn tiebreak + (_measurement("u#6", relationship=MeasurementRelationship.protein_consequence), date(2024, 6, 1), 3), # related + (_measurement("u#7", is_current=False), date(2024, 6, 1), 3), # superseded sorts last ] - shuffled = [ranked[i] for i in (5, 0, 7, 2, 4, 1, 6, 3)] - ordered = sorted(shuffled, key=lambda pair: _ordering_key(pair[0], pair[1])) + shuffled = [ranked[i] for i in (5, 0, 8, 7, 2, 4, 1, 6, 3)] + ordered = sorted(shuffled, key=lambda entry: _ordering_key(*entry)) - assert [m.variant_urn for m, _ in ordered] == [m.variant_urn for m, _ in ranked] + assert [m.variant_urn for m, _, _ in ordered] == [m.variant_urn for m, _, _ in ranked] def test_ordering_key_missing_published_date_sorts_after_dated(): """A null published date is treated as oldest (``0`` ordinal), so a dated measurement leads an otherwise-identical undated one.""" - dated = (_measurement("u#dated"), date(2024, 1, 1)) - undated = (_measurement("u#undated"), None) + dated = (_measurement("u#dated"), date(2024, 1, 1), 0) + undated = (_measurement("u#undated"), None, 0) - ordered = sorted([undated, dated], key=lambda pair: _ordering_key(pair[0], pair[1])) + ordered = sorted([undated, dated], key=lambda entry: _ordering_key(*entry)) - assert [m.variant_urn for m, _ in ordered] == ["u#dated", "u#undated"] + assert [m.variant_urn for m, _, _ in ordered] == ["u#dated", "u#undated"] diff --git a/tests/view_models/test_allele_measurement.py b/tests/view_models/test_allele_measurement.py new file mode 100644 index 000000000..b74ebfeb6 --- /dev/null +++ b/tests/view_models/test_allele_measurement.py @@ -0,0 +1,38 @@ +import pytest +from pydantic import ValidationError + +from mavedb.view_models.allele_measurement import AlleleMeasurement +from tests.helpers.constants import TEST_SAVED_FUNCTIONAL_RANGE_ABNORMAL, TEST_SAVED_FUNCTIONAL_RANGE_NORMAL + +MEASUREMENT = { + "variantUrn": "urn:mavedb:00000001-a-1#1", + "relationship": "direct", + "scoreSetUrn": "urn:mavedb:00000001-a-1", + "scoreSetTitle": "Score set", + "isCurrent": True, +} + + +@pytest.mark.parametrize( + "classifications", + [ + {}, + {"preferredClassification": TEST_SAVED_FUNCTIONAL_RANGE_NORMAL}, + {"researchUseOnlyClassification": TEST_SAVED_FUNCTIONAL_RANGE_ABNORMAL}, + ], +) +def test_allele_measurement_accepts_at_most_one_classification(classifications): + measurement = AlleleMeasurement.model_validate({**MEASUREMENT, **classifications}) + + assert (measurement.preferred_classification is None) or (measurement.research_use_only_classification is None) + + +def test_allele_measurement_rejects_both_classifications(): + with pytest.raises(ValidationError, match="researchUseOnlyClassification may only be set"): + AlleleMeasurement.model_validate( + { + **MEASUREMENT, + "preferredClassification": TEST_SAVED_FUNCTIONAL_RANGE_NORMAL, + "researchUseOnlyClassification": TEST_SAVED_FUNCTIONAL_RANGE_ABNORMAL, + } + ) From be5aa2c83c87fc788a43d8f44bdc11d98f774401 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 7 Oct 2026 09:38:22 -0700 Subject: [PATCH 215/252] feat(csv): flag research-use-only calibrations instead of prefixing their labels AvailableCsvNamespace gains research_use_only so a picker can tag the entry; the "Research Use Only: " label prefix is dropped. --- src/mavedb/lib/csv/entries.py | 4 ++-- src/mavedb/view_models/csv_namespace.py | 3 +++ tests/lib/csv/test_variant.py | 6 ++++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/mavedb/lib/csv/entries.py b/src/mavedb/lib/csv/entries.py index 113750ff5..ed46c9ec6 100644 --- a/src/mavedb/lib/csv/entries.py +++ b/src/mavedb/lib/csv/entries.py @@ -121,7 +121,7 @@ def calibration_can_annotate(calibration: ScoreCalibration) -> bool: def calibration_namespace_entries(calibrations: Iterable[ScoreCalibration]) -> list[AvailableCsvNamespaceEntry]: """Build labeled entries for calibrations, named by title so a picker can identify them. - Research-use-only calibrations (labelled with a prefix) and rangeless ones are offered but excluded + Research-use-only calibrations (flagged for a picker to tag) and rangeless ones are offered but excluded from the default selection. """ entries = [] @@ -134,7 +134,7 @@ def calibration_namespace_entries(calibrations: Iterable[ScoreCalibration]) -> l entries.append( AvailableCsvNamespaceEntry( namespace=calibration_namespace_for_urn(str(calibration.urn)), - label=f"Research Use Only: {title}" if research_use_only else title, + label=title, group=CsvNamespaceGroup.CALIBRATION, score_set=calibration.score_set, research_use_only=research_use_only, diff --git a/src/mavedb/view_models/csv_namespace.py b/src/mavedb/view_models/csv_namespace.py index e4048cf2e..3a868ecf1 100644 --- a/src/mavedb/view_models/csv_namespace.py +++ b/src/mavedb/view_models/csv_namespace.py @@ -30,6 +30,9 @@ class AvailableCsvNamespace(BaseModel): A picker should group calibrations by this when a response spans more than one score set. """ + research_use_only: bool = False + """Whether this is a research-use-only calibration, for a picker to tag beside the label.""" + selected_by_default: bool = True """Whether a picker should open with this group checked. diff --git a/tests/lib/csv/test_variant.py b/tests/lib/csv/test_variant.py index 66f37dcab..a2e30c313 100644 --- a/tests/lib/csv/test_variant.py +++ b/tests/lib/csv/test_variant.py @@ -372,7 +372,8 @@ def test_research_use_only_calibration_is_offered_but_labelled(self, session, se if entry.namespace == CALIBRATION_NS_1 ) - assert entry.label == "Research Use Only: Provisional Calibration" + assert entry.label == "Provisional Calibration" + assert entry.research_use_only is True assert entry.selected_by_default is False def test_research_use_only_calibration_is_not_in_the_default_download( @@ -915,7 +916,8 @@ def test_research_use_only_calibration_is_offered_unchecked(self, session, setup if entry.namespace == CALIBRATION_NS_1 ) - assert entry.label == "Research Use Only: Provisional Calibration" + assert entry.label == "Provisional Calibration" + assert entry.research_use_only is True assert entry.selected_by_default is False # Reported in its own right, not left to be inferred from the label or from the unchecked box: # this is the one reason for unchecking that decides whether the data may be published. From 6209dc7a257ca2b5a797dd9e28fe12df82ad30aa Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 7 Oct 2026 09:38:22 -0700 Subject: [PATCH 216/252] fix(uniprot): submit unversioned NM_ accessions to ID mapping UniProt's RefSeq_Nucleotide index matches versioned accessions exactly and lags RefSeq releases, so current transcript versions often returned no results. The submitted form is stored so polling still matches the accession UniProt echoes back. --- src/mavedb/lib/uniprot/utils.py | 22 +++++ .../worker/jobs/external_services/uniprot.py | 7 +- tests/helpers/constants.py | 1 + tests/lib/uniprot/test_utils.py | 22 ++++- .../jobs/external_services/test_uniprot.py | 83 +++++++++++++++---- 5 files changed, 117 insertions(+), 18 deletions(-) diff --git a/src/mavedb/lib/uniprot/utils.py b/src/mavedb/lib/uniprot/utils.py index ec5a8a185..15c96faf3 100644 --- a/src/mavedb/lib/uniprot/utils.py +++ b/src/mavedb/lib/uniprot/utils.py @@ -31,3 +31,25 @@ def infer_db_name_from_sequence_accession( raise NotImplementedError( "Only RefSeq (NM_/NP_) and Ensembl (ENSP/ENST) identifiers are currently supported for inference." ) + + +def id_mapping_query_accession(sequence_accession: str, from_db: str) -> str: + """ + Returns the form of a sequence accession to submit to UniProt ID mapping. + + UniProt's RefSeq_Nucleotide index matches versioned NM_ accessions exactly and lags RefSeq + releases (e.g. it holds NM_002878.3 while RefSeq and the UniProtKB entry are at NM_002878.4), + so current transcript versions often return no results. Unversioned NM_ accessions match + reliably. The other supported databases match versioned accessions, so those pass through. + + Args: + sequence_accession (str): The sequence accession to submit. + from_db (str): The UniProt ID mapping database the accession belongs to. + + Returns: + str: The accession to submit. + """ + if from_db == "RefSeq_Nucleotide": + return sequence_accession.split(".", 1)[0] + + return sequence_accession diff --git a/src/mavedb/worker/jobs/external_services/uniprot.py b/src/mavedb/worker/jobs/external_services/uniprot.py index e16984292..5e7a5ab2a 100644 --- a/src/mavedb/worker/jobs/external_services/uniprot.py +++ b/src/mavedb/worker/jobs/external_services/uniprot.py @@ -17,7 +17,7 @@ from mavedb.lib.mapping import extract_ids_from_post_mapped_metadata from mavedb.lib.types.workflow import JobExecutionOutcome from mavedb.lib.uniprot.id_mapping import UniProtIDMappingAPI -from mavedb.lib.uniprot.utils import infer_db_name_from_sequence_accession +from mavedb.lib.uniprot.utils import id_mapping_query_accession, infer_db_name_from_sequence_accession from mavedb.models.enums.job_pipeline import FailureCategory from mavedb.models.job_dependency import JobDependency from mavedb.models.score_set import ScoreSet @@ -123,8 +123,9 @@ async def submit_uniprot_mapping_jobs_for_score_set( ) continue - ac_to_map = acs[0] - from_db = infer_db_name_from_sequence_accession(ac_to_map) + from_db = infer_db_name_from_sequence_accession(acs[0]) + # Store the submitted form: polling matches results by the accession UniProt echoes back. + ac_to_map = id_mapping_query_accession(acs[0], from_db) spawned_job = uniprot_api.submit_id_mapping(from_db, "UniProtKB", [ac_to_map]) # type: ignore # Explicitly cast ints to strs in mapping job keys. These are converted to strings internally diff --git a/tests/helpers/constants.py b/tests/helpers/constants.py index c630c491a..eb52f0bc2 100644 --- a/tests/helpers/constants.py +++ b/tests/helpers/constants.py @@ -41,6 +41,7 @@ VALID_CHR_ACCESSION = "NC_000001.11" VALID_ACCESSION = "NM_001637.3" VALID_NT_ACCESSION = "NM_001637.3" +VALID_NT_ACCESSION_UNVERSIONED = "NM_001637" VALID_PRO_ACCESSION = "NP_001637.4" VALID_GENE = "BRCA1" VALID_UNIPROT_ACCESSION = "P05067" diff --git a/tests/lib/uniprot/test_utils.py b/tests/lib/uniprot/test_utils.py index 5d51c2afe..d0643623b 100644 --- a/tests/lib/uniprot/test_utils.py +++ b/tests/lib/uniprot/test_utils.py @@ -1,6 +1,6 @@ import pytest -from mavedb.lib.uniprot.utils import infer_db_name_from_sequence_accession +from mavedb.lib.uniprot.utils import id_mapping_query_accession, infer_db_name_from_sequence_accession from tests.helpers.constants import ( VALID_CHR_ACCESSION, VALID_NT_ACCESSION, @@ -47,3 +47,23 @@ def test_infer_db_name_from_non_refseq_accession(): def test_infer_db_name_from_sequence_accession_empty_string(): with pytest.raises(NotImplementedError): infer_db_name_from_sequence_accession("") + + +def test_id_mapping_query_accession_strips_refseq_nucleotide_version(): + assert id_mapping_query_accession("NM_002878.4", "RefSeq_Nucleotide") == "NM_002878" + + +def test_id_mapping_query_accession_unversioned_refseq_nucleotide_unchanged(): + assert id_mapping_query_accession("NM_002878", "RefSeq_Nucleotide") == "NM_002878" + + +@pytest.mark.parametrize( + "accession, from_db", + [ + ("NP_002869.3", "RefSeq_Protein"), + ("ENST00000345365.11", "Ensembl_Transcript"), + ("ENSP00000338790.6", "Ensembl_Protein"), + ], +) +def test_id_mapping_query_accession_keeps_version_for_other_databases(accession, from_db): + assert id_mapping_query_accession(accession, from_db) == accession diff --git a/tests/worker/jobs/external_services/test_uniprot.py b/tests/worker/jobs/external_services/test_uniprot.py index 8f1bf1304..8d58e607c 100644 --- a/tests/worker/jobs/external_services/test_uniprot.py +++ b/tests/worker/jobs/external_services/test_uniprot.py @@ -19,6 +19,8 @@ TEST_UNIPROT_ID_MAPPING_SWISS_PROT_RESPONSE, TEST_UNIPROT_SWISS_PROT_TYPE, VALID_NT_ACCESSION, + VALID_NT_ACCESSION_UNVERSIONED, + VALID_PRO_ACCESSION, VALID_UNIPROT_ACCESSION, ) @@ -162,7 +164,7 @@ async def test_submit_uniprot_mapping_jobs_no_jobs_submitted( # Verify that the job metadata contains no submitted jobs session.refresh(sample_submit_uniprot_mapping_jobs_run) assert sample_submit_uniprot_mapping_jobs_run.metadata_["submitted_jobs"] == { - "1": {"job_id": None, "accession": VALID_NT_ACCESSION} + "1": {"job_id": None, "accession": VALID_NT_ACCESSION_UNVERSIONED} } async def test_submit_uniprot_mapping_jobs_api_failure_raises( @@ -239,7 +241,7 @@ async def test_submit_uniprot_mapping_jobs_raises_dependent_job_not_available( # Verify that the job metadata contains the submitted jobs (which were submitted before the error) session.refresh(sample_submit_uniprot_mapping_jobs_run) assert sample_submit_uniprot_mapping_jobs_run.metadata_["submitted_jobs"] == { - "1": {"job_id": "job_12345", "accession": VALID_NT_ACCESSION} + "1": {"job_id": "job_12345", "accession": VALID_NT_ACCESSION_UNVERSIONED} } async def test_submit_uniprot_mapping_jobs_successful_submission( @@ -277,7 +279,7 @@ async def test_submit_uniprot_mapping_jobs_successful_submission( assert isinstance(job_result, JobExecutionOutcome) assert job_result.status == JobStatus.SUCCEEDED - expected_submitted_jobs = {"1": {"job_id": "job_12345", "accession": VALID_NT_ACCESSION}} + expected_submitted_jobs = {"1": {"job_id": "job_12345", "accession": VALID_NT_ACCESSION_UNVERSIONED}} # Verify that the job metadata contains the submitted job session.refresh(sample_submit_uniprot_mapping_jobs_run) @@ -287,6 +289,59 @@ async def test_submit_uniprot_mapping_jobs_successful_submission( session.refresh(sample_dummy_polling_job_for_submission_run) assert sample_dummy_polling_job_for_submission_run.job_params["mapping_jobs"] == expected_submitted_jobs + @pytest.mark.parametrize( + "post_mapped_metadata, expected_from_db, expected_accession", + [ + ( + {"cdna": {"sequence_accessions": [VALID_NT_ACCESSION]}}, + "RefSeq_Nucleotide", + VALID_NT_ACCESSION_UNVERSIONED, + ), + ({"protein": {"sequence_accessions": [VALID_PRO_ACCESSION]}}, "RefSeq_Protein", VALID_PRO_ACCESSION), + ], + ) + async def test_submit_uniprot_mapping_jobs_submits_query_form_of_accession( + self, + session, + mock_worker_ctx, + with_populated_domain_data, + with_submit_uniprot_mapping_job, + with_dummy_polling_job_for_submission_run, + sample_score_set, + sample_submit_uniprot_mapping_jobs_run, + sample_dummy_polling_job_for_submission_run, + post_mapped_metadata, + expected_from_db, + expected_accession, + ): + """NM_ accessions are submitted unversioned, other accessions as-is, and the submitted + form is what the polling job receives to match results against.""" + target_gene = sample_score_set.target_genes[0] + target_gene.post_mapped_metadata = post_mapped_metadata + session.commit() + + with patch( + "mavedb.worker.jobs.external_services.uniprot.UniProtIDMappingAPI.submit_id_mapping", + return_value="job_12345", + ) as mock_submit: + job_result = await submit_uniprot_mapping_jobs_for_score_set( + mock_worker_ctx, + 1, + JobManager( + db=session, + redis=mock_worker_ctx["redis"], + job_id=sample_submit_uniprot_mapping_jobs_run.id, + ), + ) + + assert job_result.status == JobStatus.SUCCEEDED + mock_submit.assert_called_once_with(expected_from_db, "UniProtKB", [expected_accession]) + + session.refresh(sample_dummy_polling_job_for_submission_run) + assert sample_dummy_polling_job_for_submission_run.job_params["mapping_jobs"] == { + "1": {"job_id": "job_12345", "accession": expected_accession} + } + async def test_submit_uniprot_mapping_jobs_partial_submission( self, session, @@ -335,7 +390,7 @@ async def test_submit_uniprot_mapping_jobs_partial_submission( assert job_result.status == JobStatus.SUCCEEDED expected_submitted_jobs = { - "1": {"job_id": "job_12345", "accession": VALID_NT_ACCESSION}, + "1": {"job_id": "job_12345", "accession": VALID_NT_ACCESSION_UNVERSIONED}, "2": {"job_id": None, "accession": "NM_000546"}, } @@ -385,7 +440,7 @@ async def test_submit_uniprot_mapping_jobs_success_independent_ctx( assert isinstance(job_result, JobExecutionOutcome) assert job_result.status == JobStatus.SUCCEEDED - expected_submitted_jobs = {"1": {"job_id": "job_12345", "accession": VALID_NT_ACCESSION}} + expected_submitted_jobs = {"1": {"job_id": "job_12345", "accession": VALID_NT_ACCESSION_UNVERSIONED}} # Verify that the job metadata contains the submitted job session.refresh(sample_submit_uniprot_mapping_jobs_run) @@ -436,7 +491,7 @@ async def test_submit_uniprot_mapping_jobs_success_pipeline_ctx( assert isinstance(job_result, JobExecutionOutcome) assert job_result.status == JobStatus.SUCCEEDED - expected_submitted_jobs = {"1": {"job_id": "job_12345", "accession": VALID_NT_ACCESSION}} + expected_submitted_jobs = {"1": {"job_id": "job_12345", "accession": VALID_NT_ACCESSION_UNVERSIONED}} # Verify that the job metadata contains the submitted job session.refresh(sample_submit_uniprot_mapping_jobs_run_in_pipeline) @@ -662,7 +717,7 @@ async def test_submit_uniprot_mapping_jobs_no_jobs_submitted( # Verify that the job metadata contains no submitted jobs session.refresh(sample_submit_uniprot_mapping_jobs_run) assert sample_submit_uniprot_mapping_jobs_run.metadata_["submitted_jobs"] == { - "1": {"job_id": None, "accession": VALID_NT_ACCESSION} + "1": {"job_id": None, "accession": VALID_NT_ACCESSION_UNVERSIONED} } # Verify that the submission job was completed successfully @@ -697,8 +752,8 @@ async def test_submit_uniprot_mapping_jobs_partial_submission( session.commit() # Add accessions to both target genes' post mapped metadata - for idx, tg in enumerate(sample_score_set.target_genes): - tg.post_mapped_metadata = {"protein": {"sequence_accessions": [VALID_NT_ACCESSION + f"{idx:05d}"]}} + for tg, accession in zip(sample_score_set.target_genes, [VALID_NT_ACCESSION, "NM_000546.6"]): + tg.post_mapped_metadata = {"protein": {"sequence_accessions": [accession]}} session.commit() with ( @@ -715,8 +770,8 @@ async def test_submit_uniprot_mapping_jobs_partial_submission( assert job_result.status == JobStatus.SUCCEEDED expected_submitted_jobs = { - "1": {"job_id": "job_12345", "accession": VALID_NT_ACCESSION + "00000"}, - "2": {"job_id": None, "accession": VALID_NT_ACCESSION + "00001"}, + "1": {"job_id": "job_12345", "accession": VALID_NT_ACCESSION_UNVERSIONED}, + "2": {"job_id": None, "accession": "NM_000546"}, } # Verify that the job metadata contains both submitted and failed jobs @@ -765,7 +820,7 @@ async def test_submit_uniprot_mapping_jobs_no_dependent_job_raises( # Verify that the job metadata contains the job we submitted before the error session.refresh(sample_submit_uniprot_mapping_jobs_run) assert sample_submit_uniprot_mapping_jobs_run.metadata_["submitted_jobs"] == { - "1": {"job_id": "job_12345", "accession": VALID_NT_ACCESSION} + "1": {"job_id": "job_12345", "accession": VALID_NT_ACCESSION_UNVERSIONED} } # Verify that the submission job failed @@ -810,7 +865,7 @@ async def test_submit_uniprot_mapping_jobs_with_arq_context_independent( await arq_worker.async_run() await arq_worker.run_check() - expected_submitted_jobs = {"1": {"job_id": "job_12345", "accession": VALID_NT_ACCESSION}} + expected_submitted_jobs = {"1": {"job_id": "job_12345", "accession": VALID_NT_ACCESSION_UNVERSIONED}} # Verify that the job metadata contains the submitted job session.refresh(sample_submit_uniprot_mapping_jobs_run) @@ -859,7 +914,7 @@ async def test_submit_uniprot_mapping_jobs_with_arq_context_pipeline( await arq_worker.async_run() await arq_worker.run_check() - expected_submitted_jobs = {"1": {"job_id": "job_12345", "accession": VALID_NT_ACCESSION}} + expected_submitted_jobs = {"1": {"job_id": "job_12345", "accession": VALID_NT_ACCESSION_UNVERSIONED}} # Verify that the job metadata contains the submitted job session.refresh(sample_submit_uniprot_mapping_jobs_run_in_pipeline) From 040a7c0e18773611997b848b3062321995506b8a Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 7 Oct 2026 09:38:22 -0700 Subject: [PATCH 217/252] fix(scripts): separate expected cases from errors in the reverse-translation audit ClinGen ids shared with a retired allele, and measured indels that are not codon-aligned, are expected outcomes rather than writer errors. Report each as its own info-level check and keep the error-level checks to live alleles and length-preserving changes. --- .../scripts/audit_reverse_translation.py | 78 +++++++++++++++---- .../scripts/test_audit_reverse_translation.py | 64 +++++++++++++++ 2 files changed, 125 insertions(+), 17 deletions(-) diff --git a/src/mavedb/scripts/audit_reverse_translation.py b/src/mavedb/scripts/audit_reverse_translation.py index 3f8155e60..b1ffdbe38 100644 --- a/src/mavedb/scripts/audit_reverse_translation.py +++ b/src/mavedb/scripts/audit_reverse_translation.py @@ -80,6 +80,36 @@ class SqlCheck: scoped: bool = True +# An allele some live mapping record links to. +_ALLELE_IS_LIVE = "EXISTS (SELECT 1 FROM mapping_record_alleles ll WHERE ll.allele_id = a.id AND ll.valid_to IS NULL)" + +# An allele whose state is longer or shorter than the span it replaces. An allele with a non-numeric +# (imprecise) or unreadable location counts as length-preserving, so it stays under the error-level check. +# CASE, not AND, guards the casts: Postgres does not promise to evaluate AND operands in order. +_LENGTH_CHANGING = """coalesce( + CASE WHEN jsonb_typeof(a.post_mapped #> '{location,start}') = 'number' + AND jsonb_typeof(a.post_mapped #> '{location,end}') = 'number' + THEN (a.post_mapped #>> '{location,end}')::int - (a.post_mapped #>> '{location,start}')::int + <> coalesce(length(a.post_mapped #>> '{state,sequence}'), (a.post_mapped #>> '{state,length}')::int) + END, + false +)""" + +_MEASURED_OUTSIDE_GROUP_SQL = """ + SELECT auth.mapping_record_id, auth.allele_id, a.level, coalesce(a.hgvs_g, a.hgvs_c) AS hgvs + FROM mapping_record_alleles auth + JOIN alleles a ON a.id = auth.allele_id + WHERE auth.is_authoritative AND auth.valid_to IS NULL AND auth.projection_group IS NULL + AND a.level IN ('cdna', 'genomic') + AND {length_condition} + {auth_filter} + AND EXISTS (SELECT 1 FROM mapping_record_alleles d + JOIN alleles da ON da.id = d.allele_id + WHERE d.mapping_record_id = auth.mapping_record_id + AND NOT d.is_authoritative AND d.valid_to IS NULL + AND da.level IN ('cdna', 'genomic')) +""" + SQL_CHECKS: tuple[SqlCheck, ...] = ( SqlCheck( name="accession_with_multiple_refgets", @@ -126,18 +156,34 @@ class SqlCheck: name="clingen_allele_id_on_twin_alleles", severity=ERROR, description=( - "One ClinGen allele id sits on more than one allele on the same accession, so two writers built one " + "One ClinGen allele id sits on more than one live allele on the same accession, so two writers built one " "change differently. A CA/PA id spans reference sequences, so the same id across accessions is expected." ), - sql=""" + sql=f""" SELECT a.clingen_allele_id, split_part(coalesce(a.hgvs_g, a.hgvs_c, a.hgvs_p), ':', 1) AS accession, count(*) AS alleles, array_agg(coalesce(a.hgvs_g, a.hgvs_c, a.hgvs_p) ORDER BY a.id) AS hgvs FROM alleles a - WHERE a.clingen_allele_id IS NOT NULL {allele_filter} + WHERE a.clingen_allele_id IS NOT NULL AND {_ALLELE_IS_LIVE} {{allele_filter}} GROUP BY 1, 2 HAVING count(*) > 1 """, ), + SqlCheck( + name="clingen_allele_id_on_retired_twin", + severity=INFO, + description=( + "A ClinGen allele id shared with an allele no live record links to. Re-registration only submits live " + "alleles, so a superseded allele keeps the id it had, including a legacy id copied by variant URN." + ), + sql=f""" + SELECT a.clingen_allele_id, split_part(coalesce(a.hgvs_g, a.hgvs_c, a.hgvs_p), ':', 1) AS accession, + count(*) AS alleles, array_agg(coalesce(a.hgvs_g, a.hgvs_c, a.hgvs_p) ORDER BY a.id) AS hgvs + FROM alleles a + WHERE a.clingen_allele_id IS NOT NULL {{allele_filter}} + GROUP BY 1, 2 + HAVING count(*) > 1 AND count(*) FILTER (WHERE {_ALLELE_IS_LIVE}) <= 1 + """, + ), SqlCheck( name="mapped_record_without_authoritative_link", severity=ERROR, @@ -169,21 +215,19 @@ class SqlCheck: severity=ERROR, description=( "RT attached derived nucleotide alleles to a record, but the measured nucleotide allele is in no " - "projection group: the fold-in missed it, so its convergent siblings read as projections." + "projection group: the fold-in missed it, so its convergent siblings read as projections. Only " + "length-preserving changes, which RT's codon-level candidates always include." ), - sql=""" - SELECT auth.mapping_record_id, auth.allele_id, a.level, coalesce(a.hgvs_g, a.hgvs_c) AS hgvs - FROM mapping_record_alleles auth - JOIN alleles a ON a.id = auth.allele_id - WHERE auth.is_authoritative AND auth.valid_to IS NULL AND auth.projection_group IS NULL - AND a.level IN ('cdna', 'genomic') - {auth_filter} - AND EXISTS (SELECT 1 FROM mapping_record_alleles d - JOIN alleles da ON da.id = d.allele_id - WHERE d.mapping_record_id = auth.mapping_record_id - AND NOT d.is_authoritative AND d.valid_to IS NULL - AND da.level IN ('cdna', 'genomic')) - """, + sql=_MEASURED_OUTSIDE_GROUP_SQL.replace("{length_condition}", f"NOT {_LENGTH_CHANGING}"), + ), + SqlCheck( + name="measured_indel_outside_projection_group", + severity=INFO, + description=( + "A measured indel in no projection group. Expected when the indel is not codon-aligned: it changes the " + "protein the same way as a codon-aligned candidate but is a different DNA change, so RT does not list it." + ), + sql=_MEASURED_OUTSIDE_GROUP_SQL.replace("{length_condition}", _LENGTH_CHANGING), ), SqlCheck( name="malformed_projection_group", diff --git a/tests/scripts/test_audit_reverse_translation.py b/tests/scripts/test_audit_reverse_translation.py index 71b5ae5dd..21ead3c72 100644 --- a/tests/scripts/test_audit_reverse_translation.py +++ b/tests/scripts/test_audit_reverse_translation.py @@ -7,7 +7,10 @@ pytest.importorskip("ga4gh.vrs") from mavedb.lib.seqrepo import SequenceNotFoundError +from sqlalchemy import select + from mavedb.models.annotation_event import AnnotationEvent +from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.variant import Variant from mavedb.scripts.audit_reverse_translation import ( ERROR, @@ -112,6 +115,36 @@ def test_flags_a_measured_allele_left_outside_its_projection_group(self, session assert finding.count == 1 assert finding.samples[0]["hgvs"] == "NM_000002.1:c.5A>G" + def test_reports_a_measured_indel_outside_the_groups_as_info_not_error(self, session, make_score_set): + # A deletion straddling codons 261/262 changes the protein as deleting codon 262 does, but is a + # different DNA change, so RT's codon-aligned candidates do not include it. + score_set, (variant,) = _score_set_with_variants(session, make_score_set, "urn:mavedb:00000011-a-1", 1) + deletion = { + "type": "Allele", + "location": {"type": "SequenceLocation", "start": 67611951, "end": 67611954}, + "state": {"type": "LiteralSequenceExpression", "sequence": ""}, + } + seed_mapping_record( + session, + variant, + alleles=[ + AlleleSpec( + digest="measured", + level="genomic", + is_authoritative=True, + hgvs_g="NC_000016.10:g.67611952_67611954del", + post_mapped=deletion, + ), + AlleleSpec(digest="rt-c", level="cdna", hgvs_c="NM_006565.4:c.784_786del", projection_group=0), + AlleleSpec( + digest="rt-g", level="genomic", hgvs_g="NC_000016.10:g.67611953_67611955del", projection_group=0 + ), + ], + ) + + assert _count(session, "measured_allele_outside_projection_group", [score_set.id]).count == 0 + assert _count(session, "measured_indel_outside_projection_group", [score_set.id]).count == 1 + def test_flags_an_accession_whose_alleles_carry_two_refgets(self, session, make_score_set): score_set, variants = _score_set_with_variants(session, make_score_set, "urn:mavedb:00000003-a-1", 2) for n, (variant, refget) in enumerate(zip(variants, ("SQ.ncbi", "SQ.cdot")), start=1): @@ -162,6 +195,37 @@ def test_flags_one_clingen_allele_id_only_on_twins_on_one_accession(self, sessio assert _count(session, "clingen_allele_id_on_twin_alleles", [score_set.id]).count == flagged + def test_reports_a_twin_on_a_retired_allele_as_info_not_error(self, session, make_score_set): + score_set, variants = _score_set_with_variants(session, make_score_set, "urn:mavedb:00000010-a-1", 2) + records = [ + seed_mapping_record( + session, + variant, + alleles=[ + AlleleSpec( + digest=f"caid-{n}", + level="genomic", + is_authoritative=True, + hgvs_g=expression, + clingen_allele_id="CA999", + ) + ], + ) + for n, (variant, expression) in enumerate( + zip(variants, ("NC_000007.14:g.3043839CA=", "NC_000007.14:g.2947765_2947766delinsCA")), start=1 + ) + ] + retired_at = datetime.now(timezone.utc) + records[0].valid_to = retired_at + for link in session.scalars( + select(MappingRecordAllele).where(MappingRecordAllele.mapping_record_id == records[0].id) + ): + link.valid_to = retired_at + session.commit() + + assert _count(session, "clingen_allele_id_on_twin_alleles", [score_set.id]).count == 0 + assert _count(session, "clingen_allele_id_on_retired_twin", [score_set.id]).count == 1 + def test_flags_a_projection_group_with_one_member(self, session, make_score_set): score_set, (variant,) = _score_set_with_variants(session, make_score_set, "urn:mavedb:00000005-a-1", 1) seed_mapping_record( From 189125dbf25e7dd269ea1f348a0c4a6cd3e2a5a1 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 7 Oct 2026 15:35:53 -0700 Subject: [PATCH 218/252] feat(calibration-controls): store controls only while PHI is affirmed A calibration may hold controls only while controls_not_phi is true. Creating or replacing controls without the affirmation is rejected, and withdrawing it deletes the stored controls. This replaces the publish-time gates, which let unaffirmed controls sit in the database. A score re-upload that drops controls whose variants are gone now keeps the affirmation: the remaining controls are a subset of what was affirmed. --- src/mavedb/lib/score_calibrations.py | 65 ++++--- src/mavedb/lib/types/score_calibrations.py | 4 - src/mavedb/models/score_calibration.py | 6 +- src/mavedb/routers/score_calibrations.py | 36 +--- tests/routers/test_calibration_controls.py | 178 +++++++++++++----- .../jobs/variant_processing/test_creation.py | 7 +- 6 files changed, 188 insertions(+), 108 deletions(-) diff --git a/src/mavedb/lib/score_calibrations.py b/src/mavedb/lib/score_calibrations.py index 8f53effc3..a50ade34c 100644 --- a/src/mavedb/lib/score_calibrations.py +++ b/src/mavedb/lib/score_calibrations.py @@ -131,6 +131,23 @@ def validate_calibration_controls_in_score_set( return [control.variant_urn for control in controls] +def require_phi_affirmation(has_controls: bool, controls_not_phi: Optional[bool]) -> None: + """Reject a calibration state that would store controls without a PHI affirmation. + + A calibration may hold controls only while ``controls_not_phi`` is ``True``; ``None`` (unaddressed) + and ``False`` (declined) both block. Call this before staging any write that sets either side. + + Raises: + ValidationError: If ``has_controls`` and the affirmation is not ``True``. + """ + if has_controls and controls_not_phi is not True: + raise ValidationError( + "Calibration controls cannot be saved until they are affirmed to be free of protected health " + "information (PHI). Set controls_not_phi to true.", + custom_loc=["body", "controlsNotPhi"], + ) + + def build_calibration_controls( db: Session, score_set: ScoreSet, @@ -446,6 +463,9 @@ async def create_score_calibration_in_score_set( if not calibration_create.score_set_urn: raise ValueError("score_set_urn must be provided to create a score calibration within a score set.") + submitted_controls = getattr(calibration_create, "controls", None) + require_phi_affirmation(bool(submitted_controls), getattr(calibration_create, "controls_not_phi", None)) + containing_score_set = db.query(ScoreSet).where(ScoreSet.urn == calibration_create.score_set_urn).one() calibration = await _create_score_calibration(db, calibration_create, user, variant_classes, containing_score_set) @@ -457,9 +477,7 @@ async def create_score_calibration_in_score_set( else: calibration.investigator_provided = False - calibration.controls = build_calibration_controls( - db, containing_score_set, getattr(calibration_create, "controls", None), user - ) + calibration.controls = build_calibration_controls(db, containing_score_set, submitted_controls, user) db.add(calibration) return calibration @@ -613,6 +631,24 @@ async def modify_score_calibration( containing_score_set = db.query(ScoreSet).where(ScoreSet.urn == calibration_update.score_set_urn).one() + # Replace semantics: a provided controls list (even empty) replaces all existing controls, while None + # leaves them untouched. Re-acknowledgment: an explicit controls_not_phi in the request always wins; + # otherwise replacing the controls invalidates the prior affirmation, while leaving them preserves it. + # Resolved up front so a rejected update stages nothing. + submitted_controls = getattr(calibration_update, "controls", None) + if "controls_not_phi" in calibration_update.model_fields_set: + controls_not_phi = calibration_update.controls_not_phi + elif submitted_controls is not None: + controls_not_phi = None + else: + controls_not_phi = calibration.controls_not_phi + + # Withdrawing the affirmation removes the controls stored under it. + if controls_not_phi is not True and submitted_controls is None: + submitted_controls = [] + + require_phi_affirmation(bool(submitted_controls), controls_not_phi) + relation_sources = ( (ScoreCalibrationRelation.threshold, calibration_update.threshold_sources or []), (ScoreCalibrationRelation.evidence, calibration_update.evidence_sources or []), @@ -689,9 +725,6 @@ async def modify_score_calibration( db.add(persisted_functional_range) calibration.functional_classifications.append(persisted_functional_range) - # Replace semantics: a provided controls list (even empty) replaces all existing controls, while - # None leaves them untouched. - submitted_controls = getattr(calibration_update, "controls", None) if submitted_controls is not None: for control in list(calibration.controls): db.delete(control) @@ -699,12 +732,7 @@ async def modify_score_calibration( db.flush() calibration.controls = build_calibration_controls(db, containing_score_set, submitted_controls, user) - # Re-acknowledgment: an explicit controls_not_phi in the request always wins; otherwise, replacing the - # controls invalidates any prior affirmation, while leaving the controls untouched preserves it. - if "controls_not_phi" in calibration_update.model_fields_set: - calibration.controls_not_phi = calibration_update.controls_not_phi - elif submitted_controls is not None: - calibration.controls_not_phi = None + calibration.controls_not_phi = controls_not_phi db.add(calibration) return calibration @@ -1168,11 +1196,9 @@ def restore_calibration_variant_links( A classification always has exactly one of ``range`` or ``class_`` set (enforced by the view models), so every classification falls squarely into one branch or the other. - Dropping a control changes the control set the submitter affirmed as free of protected health - information, so ``controls_not_phi`` is reset to ``None`` on that calibration — the same - re-acknowledgment rule applied when controls are replaced through the API (see - :func:`modify_score_calibration`). A calibration whose controls all relink keeps its affirmation. - Re-binning does not reset it: bin membership carries no clinical annotation and so no PHI. + ``controls_not_phi`` is left as it was. The relinked controls are a subset of the set the + submitter affirmed, so the affirmation still covers them, and controls may only be stored while it + holds (see :func:`require_phi_affirmation`). Changes are staged on the session; the caller commits. @@ -1220,6 +1246,7 @@ def restore_calibration_variant_links( for control in snapshot.controls: control_variant = variants_by_identity.get(control.identity) if control_variant is None: + # Dropping leaves controls_not_phi untouched; the survivors are still covered by it. dropped_controls += 1 continue @@ -1264,10 +1291,6 @@ def restore_calibration_variant_links( report.classification_members_dropped += dropped_members - if dropped_controls: - calibration.controls_not_phi = None - report.calibrations_pending_phi_reaffirmation.append(calibration_id) - # Re-binning is the expected mechanical consequence of new scores, so it does not count as an # edit. Losing a hand-entered reference does. if dropped_controls or dropped_members: diff --git a/src/mavedb/lib/types/score_calibrations.py b/src/mavedb/lib/types/score_calibrations.py index b6b4753bc..06212e825 100644 --- a/src/mavedb/lib/types/score_calibrations.py +++ b/src/mavedb/lib/types/score_calibrations.py @@ -69,9 +69,6 @@ class CalibrationVariantRelinkReport: classifications_rebinned: int = 0 classification_members_rebinned: int = 0 - # Calibrations whose ``controls_not_phi`` affirmation was cleared because a control was dropped. - calibrations_pending_phi_reaffirmation: list[int] = field(default_factory=list) - def to_dict(self) -> dict[str, Any]: """Return a JSON-serializable summary suitable for a job's logging context.""" return { @@ -81,5 +78,4 @@ def to_dict(self) -> dict[str, Any]: "calibration_classification_members_dropped": self.classification_members_dropped, "calibration_classifications_rebinned": self.classifications_rebinned, "calibration_classification_members_rebinned": self.classification_members_rebinned, - "calibrations_pending_phi_reaffirmation": list(self.calibrations_pending_phi_reaffirmation), } diff --git a/src/mavedb/models/score_calibration.py b/src/mavedb/models/score_calibration.py index 29b7425a0..2b7ecc3cc 100644 --- a/src/mavedb/models/score_calibration.py +++ b/src/mavedb/models/score_calibration.py @@ -48,9 +48,9 @@ class ScoreCalibration(Base): disease_term_id = Column(Integer, ForeignKey("mondo_terms.id"), nullable=False, index=True) disease_term: Mapped["MondoTerm"] = relationship("MondoTerm") - # Submitter's affirmation that the control data contains no PHI. Tristate on purpose: # None = not yet addressed, - # False = explicitly declined, True = affirmed. Publishing a calibration that has controls is gated on this being - # True (see #752). + # Submitter's affirmation that the control data contains no PHI. Tristate on purpose: None = not yet addressed, + # False = explicitly declined, True = affirmed. Controls may be stored only while this is True; withdrawing it + # deletes them (see lib.score_calibrations.require_phi_affirmation, #752). controls_not_phi = Column(Boolean, nullable=True) functional_classifications: Mapped[list["ScoreCalibrationFunctionalClassification"]] = relationship( diff --git a/src/mavedb/routers/score_calibrations.py b/src/mavedb/routers/score_calibrations.py index 193fd0a9a..78217fee2 100644 --- a/src/mavedb/routers/score_calibrations.py +++ b/src/mavedb/routers/score_calibrations.py @@ -323,7 +323,8 @@ async def create_score_calibration_route( The `controls_file` parameter accepts a CSV of calibration controls with a variant column (one of `variant_urn`, `hgvs_nt`, `hgvs_pro`) and a `clinical_status` column (`pathogenic` or `benign`, case-insensitive). Controls may be supplied either via this file or inline in - `calibration_json`, but not both. + `calibration_json`, but not both. Controls are accepted only with `controls_not_phi: true`, the + submitter's affirmation that they contain no protected health information (PHI). ## Response Returns the created score calibration with its generated URN and associated score set information. @@ -520,6 +521,10 @@ async def modify_score_calibration_route( Controls may be supplied either via this file or inline in `calibration_json`, but not both; omitting both leaves existing controls unchanged. + Controls are stored only while `controls_not_phi` is true. New controls must be submitted with + `controls_not_phi: true` in the same request; setting it to false or null without new controls + deletes the existing controls. + ## Response Returns the updated score calibration with all modifications applied and any new classification data from the uploaded file. @@ -620,22 +625,6 @@ async def modify_score_calibration_route( detail=[{"loc": e.custom_loc or ["body"], "msg": str(e), "type": "value_error"}], ) - # A public calibration may not carry unacknowledged controls, on this route the same as at - # publish time. controls_not_phi is tristate: only True clears the gate — None (unaddressed) and - # False (declined) both block. Only meaningful when controls exist. - if ( - not updated_calibration.private - and updated_calibration.controls - and updated_calibration.controls_not_phi is not True - ): - raise HTTPException( - status_code=422, - detail=( - "This calibration has controls that have not been affirmed to be free of protected health " - "information (PHI). Set controls_not_phi to true." - ), - ) - db.commit() db.refresh(updated_calibration) @@ -823,19 +812,6 @@ def publish_score_calibration_route( logger.debug("The requested score calibration is already public", extra=logging_context()) return item - # Control data must be affirmed free of PHI before it can be made public. The affirmation is only - # meaningful when controls exist, so zero-control calibrations publish regardless. controls_not_phi - # is tristate: only True clears the gate — None (unaddressed) and False (declined) both block. - if item.controls and item.controls_not_phi is not True: - logger.debug("Calibration controls have not been affirmed free of PHI", extra=logging_context()) - raise HTTPException( - status_code=422, - detail=( - "This calibration has controls that have not been affirmed to be free of protected health " - "information (PHI). Set controls_not_phi to true before publishing." - ), - ) - if item.score_set.private: logger.debug( "Score calibrations associated with private score sets cannot be published", extra=logging_context() diff --git a/tests/routers/test_calibration_controls.py b/tests/routers/test_calibration_controls.py index cc4b98a6f..98ca322ec 100644 --- a/tests/routers/test_calibration_controls.py +++ b/tests/routers/test_calibration_controls.py @@ -105,24 +105,6 @@ def _attach_control(session, calibration_urn, controls_not_phi): session.commit() -@pytest.mark.parametrize("mock_publication_fetch", [CALIBRATION_PUBLICATIONS], indirect=["mock_publication_fetch"]) -@pytest.mark.parametrize("controls_not_phi", [None, False]) -def test_cannot_publish_calibration_with_controls_when_phi_not_affirmed( - client, setup_router_db, mock_publication_fetch, session, data_provider, data_files, controls_not_phi -): - calibration = _create_private_calibration(client, session, data_provider, data_files) - _attach_control(session, calibration["urn"], controls_not_phi=controls_not_phi) - - response = client.post(f"/api/v1/score-calibrations/{calibration['urn']}/publish") - - assert response.status_code == 422 - assert "protected health information" in response.json()["detail"].lower() - - # The gate must not have published the calibration. - refreshed = session.query(CalibrationDbModel).where(CalibrationDbModel.urn == calibration["urn"]).one() - assert refreshed.private is True - - @pytest.mark.parametrize("mock_publication_fetch", [CALIBRATION_PUBLICATIONS], indirect=["mock_publication_fetch"]) def test_can_publish_calibration_with_controls_when_phi_affirmed( client, setup_router_db, mock_publication_fetch, session, data_provider, data_files @@ -161,7 +143,8 @@ def test_can_publish_calibration_without_controls_regardless_of_phi_flag( def _create_with_controls(client, score_set_urn, controls, **extra): response = client.post( - "/api/v1/score-calibrations/", json=_calibration_payload(score_set_urn, controls=controls, **extra) + "/api/v1/score-calibrations/", + json=_calibration_payload(score_set_urn, controls=controls, **{"controlsNotPhi": True, **extra}), ) assert response.status_code == 200, response.text return response.json() @@ -197,7 +180,7 @@ def test_create_calibration_with_controls_file( response = client.post( "/api/v1/score-calibrations/", - data={"calibration_json": json.dumps(_calibration_payload(score_set["urn"]))}, + data={"calibration_json": json.dumps(_calibration_payload(score_set["urn"], controlsNotPhi=True))}, files={"controls_file": ("controls.csv", csv, "text/csv")}, ) @@ -279,7 +262,9 @@ def test_modify_replaces_controls(client, setup_router_db, mock_publication_fetc response = client.put( f"/api/v1/score-calibrations/{calibration['urn']}", json=_calibration_payload( - score_set["urn"], controls=[{"variantUrn": variant_urns[1], "clinicalStatus": "benign"}] + score_set["urn"], + controls=[{"variantUrn": variant_urns[1], "clinicalStatus": "benign"}], + controlsNotPhi=True, ), ) @@ -327,19 +312,15 @@ def test_modify_without_controls_leaves_them_unchanged( @pytest.mark.parametrize("mock_publication_fetch", [CALIBRATION_PUBLICATIONS], indirect=["mock_publication_fetch"]) -def test_modify_controls_without_affirmation_resets_phi_flag( +def test_cannot_replace_controls_without_reaffirming( client, setup_router_db, mock_publication_fetch, session, data_provider, data_files ): score_set, variant_urns = _score_set_with_variant_urns(client, session, data_provider, data_files) calibration = _create_with_controls( - client, - score_set["urn"], - [{"variantUrn": variant_urns[0], "clinicalStatus": "pathogenic"}], - controlsNotPhi=True, + client, score_set["urn"], [{"variantUrn": variant_urns[0], "clinicalStatus": "pathogenic"}] ) - assert calibration["controlsNotPhi"] is True - # Changing controls without re-affirming invalidates the prior affirmation. + # The prior affirmation covered the old controls, not the new ones. response = client.put( f"/api/v1/score-calibrations/{calibration['urn']}", json=_calibration_payload( @@ -347,8 +328,12 @@ def test_modify_controls_without_affirmation_resets_phi_flag( ), ) - assert response.status_code == 200, response.text - assert response.json()["controlsNotPhi"] is None + assert response.status_code == 422 + assert "protected health information" in str(response.json()["detail"]).lower() + + refreshed = session.query(CalibrationDbModel).where(CalibrationDbModel.urn == calibration["urn"]).one() + assert refreshed.controls_not_phi is True + assert [control.variant.urn for control in refreshed.controls] == [variant_urns[0]] @pytest.mark.parametrize("mock_publication_fetch", [CALIBRATION_PUBLICATIONS], indirect=["mock_publication_fetch"]) @@ -360,7 +345,7 @@ def test_modify_controls_with_affirmation_keeps_phi_flag( client, score_set["urn"], [{"variantUrn": variant_urns[0], "clinicalStatus": "pathogenic"}] ) - # Affirming in the same request that changes controls keeps the flag set. + # Re-affirming in the same request that changes controls keeps the flag set. response = client.put( f"/api/v1/score-calibrations/{calibration['urn']}", json=_calibration_payload( @@ -375,16 +360,120 @@ def test_modify_controls_with_affirmation_keeps_phi_flag( ########################################################### -# PHI re-affirmation invariant on published calibrations +# PHI affirmation invariant: controls are stored only while affirmed ########################################################### +@pytest.mark.parametrize("mock_publication_fetch", [CALIBRATION_PUBLICATIONS], indirect=["mock_publication_fetch"]) +@pytest.mark.parametrize("controls_not_phi", [None, False]) +def test_cannot_create_calibration_with_inline_controls_without_phi_affirmation( + client, setup_router_db, mock_publication_fetch, session, data_provider, data_files, controls_not_phi +): + score_set, variant_urns = _score_set_with_variant_urns(client, session, data_provider, data_files) + + response = client.post( + "/api/v1/score-calibrations/", + json=_calibration_payload( + score_set["urn"], + controls=[{"variantUrn": variant_urns[0], "clinicalStatus": "pathogenic"}], + controlsNotPhi=controls_not_phi, + ), + ) + + assert response.status_code == 422 + assert "protected health information" in str(response.json()["detail"]).lower() + assert session.scalars(select(CalibrationControl)).all() == [] + + +@pytest.mark.parametrize("mock_publication_fetch", [CALIBRATION_PUBLICATIONS], indirect=["mock_publication_fetch"]) +def test_cannot_create_calibration_with_controls_file_without_phi_affirmation( + client, setup_router_db, mock_publication_fetch, session, data_provider, data_files +): + score_set, variant_urns = _score_set_with_variant_urns(client, session, data_provider, data_files) + csv = f"{CONTROLS_CSV_HEADER}\n{variant_urns[0]},pathogenic\n" + + response = client.post( + "/api/v1/score-calibrations/", + data={"calibration_json": json.dumps(_calibration_payload(score_set["urn"]))}, + files={"controls_file": ("controls.csv", csv, "text/csv")}, + ) + + assert response.status_code == 422 + assert "protected health information" in str(response.json()["detail"]).lower() + assert session.scalars(select(CalibrationControl)).all() == [] + + +@pytest.mark.parametrize("mock_publication_fetch", [CALIBRATION_PUBLICATIONS], indirect=["mock_publication_fetch"]) +def test_can_create_calibration_without_controls_or_phi_affirmation( + client, setup_router_db, mock_publication_fetch, session, data_provider, data_files +): + score_set, _ = _score_set_with_variant_urns(client, session, data_provider, data_files) + + response = client.post("/api/v1/score-calibrations/", json=_calibration_payload(score_set["urn"])) + + assert response.status_code == 200, response.text + assert response.json()["controlsNotPhi"] is None + + +@pytest.mark.parametrize("mock_publication_fetch", [CALIBRATION_PUBLICATIONS], indirect=["mock_publication_fetch"]) +@pytest.mark.parametrize("controls_not_phi", [None, False]) +def test_withdrawing_phi_affirmation_deletes_controls( + client, setup_router_db, mock_publication_fetch, session, data_provider, data_files, controls_not_phi +): + score_set, variant_urns = _score_set_with_variant_urns(client, session, data_provider, data_files) + calibration = _create_with_controls( + client, + score_set["urn"], + [ + {"variantUrn": variant_urns[0], "clinicalStatus": "pathogenic"}, + {"variantUrn": variant_urns[1], "clinicalStatus": "benign"}, + ], + ) + + response = client.put( + f"/api/v1/score-calibrations/{calibration['urn']}", + json=_calibration_payload(score_set["urn"], controlsNotPhi=controls_not_phi), + ) + + assert response.status_code == 200, response.text + assert response.json()["controls"] == [] + assert response.json()["controlsNotPhi"] is controls_not_phi + assert session.scalars(select(CalibrationControl)).all() == [] + + +@pytest.mark.parametrize("mock_publication_fetch", [CALIBRATION_PUBLICATIONS], indirect=["mock_publication_fetch"]) +def test_cannot_submit_controls_while_withdrawing_phi_affirmation( + client, setup_router_db, mock_publication_fetch, session, data_provider, data_files +): + score_set, variant_urns = _score_set_with_variant_urns(client, session, data_provider, data_files) + calibration = _create_with_controls( + client, score_set["urn"], [{"variantUrn": variant_urns[0], "clinicalStatus": "pathogenic"}] + ) + + response = client.put( + f"/api/v1/score-calibrations/{calibration['urn']}", + json=_calibration_payload( + score_set["urn"], + controls=[{"variantUrn": variant_urns[1], "clinicalStatus": "benign"}], + controlsNotPhi=False, + ), + ) + + assert response.status_code == 422 + assert "protected health information" in str(response.json()["detail"]).lower() + + # A rejected update stages nothing: the original controls and affirmation stand. + refreshed = session.query(CalibrationDbModel).where(CalibrationDbModel.urn == calibration["urn"]).one() + assert refreshed.controls_not_phi is True + assert [control.variant.urn for control in refreshed.controls] == [variant_urns[0]] + + @pytest.mark.parametrize("mock_publication_fetch", [CALIBRATION_PUBLICATIONS], indirect=["mock_publication_fetch"]) def test_create_calibration_ignores_client_supplied_private_field( client, setup_router_db, mock_publication_fetch, session, data_provider, data_files ): - # The create schema has no `private` field — creation always starts private, and publishing - # (which independently gates PHI-bearing controls) is the only path to making one public. + # The create schema has no `private` field — creation always starts private, and publishing is the + # only path to making one public. score_set, variant_urns = _score_set_with_variant_urns(client, session, data_provider, data_files) response = client.post( @@ -393,6 +482,7 @@ def test_create_calibration_ignores_client_supplied_private_field( **_calibration_payload(score_set["urn"]), "private": False, "controls": [{"variantUrn": variant_urns[0], "clinicalStatus": "pathogenic"}], + "controlsNotPhi": True, }, ) @@ -419,7 +509,7 @@ def test_cannot_add_controls_to_published_calibration_without_phi_affirmation_as ) assert response.status_code == 422 - assert "protected health information" in response.json()["detail"].lower() + assert "protected health information" in str(response.json()["detail"]).lower() # The gate must not have attached the controls. refreshed = session.query(CalibrationDbModel).where(CalibrationDbModel.urn == calibration["urn"]).one() @@ -464,8 +554,7 @@ def test_cannot_replace_controls_on_published_calibration_without_reaffirming_as ) publish_test_score_calibration_via_client(client, calibration["urn"]) - # Replacing controls invalidates the prior affirmation just as it does on a private calibration, - # so an admin who doesn't re-affirm in the same request must be rejected here too. + # The invariant holds for published calibrations and for admins alike. with DependencyOverrider(admin_app_overrides): response = client.put( f"/api/v1/score-calibrations/{calibration['urn']}", @@ -475,7 +564,7 @@ def test_cannot_replace_controls_on_published_calibration_without_reaffirming_as ) assert response.status_code == 422 - assert "protected health information" in response.json()["detail"].lower() + assert "protected health information" in str(response.json()["detail"]).lower() # The gate must not have replaced the controls or reset the affirmation. refreshed = session.query(CalibrationDbModel).where(CalibrationDbModel.urn == calibration["urn"]).one() @@ -484,27 +573,24 @@ def test_cannot_replace_controls_on_published_calibration_without_reaffirming_as @pytest.mark.parametrize("mock_publication_fetch", [CALIBRATION_PUBLICATIONS], indirect=["mock_publication_fetch"]) -def test_cannot_decline_phi_affirmation_on_published_calibration_with_existing_controls_as_admin( +def test_withdrawing_phi_affirmation_on_published_calibration_deletes_controls_as_admin( client, setup_router_db, mock_publication_fetch, session, data_provider, data_files, admin_app_overrides ): score_set, variant_urns = _score_set_with_variant_urns(client, session, data_provider, data_files) calibration = _create_with_controls( - client, - score_set["urn"], - [{"variantUrn": variant_urns[0], "clinicalStatus": "pathogenic"}], - controlsNotPhi=True, + client, score_set["urn"], [{"variantUrn": variant_urns[0], "clinicalStatus": "pathogenic"}] ) publish_test_score_calibration_via_client(client, calibration["urn"]) - # Controls are left unchanged; only the affirmation is walked back to False. with DependencyOverrider(admin_app_overrides): response = client.put( f"/api/v1/score-calibrations/{calibration['urn']}", json=_calibration_payload(score_set["urn"], controlsNotPhi=False), ) - assert response.status_code == 422 - assert "protected health information" in response.json()["detail"].lower() + assert response.status_code == 200, response.text + assert response.json()["private"] is False + assert response.json()["controls"] == [] @pytest.mark.parametrize("mock_publication_fetch", [CALIBRATION_PUBLICATIONS], indirect=["mock_publication_fetch"]) diff --git a/tests/worker/jobs/variant_processing/test_creation.py b/tests/worker/jobs/variant_processing/test_creation.py index 5ddf7687f..e791e87a4 100644 --- a/tests/worker/jobs/variant_processing/test_creation.py +++ b/tests/worker/jobs/variant_processing/test_creation.py @@ -1454,7 +1454,7 @@ async def test_controls_relink_to_new_variants_when_every_identity_persists( session.refresh(calibration) assert calibration.controls_not_phi is True - async def test_controls_absent_from_new_upload_are_dropped_and_reset_phi_affirmation( + async def test_controls_absent_from_new_upload_are_dropped_and_keep_phi_affirmation( self, session, with_independent_processing_runs, @@ -1516,13 +1516,12 @@ def _capture_context(self, ctx): assert len(controls) == 1 assert controls[0].variant_id == new_variants["c.1A>T"].id - # A control set that lost a member is no longer the one the submitter affirmed as PHI-free. + # The surviving controls are a subset of the affirmed set, so the affirmation still covers them. session.refresh(calibration) - assert calibration.controls_not_phi is None + assert calibration.controls_not_phi is True assert recorded_context["calibration_controls_relinked"] == 1 assert recorded_context["calibration_controls_dropped"] == 1 - assert recorded_context["calibrations_pending_phi_reaffirmation"] == [calibration.id] async def test_range_based_classification_membership_is_rebinned_against_new_scores( self, From c9c68259e14358212bf2975ee1fcf12b0c9c3488 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 7 Oct 2026 15:35:58 -0700 Subject: [PATCH 219/252] fix(score-calibrations): check READ before ranking a calibration CHANGE_RANK admits score set contributors to community calibrations they may not read, and demote returned a non-primary calibration in full, controls included. Promote and demote now assert READ first. Only public calibrations can be ranked, so no legitimate request is refused. --- src/mavedb/routers/score_calibrations.py | 8 ++++ tests/routers/test_score_calibrations.py | 47 ++++++++++++++++++++++++ 2 files changed, 55 insertions(+) diff --git a/src/mavedb/routers/score_calibrations.py b/src/mavedb/routers/score_calibrations.py index 78217fee2..484575381 100644 --- a/src/mavedb/routers/score_calibrations.py +++ b/src/mavedb/routers/score_calibrations.py @@ -701,6 +701,10 @@ async def promote_score_calibration_to_primary_route( logger.debug("The requested score calibration does not exist", extra=logging_context()) raise HTTPException(status_code=404, detail="The requested score calibration does not exist") + # CHANGE_RANK admits score set contributors to community calibrations they may not READ. Only public + # calibrations can be ranked, so asserting READ first refuses nothing legitimate and keeps this route + # from returning, or describing, a calibration the caller cannot see. + assert_permission(user_data, item, Action.READ) assert_permission(user_data, item, Action.CHANGE_RANK) if item.primary: @@ -765,6 +769,10 @@ def demote_score_calibration_from_primary_route( logger.debug("The requested score calibration does not exist", extra=logging_context()) raise HTTPException(status_code=404, detail="The requested score calibration does not exist") + # CHANGE_RANK admits score set contributors to community calibrations they may not READ. Only public + # calibrations can be ranked, so asserting READ first refuses nothing legitimate and keeps this route + # from returning, or describing, a calibration the caller cannot see. + assert_permission(user_data, item, Action.READ) assert_permission(user_data, item, Action.CHANGE_RANK) if not item.primary: diff --git a/tests/routers/test_score_calibrations.py b/tests/routers/test_score_calibrations.py index dbd6fc3d0..c142ba8e0 100644 --- a/tests/routers/test_score_calibrations.py +++ b/tests/routers/test_score_calibrations.py @@ -5380,3 +5380,50 @@ def test_user_sees_calibrations_across_multiple_score_sets( returned_urns = {c["urn"] for c in calibrations} assert cal_1["urn"] in returned_urns assert cal_2["urn"] in returned_urns + + +@pytest.mark.parametrize( + "mock_publication_fetch", + [ + [ + {"dbName": "PubMed", "identifier": TEST_PUBMED_IDENTIFIER}, + {"dbName": "bioRxiv", "identifier": TEST_BIORXIV_IDENTIFIER}, + ] + ], + indirect=["mock_publication_fetch"], +) +@pytest.mark.parametrize("action", ["promote-to-primary", "demote-from-primary"]) +def test_score_set_owner_cannot_rank_or_see_private_community_calibration( + client, + setup_router_db, + mock_publication_fetch, + session, + data_provider, + data_files, + extra_user_app_overrides, + action, +): + """CHANGE_RANK admits the score set owner to community calibrations, but a private one is not theirs to + read, so the rank routes must not return or describe it.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, + session, + data_provider, + experiment["urn"], + data_files / "scores.csv", + ) + + with patch.object(ArqRedis, "enqueue_job", return_value=None): + score_set = publish_score_set(client, score_set["urn"]) + + with DependencyOverrider(extra_user_app_overrides): + calibration = create_test_score_calibration_in_score_set_via_client( + client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) + ) + assert calibration["investigatorProvided"] is False + + response = client.post(f"/api/v1/score-calibrations/{calibration['urn']}/{action}") + + assert response.status_code == 404 + assert "functionalClassifications" not in response.json() From 205d3ffddadfac9bdbff802352c64fda6bb15ef0 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 7 Oct 2026 15:36:07 -0700 Subject: [PATCH 220/252] feat(alleles): hide alleles that only unreadable score sets link An allele exists only because a score set's mapping produced it, so one produced only by a private score set discloses that someone measured it. GET /alleles/{identifier} now returns 404 unless the caller may read a score set linking the allele, and its equivalence class and projection pairing follow only records of readable score sets. --- src/mavedb/lib/allele_detail.py | 39 ++++++++++----- src/mavedb/lib/alleles.py | 51 +++++++++++++++++-- src/mavedb/routers/alleles.py | 34 ++++++++++--- tests/routers/test_allele.py | 86 +++++++++++++++++++++++++++++---- 4 files changed, 178 insertions(+), 32 deletions(-) diff --git a/src/mavedb/lib/allele_detail.py b/src/mavedb/lib/allele_detail.py index 109703ede..4853f1e19 100644 --- a/src/mavedb/lib/allele_detail.py +++ b/src/mavedb/lib/allele_detail.py @@ -14,15 +14,17 @@ *relative to the focus allele*. **The equivalence class is the full cross-record union** (:func:`lib.alleles.get_allele_translations`): -every allele co-linked to any live record touching the focus. That is the discoverable "everything -related to this change" set. Each member is then labelled relative to the focus allele. Even though this -level is unable to determine which allele was actually measured, we can still say how a given allele relates -to the focus allele (e.g., faithful, candidate, convergent) based on the graph structure. +every allele co-linked to any live record touching the focus, restricted by the caller to records of +score sets they may read. That is the discoverable "everything related to this change" set. Each member +is then labelled relative to the focus allele. Even though this level is unable to determine which +allele was actually measured, we can still say how a given allele relates to the focus allele (e.g., +faithful, candidate, convergent) based on the graph structure. ``as_of`` reconstructs the molecular layer, class membership + annotations, at the past instant. The focus allele's own identity is immutable and content-addressed. ``as_of`` defaults to this instant. """ +from collections.abc import Collection from dataclasses import dataclass from datetime import datetime from typing import Any, Optional @@ -64,7 +66,13 @@ class AlleleDetail: """External annotations for each member of the cross-layer equivalence class, keyed by VRS digest.""" -def _focus_projection_pairs(db: Session, focus_alleles: list[Allele], *, as_of: Optional[datetime]) -> dict[str, str]: +def _focus_projection_pairs( + db: Session, + focus_alleles: list[Allele], + *, + as_of: Optional[datetime], + score_set_ids: Optional[Collection[int]], +) -> dict[str, str]: """A symmetric ``digest -> digest`` map pairing each focus allele with its c↔g projection. A ``projection_group`` is one c↔g pair, local to a single mapping record. The projection is the ≤1 @@ -82,13 +90,16 @@ def _focus_projection_pairs(db: Session, focus_alleles: list[Allele], *, as_of: continue # limit(1) rather than one_or_none(), which would raise for any allele measured more than once. - link = db.scalar( + link_query = ( select(MappingRecordAllele) .where(MappingRecordAllele.allele_id == focus.id) .where(MappingRecordAllele.projection_group.isnot(None)) .where(MappingRecordAllele.live_at(as_of)) - .limit(1) ) + if score_set_ids is not None: + link_query = link_query.where(MappingRecordAllele.score_set_id.in_(score_set_ids)) + + link = db.scalar(link_query.limit(1)) if link is None: continue @@ -138,7 +149,12 @@ def _member_label( def get_allele_detail( - db: Session, focus: Allele, *, focus_digests: set[str], as_of: Optional[datetime] = None + db: Session, + focus: Allele, + *, + focus_digests: set[str], + as_of: Optional[datetime] = None, + score_set_ids: Optional[Collection[int]] = None, ) -> AlleleDetail: """Assemble the allele-detail envelope anchored on ``focus``. @@ -146,15 +162,16 @@ def get_allele_detail( a by-digest fetch, or the (genomic + coding) representations of a CAID for a by-CAID fetch. The equivalence class is every allele co-linked to a live record touching the ``focus`` (:func:`get_allele_translations`). An orphan allele falls back to ``focus`` alone. Non-focus members - are labelled by :func:`_member_label`. + are labelled by :func:`_member_label`. ``score_set_ids``, when given, restricts the equivalence + class and projection pairing to records of those score sets. """ - equivalence_class = get_allele_translations(db, focus.id, as_of=as_of) or [focus] + equivalence_class = get_allele_translations(db, focus.id, as_of=as_of, score_set_ids=score_set_ids) or [focus] annotations = get_allele_annotations(db, equivalence_class, as_of=as_of) focus_alleles = [a for a in equivalence_class if a.vrs_digest in focus_digests] or [focus] focus_is_protein = any(a.level == SequenceLevel.protein.value for a in focus_alleles) - pairs = _focus_projection_pairs(db, focus_alleles, as_of=as_of) + pairs = _focus_projection_pairs(db, focus_alleles, as_of=as_of, score_set_ids=score_set_ids) # The focus's projection(s) that are not themselves focus (for a CAID fetch the projection pair is # jointly the focus, so this is empty and they are simply flagged is_focus instead). projection_digests = {pairs[d] for d in focus_digests if d in pairs} - focus_digests diff --git a/src/mavedb/lib/alleles.py b/src/mavedb/lib/alleles.py index 08f13ef0b..b76a572ee 100644 --- a/src/mavedb/lib/alleles.py +++ b/src/mavedb/lib/alleles.py @@ -8,18 +8,50 @@ ties all three together. """ +from collections.abc import Collection from datetime import datetime from typing import Optional from sqlalchemy import select from sqlalchemy.orm import Session, joinedload +from mavedb.lib.permissions import Action, has_permission +from mavedb.lib.types.authentication import UserData from mavedb.models.allele import Allele from mavedb.models.mapping_record import MappingRecord from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.score_set import ScoreSet from mavedb.models.variant import Variant +def readable_score_set_ids_by_allele( + db: Session, user_data: Optional[UserData], allele_ids: Collection[int] +) -> dict[int, set[int]]: + """Map each of ``allele_ids`` to the ids of the score sets linking it that the caller may read. + + Every allele exists because some score set's mapping produced it, so these are the score sets through + which the caller may see it. An allele with no readable score set is omitted. Links are considered + regardless of valid time: a retired link still records why the allele was created. + """ + if not allele_ids: + return {} + + links = db.execute( + select(MappingRecordAllele.allele_id, MappingRecordAllele.score_set_id) + .where(MappingRecordAllele.allele_id.in_(allele_ids)) + .distinct() + ).all() + score_sets = db.scalars(select(ScoreSet).where(ScoreSet.id.in_({score_set_id for _, score_set_id in links}))).all() + readable = {score_set.id for score_set in score_sets if has_permission(user_data, score_set, Action.READ).permitted} + + by_allele: dict[int, set[int]] = {} + for allele_id, score_set_id in links: + if score_set_id in readable: + by_allele.setdefault(allele_id, set()).add(score_set_id) + + return by_allele + + def get_live_record_allele_links( db: Session, variant_id: int, *, as_of: Optional[datetime] = None ) -> list[MappingRecordAllele]: @@ -80,7 +112,13 @@ def find_variants_by_vrs_identifier( return [(row[0], row[1]) for row in db.execute(stmt).unique().all()] -def get_allele_translations(db: Session, allele_id: int, *, as_of: Optional[datetime] = None) -> list[Allele]: +def get_allele_translations( + db: Session, + allele_id: int, + *, + as_of: Optional[datetime] = None, + score_set_ids: Optional[Collection[int]] = None, +) -> list[Allele]: """Return the cross-layer equivalence set of ``allele_id``: every allele co-linked to a ``MappingRecord`` that links it (the anchor allele itself included), spanning the genomic, coding, and protein layers. @@ -96,12 +134,19 @@ def get_allele_translations(db: Session, allele_id: int, *, as_of: Optional[date the same half-open ``[valid_from, valid_to)`` predicate, so the whole set is evaluated at one instant. The retire-cascade invariant (a live link implies a live record) holds under ``as_of`` too, so filtering the links alone is sufficient. + + ``score_set_ids``, when given, restricts the union to records of those score sets (e.g. the ones the + caller may read). Only the anchor hop needs the filter: every link of a record shares its score set. """ - record_ids = db.scalars( + record_query = ( select(MappingRecordAllele.mapping_record_id) .where(MappingRecordAllele.allele_id == allele_id) .where(MappingRecordAllele.live_at(as_of)) - ).all() + ) + if score_set_ids is not None: + record_query = record_query.where(MappingRecordAllele.score_set_id.in_(score_set_ids)) + + record_ids = db.scalars(record_query).all() if not record_ids: return [] diff --git a/src/mavedb/routers/alleles.py b/src/mavedb/routers/alleles.py index 470765063..dcd90309c 100644 --- a/src/mavedb/routers/alleles.py +++ b/src/mavedb/routers/alleles.py @@ -10,8 +10,11 @@ from mavedb import deps from mavedb.lib.allele_detail import get_allele_detail +from mavedb.lib.alleles import readable_score_set_ids_by_allele +from mavedb.lib.authentication import get_current_user from mavedb.lib.logging import LoggedRoute from mavedb.lib.logging.context import save_to_logging_context +from mavedb.lib.types.authentication import UserData from mavedb.models.allele import Allele from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.routers.shared import ( @@ -93,6 +96,7 @@ def get_allele( ), ), db: Session = Depends(deps.get_db), + user_data: Optional[UserData] = Depends(get_current_user), ): """Fetch the detail envelope for a deduplicated allele, by any of its identifiers. @@ -106,24 +110,38 @@ def get_allele( - a **PAID** (``PA…``) — the protein change; the protein allele is focused and its nucleotide equivalents surface as reverse-translation candidates. - This is a **public molecular resource**. It carries no score-set-level information. No scores, - classifications, measurements, or version standing. Only the allele's own identity, its cross-layer - equivalence class, and public reference annotations (VEP / gnomAD / ClinVar). + The envelope carries no score-set-level information. No scores, classifications, measurements, or + version standing. Only the allele's own identity, its cross-layer equivalence class, and public + reference annotations (VEP / gnomAD / ClinVar). + + An allele is in MaveDB only because a score set's mapping produced it, so it is served only to + callers who may read at least one such score set; otherwise the response is a 404, indistinguishable + from an unknown identifier. The equivalence class is likewise limited to readable score sets. """ save_to_logging_context({"requested_resource": identifier, "as_of": as_of}) response.headers["X-As-Of"] = as_of.isoformat() if as_of is not None else "current" if identifier.startswith("ga4gh:"): allele = db.scalar(select(Allele).where(Allele.vrs_digest == identifier)) - if allele is None: + readable = readable_score_set_ids_by_allele(db, user_data, [allele.id]) if allele is not None else {} + if allele is None or allele.id not in readable: raise HTTPException(status_code=404, detail=f"allele with VRS digest '{identifier}' not found") - return get_allele_detail(db, allele, focus_digests={identifier}, as_of=as_of) + + return get_allele_detail(db, allele, focus_digests={identifier}, as_of=as_of, score_set_ids=readable[allele.id]) # A ClinGen allele id (CAID or PAID): resolve to its allele(s) — the nt-canonical change's genomic + - # coding frames for a CAID, or the single protein allele for a PAID — and focus all of them. - matches = list(db.scalars(select(Allele).where(Allele.clingen_allele_id == identifier)).all()) + # coding frames for a CAID, or the single protein allele for a PAID — and focus the readable ones. + candidates = db.scalars(select(Allele).where(Allele.clingen_allele_id == identifier)).all() + readable = readable_score_set_ids_by_allele(db, user_data, [a.id for a in candidates]) + matches = [a for a in candidates if a.id in readable] if not matches: raise HTTPException(status_code=404, detail=f"no allele with ClinGen allele id '{identifier}' found") focus_digests = {a.vrs_digest for a in matches if a.vrs_digest is not None} - return get_allele_detail(db, _representative(matches), focus_digests=focus_digests, as_of=as_of) + return get_allele_detail( + db, + _representative(matches), + focus_digests=focus_digests, + as_of=as_of, + score_set_ids=set().union(*readable.values()), + ) diff --git a/tests/routers/test_allele.py b/tests/routers/test_allele.py index 7ce80c0c9..728700aab 100644 --- a/tests/routers/test_allele.py +++ b/tests/routers/test_allele.py @@ -3,9 +3,8 @@ Exercise the HTTP surface end to end: the envelope serializes and validates, the equivalence class is labelled relative to the focus, CAID fetch focuses the nt-canonical change, ``as_of`` is echoed and -reconstructs membership, an unknown id is a plain 404 — and, unlike ``GET /variants/{urn}``, the -endpoint is **public with no privacy gate**: an allele reachable only through a private score set is -still served. +reconstructs membership, an unknown id is a plain 404, and an allele reachable only through score sets +the caller may not read is the same 404, with the equivalence class scoped to readable score sets. """ import pytest @@ -14,7 +13,13 @@ cdot = pytest.importorskip("cdot") fastapi = pytest.importorskip("fastapi") +from sqlalchemy import select + +from mavedb.models.allele import Allele +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.score_set import ScoreSet as ScoreSetDbModel +from mavedb.models.variant import Variant from mavedb.view_models.allele_detail import AlleleDetail from tests.helpers.dependency_overrider import DependencyOverrider from tests.helpers.util.annotation import AlleleSpec, seed_mapping_record @@ -189,11 +194,12 @@ def test_get_allele_detail_unknown_caid_is_404(client, setup_router_db): assert response.status_code == 404 -def test_get_allele_detail_is_public_no_privacy_gate( - client, session, data_provider, data_files, setup_router_db, anonymous_app_overrides +@pytest.mark.parametrize("identifier", [CDNA, "CA1", "PA1"]) +@pytest.mark.parametrize("anonymous", [True, False]) +def test_get_allele_detail_is_404_when_only_unreadable_score_sets_link_it( + client, session, data_provider, data_files, setup_router_db, anonymous_app_overrides, identifier, anonymous ): - """The decision: no privacy gate. An allele reachable only through a private score set is still - served to an anonymous caller — contrast ``GET /variants/{urn}``, which 404s the same variant.""" + """An allele exists only because of a private score set the caller may not read, so it is not found.""" experiment = create_experiment(client) score_set = create_seq_score_set_with_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" @@ -201,11 +207,71 @@ def test_get_allele_detail_is_public_no_privacy_gate( _seed(session, f"{score_set['urn']}#1") change_ownership(session, score_set["urn"], ScoreSetDbModel) - with DependencyOverrider(anonymous_app_overrides): - response = client.get(f"/api/v1/alleles/{CDNA}") + if anonymous: + with DependencyOverrider(anonymous_app_overrides): + response = client.get(f"/api/v1/alleles/{identifier}") + else: + response = client.get(f"/api/v1/alleles/{identifier}") + + assert response.status_code == 404 + + +def test_get_allele_detail_is_404_for_an_allele_no_score_set_links(client, session, setup_router_db): + session.add(Allele(vrs_digest=CDNA, level="cdna", hgvs_c="NM_000546.6:c.1216G>A", post_mapped=_post_mapped())) + session.commit() + + response = client.get(f"/api/v1/alleles/{CDNA}") + + assert response.status_code == 404 + + +def test_get_allele_detail_scopes_equivalence_class_to_readable_score_sets( + client, session, data_provider, data_files, setup_router_db +): + """A shared allele is served through the readable score set, but members co-linked only by an + unreadable score set's record are withheld.""" + experiment = create_experiment(client) + private_score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + _seed(session, f"{private_score_set['urn']}#1") + change_ownership(session, private_score_set["urn"], ScoreSetDbModel) + + readable_score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + variant = session.scalar(select(Variant).where(Variant.urn == f"{readable_score_set['urn']}#1")) + record = MappingRecord( + variant_id=variant.id, score_set_id=variant.score_set_id, assay_level="cdna", mapping_api_version="test.0.0" + ) + session.add(record) + session.commit() + for digest, is_authoritative in ((CDNA, True), (PROT, False)): + allele = session.scalar(select(Allele).where(Allele.vrs_digest == digest)) + session.add( + MappingRecordAllele( + mapping_record_id=record.id, + score_set_id=record.score_set_id, + allele_id=allele.id, + is_authoritative=is_authoritative, + ) + ) + session.commit() + + response = client.get(f"/api/v1/alleles/{CDNA}") assert response.status_code == 200 - assert response.json()["digest"] == CDNA + body = response.json() + assert set(body["alleles"]) == {CDNA, PROT} + assert set(body["annotations"]) == {CDNA, PROT} + # The c↔g pairing lives only on the unreadable record. + assert "projectionOf" not in body["alleles"][CDNA] + + # The genomic frame of CA1 is reachable only through the unreadable record, so a CAID fetch + # focuses the coding frame alone. + caid_response = client.get("/api/v1/alleles/CA1") + assert caid_response.status_code == 200 + assert set(caid_response.json()["alleles"]) == {CDNA, PROT} def test_get_allele_detail_echoes_as_of_header(client, session, data_provider, data_files, setup_router_db): From 6c1a0afdd72ef3ebfd4fba15a7f2b990627c7c0f Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 7 Oct 2026 15:36:07 -0700 Subject: [PATCH 221/252] fix(clingen-alleles): reach protein consequences only through readable score sets Measurements for a CAID found its protein consequence by joining across every score set's mapping records, so a private score set's reverse translation could pull in public measurements of that protein change. The CAID and its protein consequence now resolve only through score sets the caller may read. --- src/mavedb/lib/allele_measurements.py | 18 +++++++++++-- tests/lib/test_allele_measurements.py | 39 +++++++++++++++++++++++++-- 2 files changed, 53 insertions(+), 4 deletions(-) diff --git a/src/mavedb/lib/allele_measurements.py b/src/mavedb/lib/allele_measurements.py index ebe0f879d..10eb9b1e3 100644 --- a/src/mavedb/lib/allele_measurements.py +++ b/src/mavedb/lib/allele_measurements.py @@ -39,6 +39,7 @@ from sqlalchemy import and_, select from sqlalchemy.orm import Session, aliased, joinedload +from mavedb.lib.alleles import readable_score_set_ids_by_allele from mavedb.lib.logging.context import logging_context, save_to_logging_context from mavedb.lib.permissions import Action, has_permission from mavedb.lib.score_calibrations import calibration_preference_key, classification_evidence_strength @@ -179,11 +180,17 @@ def _ordering_key(measurement: AlleleMeasurement, published_date: Optional[date] ) -def _resolve_protein_apex(db: Session, anchor_ids: list[int], *, as_of: Optional[datetime]) -> _ProteinApex: +def _resolve_protein_apex( + db: Session, anchor_ids: list[int], *, as_of: Optional[datetime], score_set_ids: set[int] +) -> _ProteinApex: """The protein consequence(s) co-membered with the nt ``anchor_ids`` via a shared live record. The single site the apex is resolved, so its cardinality (see :class:`_ProteinApex`) is measured rather than being a silent side effect of the union. Reads ``clingen_allele_id`` to count PAIDs. + + Only records of ``score_set_ids`` are followed. A protein consequence that only an unreadable score + set's mapping produced would otherwise pull in public measurements of that protein change, disclosing + the private one. """ anchor_link = aliased(MappingRecordAllele) rows = db.execute( @@ -191,6 +198,7 @@ def _resolve_protein_apex(db: Session, anchor_ids: list[int], *, as_of: Optional .join(MappingRecordAllele, MappingRecordAllele.allele_id == Allele.id) .join(anchor_link, anchor_link.mapping_record_id == MappingRecordAllele.mapping_record_id) .where(anchor_link.allele_id.in_(anchor_ids)) + .where(anchor_link.score_set_id.in_(score_set_ids)) .where(anchor_link.live_at(as_of)) .where(MappingRecordAllele.live_at(as_of)) .where(Allele.level == SequenceLevel.protein.value) @@ -219,9 +227,15 @@ def get_allele_measurements( members. A PA query returns its ``direct`` protein measurements and their ``nucleotide_encoding`` encodings. """ anchor = db.execute(select(Allele.id, Allele.level).where(Allele.clingen_allele_id == clingen_allele_id)).all() + + # An allele only the caller's unreadable score sets link is treated as absent, as on GET /alleles. + readable_by_allele = readable_score_set_ids_by_allele(db, user_data, [row.id for row in anchor]) + anchor = [row for row in anchor if row.id in readable_by_allele] if not anchor: return [] + readable_score_set_ids: set[int] = set().union(*readable_by_allele.values()) + anchor_ids = [row.id for row in anchor] entry_is_protein = any(row.level == SequenceLevel.protein.value for row in anchor) @@ -230,7 +244,7 @@ def get_allele_measurements( # the calculation should be deterministic. apex: Optional[_ProteinApex] = None if not entry_is_protein: - apex = _resolve_protein_apex(db, anchor_ids, as_of=as_of) + apex = _resolve_protein_apex(db, anchor_ids, as_of=as_of, score_set_ids=readable_score_set_ids) anchor_ids += apex.allele_ids # Warn if multiple distinct protein apexes were resolved for this CA. diff --git a/tests/lib/test_allele_measurements.py b/tests/lib/test_allele_measurements.py index 64eeb9348..da5f675c7 100644 --- a/tests/lib/test_allele_measurements.py +++ b/tests/lib/test_allele_measurements.py @@ -262,7 +262,7 @@ def test_resolve_protein_apex_single(session, setup_lib_db_with_score_set): _link(session, r, nt, is_authoritative=True) _link(session, r, prot) - apex = _resolve_protein_apex(session, [nt.id], as_of=None) + apex = _resolve_protein_apex(session, [nt.id], as_of=None, score_set_ids={score_set.id}) assert apex.allele_ids == [prot.id] assert apex.caids == ["PA9"] assert apex.unregistered == 0 @@ -279,7 +279,7 @@ def test_resolve_protein_apex_unregistered(session, setup_lib_db_with_score_set) _link(session, r, nt, is_authoritative=True) _link(session, r, prot) - apex = _resolve_protein_apex(session, [nt.id], as_of=None) + apex = _resolve_protein_apex(session, [nt.id], as_of=None, score_set_ids={score_set.id}) assert apex.allele_ids == [prot.id] assert apex.caids == [] assert apex.unregistered == 1 @@ -396,6 +396,41 @@ def test_private_score_set_measurement_excluded(session, setup_lib_db_with_score assert {m.variant_urn for m in result} == {visible.urn} +@pytest.mark.integration +@pytest.mark.parametrize("ca_also_measured_publicly", [False, True]) +def test_protein_consequence_reached_only_through_unreadable_score_set_is_withheld( + session, setup_lib_db_with_score_set, ca_also_measured_publicly +): + """A protein consequence that only an unreadable score set's mapping links to the CA must not pull in + public measurements of that protein change: they would disclose the private record.""" + private = setup_lib_db_with_score_set # private, owned by TEST_USER + public = _second_score_set(session, private, "urn:mavedb:00000002-a-1", private=False) + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + prot = _allele(session, "prot-P", level="protein", clingen_allele_id="PA9") + + # The private score set's reverse translation is the only record linking CA123 to PA9. + private_measured = _variant(session, private, 1, data={"score_data": {"score": -2.0}}, hgvs_nt="c.1A>T") + private_record = _record(session, private_measured, assay_level="cdna") + _link(session, private_record, nt, is_authoritative=True) + _link(session, private_record, prot) + + # A public protein assay of PA9, with no reverse translation of its own. + public_protein = _variant(session, public, 1, data={"score_data": {"score": 1.0}}, hgvs_pro="p.Met1Leu") + _link(session, _record(session, public_protein, assay_level="protein"), prot, is_authoritative=True) + + expected: set[str] = set() + if ca_also_measured_publicly: + # CA123 is itself readable through the public set, but that record doesn't reach PA9. + public_nt = _variant(session, public, 2, data={"score_data": {"score": 0.5}}, hgvs_nt="c.1A>T") + _link(session, _record(session, public_nt, assay_level="cdna"), nt, is_authoritative=True) + expected = {public_nt.urn} + + with request_cycle_context({}): + result = get_allele_measurements(session, "CA123", user_data=None) + + assert {m.variant_urn for m in result} == expected + + @pytest.mark.integration def test_calibration_gate_withholds_classification(session, setup_lib_db_with_score_set): """Calibration READ gates only the inline classification: on a public score set with a private From b55ce20a3c781483cc646beb1eadaedf22a53a96 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 7 Oct 2026 15:36:08 -0700 Subject: [PATCH 222/252] fix(seqrepo): match sequence aliases exactly The shared SeqRepo holds target sequences of unpublished score sets, and lookups accepted digest prefixes and SQL LIKE wildcards, so the refget and seqrepo routes could enumerate them. A query containing "%" now matches nothing, and hex digests match only at full MD5 or TRUNC512 length. --- src/mavedb/lib/seqrepo.py | 40 ++++++++++++++++++++--------------- tests/helpers/constants.py | 2 +- tests/lib/test_seqrepo.py | 24 ++++++++++++++++++--- tests/routers/test_refget.py | 9 ++++++++ tests/routers/test_seqrepo.py | 9 ++++++++ 5 files changed, 63 insertions(+), 21 deletions(-) diff --git a/src/mavedb/lib/seqrepo.py b/src/mavedb/lib/seqrepo.py index 11d53e347..4330a9d85 100644 --- a/src/mavedb/lib/seqrepo.py +++ b/src/mavedb/lib/seqrepo.py @@ -11,7 +11,7 @@ # TODO (https://github.com/VariantEffect/mavedb-api/issues/354). We need pydantic upgraded to use this package. # from ga4gh.core.identifiers import is_ga4gh_identifier, CURIE_NAMESPACE as ga4gh_namespace -from typing import Generator, Optional, Union +from typing import Generator, Optional from biocommons.seqrepo import SeqRepo, __version__ as seqrepo_dep_version from bioutils.accessions import infer_namespaces @@ -41,13 +41,15 @@ def get_sequence_ids(sr: SeqRepo, query: str) -> list[str]: The query may be: * A fully-qualified sequence alias (e.g., VMC:0123 or refseq:NM_01234.5) - * A digest or digest prefix from VMC, TRUNC512, or MD5 + * A complete MD5 or TRUNC512 hex digest * A sequence accession (without namespace) - The first match will be returned. + The first match will be returned. Matching is exact: the shared SeqRepo holds the target sequences of + unpublished score sets, so neither digest prefixes nor SQL wildcards may be used to enumerate it. """ nsa_options = _generate_nsa_options(query) + aliases: list[dict] = [] for ns, a in nsa_options: aliases = list(sr.aliases.find_aliases(namespace=ns, alias=a)) if aliases: @@ -80,7 +82,7 @@ def resolve_refget(sr: SeqRepo, accession: str) -> str: return f"SQ.{seq_ids[0]}" -def _generate_nsa_options(query: str) -> Union[list[tuple[str, ...]], list[tuple[None, str]]]: +def _generate_nsa_options(query: str) -> list[tuple[Optional[str], str]]: """ >>> _generate_nsa_options("NM_000551.3") [('refseq', 'NM_000551.3')] @@ -94,20 +96,25 @@ def _generate_nsa_options(query: str) -> Union[list[tuple[str, ...]], list[tuple >> _generate_nsa_options("SQ.test") [('ga4gh', 'test')] - >>> _generate_nsa_options("01234abcde") - [('MD5', '01234abcde%'), ('VMC', 'GS_ASNKvN4=%')] + >>> _generate_nsa_options("0123456789abcdef0123456789abcdef") + [('MD5', '0123456789abcdef0123456789abcdef')] + + >>> _generate_nsa_options("NM_%") + [] """ + # SeqRepo switches to a LIKE comparison whenever an argument contains "%". + if "%" in query: + return [] if ":" in query: # interpret as fully-qualified identifier - nsa_options = [tuple(query.split(sep=":", maxsplit=1))] - return nsa_options + namespace, alias = query.split(sep=":", maxsplit=1) + return [(namespace, alias)] namespaces = infer_namespaces(query) if namespaces: - nsa_options = [(ns, query) for ns in namespaces] - return nsa_options + return [(ns, query) for ns in namespaces] # TODO (https://github.com/VariantEffect/mavedb-api/issues/354). We need pydantic upgraded to use this package. # if ga4gh, try ga4gh. GA4GH only accepts identifiers with a namespace prefix, @@ -116,13 +123,12 @@ def _generate_nsa_options(query: str) -> Union[list[tuple[str, ...]], list[tuple # nsa_options = [("ga4gh", query)] # return nsa_options - # if hex, try MD5 - if re.match(r"^(?:[0-9A-Fa-f]{8,})$", query): - nsa_options = [("MD5", query + "%")] - # TRUNC512 isn't in seqrepo; synthesize equivalent VMC - id_b64u = hex_to_base64url(query) - nsa_options += [("VMC", "GS_" + id_b64u + "%")] - return nsa_options + # A complete MD5 (32 hex characters) or TRUNC512 (48) digest. TRUNC512 isn't in SeqRepo; synthesize the + # equivalent VMC alias. + if re.fullmatch(r"[0-9A-Fa-f]{32}", query): + return [("MD5", query.lower())] + if re.fullmatch(r"[0-9A-Fa-f]{48}", query): + return [("VMC", "GS_" + hex_to_base64url(query))] return [(None, query)] diff --git a/tests/helpers/constants.py b/tests/helpers/constants.py index eb52f0bc2..f1d7b81ff 100644 --- a/tests/helpers/constants.py +++ b/tests/helpers/constants.py @@ -53,7 +53,7 @@ VALID_CLINGEN_CA_ID = "CA341478553" VALID_CLINGEN_LDH_ID = "2786738861" -VALID_MD5_DIGEST = "01234abcde%" +VALID_MD5_DIGEST = "0123456789abcdef0123456789abcdef" VALID_VMC_DIGEST = "GS_ASNKvN4=%" PRIVATE_CALIBRATION_OWNER_ID = 42 diff --git a/tests/lib/test_seqrepo.py b/tests/lib/test_seqrepo.py index 822d1a131..e37d1750e 100644 --- a/tests/lib/test_seqrepo.py +++ b/tests/lib/test_seqrepo.py @@ -7,7 +7,7 @@ from mavedb.lib.seqrepo import get_sequence_ids, _generate_nsa_options, seqrepo_versions, sequence_generator from mavedb.lib import seqrepo as seqrepo_lib -from tests.helpers.constants import TEST_SEQREPO_INITIAL_STATE +from tests.helpers.constants import TEST_SEQREPO_INITIAL_STATE, VALID_MD5_DIGEST @pytest.mark.parametrize("entry", TEST_SEQREPO_INITIAL_STATE) @@ -35,6 +35,18 @@ def test_get_sequence_ids_no_match(seqrepo): assert [] == result +@pytest.mark.parametrize("entry", TEST_SEQREPO_INITIAL_STATE) +def test_get_sequence_ids_rejects_wildcards(entry, seqrepo): + alias = list(entry.keys())[0] + namespace = entry[alias]["namespace"] + assert get_sequence_ids(seqrepo, f"{namespace}:{entry[alias]['alias'][:4]}%") == [] + assert get_sequence_ids(seqrepo, f"{namespace}:%") == [] + + +def test_get_sequence_ids_rejects_digest_prefixes(seqrepo): + assert get_sequence_ids(seqrepo, VALID_MD5_DIGEST[:8]) == [] + + @pytest.mark.parametrize( "query,expected", [ @@ -44,8 +56,14 @@ def test_get_sequence_ids_no_match(seqrepo): # Namespace inferred ("ENST00000530893.6", [("ensembl", "ENST00000530893.6")]), ("NM_000551.3", [("refseq", "NM_000551.3")]), - # Hex digest (MD5/VMC) - ("01234abcde", [("MD5", "01234abcde%"), ("VMC", "GS_ASNKvN4=%")]), + # Complete hex digests only: MD5 (32 characters) and TRUNC512, as VMC (48) + ("0123456789ABCDEF0123456789abcdef", [("MD5", "0123456789abcdef0123456789abcdef")]), + ("0123456789abcdef" * 3, [("VMC", "GS_ASNFZ4mrze8BI0VniavN7wEjRWeJq83v")]), + ("01234abcde", [(None, "01234abcde")]), + # SeqRepo treats "%" as a LIKE wildcard, so any query containing one matches nothing. + ("ga4gh:SQ.%", []), + ("NM_%", []), + ("%", []), # No match, fallback to (None, query) ("notfound", [(None, "notfound")]), ], diff --git a/tests/routers/test_refget.py b/tests/routers/test_refget.py index 6b656e5ed..9eb58f6f9 100644 --- a/tests/routers/test_refget.py +++ b/tests/routers/test_refget.py @@ -66,6 +66,15 @@ def test_get_metadata_not_found(client): assert "Sequence not found" in resp.text +# The shared SeqRepo holds the target sequences of unpublished score sets, so lookups must not match by +# wildcard or digest prefix. +@pytest.mark.parametrize("path", ["{alias}", "{alias}/metadata"]) +@pytest.mark.parametrize("alias", ["ga4gh:SQ.%25", "MD5:%25", "%25", "0123456789"]) +def test_get_sequence_does_not_match_wildcards_or_digest_prefixes(client, path, alias): + resp = client.get(f"/api/v1/refget/sequence/{path.format(alias=alias)}") + assert resp.status_code == 404 + + @pytest.mark.parametrize("entry", TEST_SEQREPO_INITIAL_STATE) def test_get_sequence_success(client, entry): alias = list(entry.keys())[0] diff --git a/tests/routers/test_seqrepo.py b/tests/routers/test_seqrepo.py index cfa8da063..c54bd1040 100644 --- a/tests/routers/test_seqrepo.py +++ b/tests/routers/test_seqrepo.py @@ -130,3 +130,12 @@ def test_get_versions(client): resp = client.get("/api/v1/seqrepo/version") assert resp.status_code == 200 assert isinstance(resp.json(), dict) + + +# The shared SeqRepo holds the target sequences of unpublished score sets, so lookups must not match by +# wildcard or digest prefix. +@pytest.mark.parametrize("path", ["sequence/{alias}", "metadata/{alias}"]) +@pytest.mark.parametrize("alias", ["ga4gh:SQ.%25", "MD5:%25", "%25", "0123456789"]) +def test_lookup_does_not_match_wildcards_or_digest_prefixes(client, path, alias): + resp = client.get(f"/api/v1/seqrepo/{path.format(alias=alias)}") + assert resp.status_code == 404 From 4e07de21379f757c5a3c359238543383265bcc90 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 7 Oct 2026 15:36:36 -0700 Subject: [PATCH 223/252] fix(permissions): withhold unreadable records from related-record URN lists Score set and experiment responses listed URNs straight from ORM relationships: private meta-analyses in metaAnalyzedByScoreSetUrns, private members and private badge collections in officialCollections, and private score sets in GET /experiments/. Each is now narrowed to what the caller may read, through _score_set_response, the experiment score-sets route and enrich_experiment_with_num_score_sets. The public dump narrows meta-analyses the same way, as a second line behind its loader's published and CC0 filter. --- src/mavedb/lib/collections.py | 45 +++++++++++++ src/mavedb/lib/experiments.py | 6 ++ src/mavedb/lib/score_sets.py | 15 ++++- src/mavedb/routers/experiments.py | 19 +++++- src/mavedb/routers/score_sets.py | 19 +++++- src/mavedb/scripts/export_public_data.py | 62 +++++++++-------- tests/routers/test_experiments.py | 22 ++++++ tests/routers/test_score_set.py | 85 ++++++++++++++++++++++++ tests/scripts/test_export_public_data.py | 24 +++++++ 9 files changed, 263 insertions(+), 34 deletions(-) create mode 100644 src/mavedb/lib/collections.py diff --git a/src/mavedb/lib/collections.py b/src/mavedb/lib/collections.py new file mode 100644 index 000000000..3e57bae28 --- /dev/null +++ b/src/mavedb/lib/collections.py @@ -0,0 +1,45 @@ +from typing import Iterable, Optional + +from mavedb.lib.permissions import Action, has_permission +from mavedb.lib.score_sets import readable_score_set_urns +from mavedb.lib.types.authentication import UserData +from mavedb.models.collection import Collection +from mavedb.view_models.collection import OfficialCollection + + +def readable_official_collections( + user_data: Optional[UserData], collections: Iterable[Collection] +) -> list[OfficialCollection]: + """Serialize official collections, carrying only what the caller may read. + + An official collection is a badge on every score set and experiment it holds, so it is serialized into + responses about those records. Collections the caller may not read are dropped, and each remaining + collection's member URNs are narrowed to the members the caller may read: a public score set can share + a badge collection with private ones. + + Narrows validated views rather than the ORM collections, whose member relationships would stage writes + if reassigned. + """ + narrowed = [] + for collection in collections: + if not has_permission(user_data, collection, Action.READ).permitted: + continue + + score_set_urns = readable_score_set_urns(user_data, collection.score_sets) + experiment_urns = { + experiment.urn + for experiment in collection.experiments + if has_permission(user_data, experiment, Action.READ).permitted + } + + view = OfficialCollection.model_validate(collection) + narrowed.append( + view.model_copy( + update={ + "score_set_urns": [urn for urn in view.score_set_urns if urn in score_set_urns], + "experiment_urns": [urn for urn in view.experiment_urns if urn in experiment_urns], + } + ) + ) + + return narrowed diff --git a/src/mavedb/lib/experiments.py b/src/mavedb/lib/experiments.py index a200e93bb..22b2c2d1c 100644 --- a/src/mavedb/lib/experiments.py +++ b/src/mavedb/lib/experiments.py @@ -5,6 +5,7 @@ from sqlalchemy.orm import Session from mavedb.lib.logging.context import logging_context, save_to_logging_context +from mavedb.lib.collections import readable_official_collections from mavedb.lib.permissions import Action from mavedb.lib.score_sets import find_superseded_score_set_tail from mavedb.lib.types.authentication import UserData @@ -137,6 +138,10 @@ def enrich_experiment_with_num_score_sets( """ Validate and update the number of score set in experiment. The superseded score set is excluded. Data structure: experiment{score_set_urns, num_score_sets} + + Also narrows the experiment's official collections to what the caller may read. Every route that returns + an ``Experiment`` view model must build it here; serializing the ORM experiment directly lists every + score set URN and every collection, private ones included. """ filter_superseded_score_set_tails = [ find_superseded_score_set_tail(score_set, Action.READ, user_data) for score_set in item_update.score_sets @@ -153,6 +158,7 @@ def enrich_experiment_with_num_score_sets( update={ "num_score_sets": len(filtered_score_set_urns), "score_set_urns": filtered_score_set_urns, + "official_collections": readable_official_collections(user_data, item_update.official_collections), } ) return updated_experiment diff --git a/src/mavedb/lib/score_sets.py b/src/mavedb/lib/score_sets.py index 6d599980f..64b6dd034 100644 --- a/src/mavedb/lib/score_sets.py +++ b/src/mavedb/lib/score_sets.py @@ -2,7 +2,7 @@ from collections import Counter, defaultdict from datetime import datetime from operator import attrgetter -from typing import TYPE_CHECKING, BinaryIO, Optional, Sequence +from typing import TYPE_CHECKING, BinaryIO, Iterable, Optional, Sequence import numpy as np import pandas as pd @@ -490,6 +490,19 @@ def find_meta_analyses_for_experiment_sets(db: Session, urns: list[str]) -> list ) +def readable_score_set_urns(user_data: Optional[UserData], score_sets: Iterable[ScoreSet]) -> set[str]: + """Return the URNs of ``score_sets`` the caller may read. + + For narrowing URN lists that view models build from ORM relationships with no knowledge of the caller, + such as ``meta_analyzed_by_score_set_urns`` or a collection's ``score_set_urns``. + """ + return { + score_set.urn + for score_set in score_sets + if score_set.urn is not None and has_permission(user_data, score_set, Action.READ).permitted + } + + def find_superseded_score_set_tail( score_set: ScoreSet, action: Optional["Action"] = None, user_data: Optional["UserData"] = None ) -> Optional[ScoreSet]: diff --git a/src/mavedb/routers/experiments.py b/src/mavedb/routers/experiments.py index 0d2e56c54..a58d127b9 100644 --- a/src/mavedb/routers/experiments.py +++ b/src/mavedb/routers/experiments.py @@ -13,6 +13,7 @@ from mavedb.lib.authorization import get_principal, require_current_user, require_current_user_with_email from mavedb.lib.contributors import find_or_create_contributor from mavedb.lib.exceptions import NonexistentOrcidUserError +from mavedb.lib.collections import readable_official_collections from mavedb.lib.experiments import enrich_experiment_with_num_score_sets from mavedb.lib.experiments import search_experiments as _search_experiments from mavedb.lib.identifiers import ( @@ -26,7 +27,7 @@ from mavedb.lib.permissions import Action, assert_permission, has_permission from mavedb.lib.permissions.principal import Principal from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer -from mavedb.lib.score_sets import find_superseded_score_set_tail +from mavedb.lib.score_sets import find_superseded_score_set_tail, readable_score_set_urns from mavedb.lib.types.authentication import UserData from mavedb.lib.validation.exceptions import ValidationError from mavedb.lib.validation.keywords import validate_keyword_list @@ -79,7 +80,7 @@ def list_experiments( editable: Optional[bool] = None, db: Session = Depends(deps.get_db), user_data: Optional[UserData] = Depends(get_current_user), -) -> list[Experiment]: +) -> list[experiment.Experiment]: """ List all experiments viewable by the current user. """ @@ -99,7 +100,11 @@ def list_experiments( ) items = query.order_by(Experiment.urn).all() - return [item for item in items if has_permission(user_data, item, Action.READ).permitted] + return [ + enrich_experiment_with_num_score_sets(item, user_data) + for item in items + if has_permission(user_data, item, Action.READ).permitted + ] @router.post( @@ -244,10 +249,18 @@ def get_experiment_score_sets( superseding_is_visible = fs.superseding_score_set is not None and ( has_permission(user_data, fs.superseding_score_set, Action.READ).permitted ) + readable_meta_analysis_urns = readable_score_set_urns( + user_data, + fs.meta_analyzed_by_score_sets, # type: ignore[attr-defined] # backref of meta_analyzes_score_sets + ) validated_item = score_set.ScoreSet.model_validate(fs) response_item = validated_item.copy( update={ "experiment": enriched_experiment, + "meta_analyzed_by_score_set_urns": [ + urn for urn in validated_item.meta_analyzed_by_score_set_urns if urn in readable_meta_analysis_urns + ], + "official_collections": readable_official_collections(user_data, fs.official_collections), "score_calibrations": [ calibration for calibration in (validated_item.score_calibrations or []) diff --git a/src/mavedb/routers/score_sets.py b/src/mavedb/routers/score_sets.py index 1f352232f..e39b0a756 100644 --- a/src/mavedb/routers/score_sets.py +++ b/src/mavedb/routers/score_sets.py @@ -66,12 +66,14 @@ from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer from mavedb.lib.score_calibrations import create_score_calibration from mavedb.lib.score_set_variants import get_lean_score_set_variants +from mavedb.lib.collections import readable_official_collections from mavedb.lib.score_sets import ( csv_data_to_df, fetch_score_set_search_filter_options, find_meta_analyses_for_experiment_sets, get_annotatable_variants, is_replaces_id_unique_violation, + readable_score_set_urns, refresh_variant_urns, ) from mavedb.lib.score_sets import ( @@ -595,13 +597,15 @@ def _score_set_response(item: ScoreSet, principal: Principal) -> score_set.Score """ Serialize a score set for a response, withholding the sub-resources this caller may not read. - Every route in this module that returns a ``ScoreSet`` view model builds it here. The two sub-resources - a score set carries have READ rules stricter than its own, and each was leaked from a different route - before this was centralized: + Every route in this module that returns a ``ScoreSet`` view model builds it here. The sub-resources a + score set carries have READ rules of their own, and each was leaked from a route before this was + centralized: - Calibrations. Publishing a score set does not publish its calibrations, and owning a score set does not entitle its owner to a community calibration someone else attached to it. - The superseding score set, which is usually still private while the score set it replaces is public. + - The score sets that meta-analyze this one. Anyone may start a meta-analysis of a published score set. + - Official collections, and the other members they list. The search routes are the deliberate exception: they answer with ``ShortScoreSet``, which carries neither sub-resource, so there is nothing for this function to narrow. Any route that widens its response model @@ -630,10 +634,19 @@ def _score_set_response(item: ScoreSet, principal: Principal) -> score_set.Score has_permission(principal.user_data, item.superseding_score_set, Action.READ).permitted ) + readable_meta_analysis_urns = readable_score_set_urns( + principal.user_data, + item.meta_analyzed_by_score_sets, # type: ignore[attr-defined] # backref of meta_analyzes_score_sets + ) + validated_item = score_set.ScoreSet.model_validate(item) return validated_item.model_copy( update={ "experiment": enrich_experiment_with_num_score_sets(item.experiment, principal.user_data), + "meta_analyzed_by_score_set_urns": [ + urn for urn in validated_item.meta_analyzed_by_score_set_urns if urn in readable_meta_analysis_urns + ], + "official_collections": readable_official_collections(principal.user_data, item.official_collections), "score_calibrations": [ calibration for calibration in (validated_item.score_calibrations or []) diff --git a/src/mavedb/scripts/export_public_data.py b/src/mavedb/scripts/export_public_data.py index fd9195b89..e19a6fd68 100644 --- a/src/mavedb/scripts/export_public_data.py +++ b/src/mavedb/scripts/export_public_data.py @@ -41,10 +41,9 @@ available_score_set_csv_namespaces, get_score_set_variants_as_csv, ) -from mavedb.lib.permissions import Action, has_permission from mavedb.lib.permissions.principal import Principal from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer -from mavedb.lib.score_sets import get_annotatable_variants +from mavedb.lib.score_sets import get_annotatable_variants, readable_score_set_urns from mavedb.models.experiment import Experiment from mavedb.models.experiment_set import ExperimentSet from mavedb.models.license import License @@ -291,27 +290,28 @@ def score_set_artifacts(db: Session, score_set: ScoreSet, principal: Principal) def public_experiment_set( experiment_set_view: ExperimentSetPublicDump, visible_calibration_ids: set[int], - readable_superseding_urns: set[str], + readable_related_urns: set[str], ) -> Optional[ExperimentSetPublicDump]: """Narrow a validated experiment set to what belongs in the public dump. Drops calibrations the caller may not READ, blanks a superseding score set the caller may not READ, - drops experiments left with no score sets, and returns None if no experiments remain. The score sets - themselves need no filter — the loading query already restricts them to published, CC0-licensed rows. + drops meta-analyses the caller may not READ from `meta_analyzed_by_score_set_urns`, drops experiments + left with no score sets, and returns None if no experiments remain. The score sets themselves need no + filter — the loading query already restricts them to published, CC0-licensed rows. Blanking supersession matters because a published score set is often superseded by a *private* in-progress replacement; without it, `main.json` would name an unreleased score set's URN and title in an archive published to Zenodo, which can't be recalled. - `readable_superseding_urns` gates on READ, the rule `_score_set_response` applies. It is - defense-in-depth rather than the live gate, and it can only subtract: `published_experiment_sets` - loads members through a published+CC0 filter that propagates to `superseding_score_set`, so a - private replacement is already `None` before this runs — and so is a *published* successor whose - license keeps it out of the archive. That second case is unresolved: the consumer reads a superseded - score set as current. Naming it would need the successor fetched by a separate unfiltered query and - injected into the view, not merely left un-blanked; whether it should be named is an open policy - question. Both behaviors are pinned by - `test_supersession_is_named_only_when_the_archive_carries_the_successor`. + `readable_related_urns` holds the superseding and meta-analysis score sets the caller may READ, the + rule `_score_set_response` applies. It is defense-in-depth rather than the live gate, and it can only + subtract: `published_experiment_sets` loads members through a published+CC0 filter that propagates to + `superseding_score_set` and `meta_analyzed_by_score_sets`, so a private replacement or meta-analysis is + already gone before this runs — and so is a *published* successor whose license keeps it out of the + archive. That second case is unresolved: the consumer reads a superseded score set as current. Naming + it would need the successor fetched by a separate unfiltered query and injected into the view, not + merely left un-blanked; whether it should be named is an open policy question. Both behaviors are + pinned by `test_supersession_is_named_only_when_the_archive_carries_the_successor`. Narrows the validated view rather than the ORM graph because `ExperimentSet.experiments` and `ScoreSet.score_calibrations` cascade-delete orphans, and this script can flush (`--commit`). @@ -332,9 +332,12 @@ def public_experiment_set( "superseding_score_set": ( score_set_view.superseding_score_set if score_set_view.superseding_score_set is not None - and score_set_view.superseding_score_set.urn in readable_superseding_urns + and score_set_view.superseding_score_set.urn in readable_related_urns else None ), + "meta_analyzed_by_score_set_urns": [ + urn for urn in score_set_view.meta_analyzed_by_score_set_urns if urn in readable_related_urns + ], } ) for score_set_view in experiment_view.score_sets @@ -404,23 +407,28 @@ def public_dump_metadata(db: Session, principal: Principal) -> tuple[dict, list[ # Issue: https://github.com/VariantEffect/mavedb-api/issues/192 # See, for instance, https://stackoverflow.com/questions/12670395/json-encoding-very-long-iterators. - # Asked of the ORM graph, not the validated views: a successor is typically outside the loading - # query's published+CC0 filter, so checking the view would miss it. READ is the gate, but this set - # only ever subtracts — the loader nulls a non-CC0 successor on the view first, so being readable - # here is not enough to get one named. See `public_experiment_set`. - readable_superseding_urns: set[str] = { - str(score_set_orm.superseding_score_set.urn) - for score_set_orm in flatmap(lambda es: flatmap(lambda e: e.score_sets, es.experiments), experiment_sets) - if score_set_orm.superseding_score_set is not None - and score_set_orm.superseding_score_set.urn is not None - and has_permission(principal.user_data, score_set_orm.superseding_score_set, Action.READ).permitted - } + # Asked of the ORM graph, not the validated views: a successor or meta-analysis is typically outside the + # loading query's published+CC0 filter, so checking the view would miss it. READ is the gate, but for + # successors this set only ever subtracts — the loader nulls a non-CC0 successor on the view first, so + # being readable here is not enough to get one named. See `public_experiment_set`. + loaded_score_sets = list(flatmap(lambda es: flatmap(lambda e: e.score_sets, es.experiments), experiment_sets)) + readable_related_urns = readable_score_set_urns( + principal.user_data, + [ + related + for score_set_orm in loaded_score_sets + for related in [ + *([score_set_orm.superseding_score_set] if score_set_orm.superseding_score_set is not None else []), + *score_set_orm.meta_analyzed_by_score_sets, # type: ignore[attr-defined] # backref + ] + ], + ) experiment_set_views = [ narrowed for narrowed in ( public_experiment_set( - ExperimentSetPublicDump.model_validate(es), visible_calibration_ids, readable_superseding_urns + ExperimentSetPublicDump.model_validate(es), visible_calibration_ids, readable_related_urns ) for es in experiment_sets ) diff --git a/tests/routers/test_experiments.py b/tests/routers/test_experiments.py index e4f56785e..71c6729bd 100644 --- a/tests/routers/test_experiments.py +++ b/tests/routers/test_experiments.py @@ -2292,3 +2292,25 @@ def test_cannot_add_experiment_to_others_public_experiment_set( assert response.status_code == 403 response_data = response.json() assert f"insufficient permissions on experiment set with URN '{experiment_set_urn}'" in response_data["detail"] + + +def test_experiment_list_omits_unreadable_score_sets( + session, data_provider, client, setup_router_db, data_files, anonymous_app_overrides +): + """A published experiment often holds a private score set, such as a new submission or an unpublished + successor; listing experiments must not name it to callers who can't read it.""" + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published = publish_score_set(client, score_set["urn"]) + private_score_set = create_seq_score_set(client, published["experiment"]["urn"]) + + owner_list = client.get("/api/v1/experiments/").json() + assert [sorted(e["scoreSetUrns"]) for e in owner_list] == [sorted([published["urn"], private_score_set["urn"]])] + + with DependencyOverrider(anonymous_app_overrides): + anonymous_list = client.get("/api/v1/experiments/").json() + + assert [e["scoreSetUrns"] for e in anonymous_list] == [[published["urn"]]] + assert [e["numScoreSets"] for e in anonymous_list] == [1] diff --git a/tests/routers/test_score_set.py b/tests/routers/test_score_set.py index 137ca757c..e33c49e83 100644 --- a/tests/routers/test_score_set.py +++ b/tests/routers/test_score_set.py @@ -26,6 +26,7 @@ from mavedb.lib.validation.urn_re import MAVEDB_EXPERIMENT_URN_RE, MAVEDB_SCORE_SET_URN_RE, MAVEDB_TMP_URN_RE from mavedb.models.annotation_event import AnnotationEvent from mavedb.models.clinvar_allele_link import ClinvarAlleleLink +from mavedb.models.collection import Collection as CollectionDbModel from mavedb.models.enums.annotation_type import AnnotationType from mavedb.models.enums.disposition import Disposition from mavedb.models.enums.processing_state import ProcessingState @@ -103,6 +104,7 @@ seed_csv_substrate, shift_hgvs_position, ) +from tests.helpers.util.collection import create_collection from tests.helpers.util.user import change_ownership from tests.helpers.util.variant import ( mock_worker_variant_insertion, @@ -6278,3 +6280,86 @@ def test_publish_returns_the_owners_own_private_calibration( published = publish_score_set(client, score_set["urn"]) assert [c["urn"] for c in (published.get("scoreCalibrations") or [])] == [calibration["urn"]] + + +def test_meta_analyses_are_listed_only_to_callers_who_may_read_them( + session, data_provider, client, setup_router_db, data_files, anonymous_app_overrides, extra_user_app_overrides +): + """Anyone may start a meta-analysis of a published score set, so the score set must not name a private + one to callers who can't read it.""" + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published = publish_score_set(client, score_set["urn"]) + + with DependencyOverrider(extra_user_app_overrides): + meta_analysis = create_seq_score_set( + client, None, update={"title": "Private meta-analysis", "metaAnalyzesScoreSetUrns": [published["urn"]]} + ) + assert client.get(f"/api/v1/score-sets/{published['urn']}").json()["metaAnalyzedByScoreSetUrns"] == [ + meta_analysis["urn"] + ] + + assert client.get(f"/api/v1/score-sets/{published['urn']}").json()["metaAnalyzedByScoreSetUrns"] == [] + with DependencyOverrider(anonymous_app_overrides): + assert client.get(f"/api/v1/score-sets/{published['urn']}").json()["metaAnalyzedByScoreSetUrns"] == [] + experiment_score_sets = client.get(f"/api/v1/experiments/{published['experiment']['urn']}/score-sets").json() + assert [ss["metaAnalyzedByScoreSetUrns"] for ss in experiment_score_sets] == [[]] + + +def test_official_collections_list_only_readable_collections_and_members( + session, data_provider, client, setup_router_db, data_files, anonymous_app_overrides +): + """An official collection badges every member, so a public record's response must not reveal the private + members it shares a badge with, or a private badge collection at all.""" + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published = publish_score_set(client, score_set["urn"]) + published_experiment_urn = published["experiment"]["urn"] + private_score_set = create_seq_score_set(client, published_experiment_urn) + private_experiment = create_experiment(client) + + public_badge = create_collection(client) + private_badge = create_collection(client) + for collection, private in ((public_badge, False), (private_badge, True)): + for member in (published, private_score_set): + response = client.post( + f"/api/v1/collections/{collection['urn']}/score-sets", json={"score_set_urn": member["urn"]} + ) + assert response.status_code == 200, response.text + for experiment_urn in (published_experiment_urn, private_experiment["urn"]): + response = client.post( + f"/api/v1/collections/{collection['urn']}/experiments", json={"experiment_urn": experiment_urn} + ) + assert response.status_code == 200, response.text + session.query(CollectionDbModel).filter(CollectionDbModel.urn == collection["urn"]).update( + {"badge_name": "Badge", "private": private} + ) + session.commit() + + def members(collections): + return {c["urn"]: (sorted(c["scoreSetUrns"]), sorted(c["experimentUrns"])) for c in collections} + + everything = ( + sorted([published["urn"], private_score_set["urn"]]), + sorted([published_experiment_urn, private_experiment["urn"]]), + ) + owner_view = client.get(f"/api/v1/score-sets/{published['urn']}").json() + assert members(owner_view["officialCollections"]) == { + public_badge["urn"]: everything, + private_badge["urn"]: everything, + } + + with DependencyOverrider(anonymous_app_overrides): + anonymous_view = client.get(f"/api/v1/score-sets/{published['urn']}").json() + experiment_view = client.get(f"/api/v1/experiments/{published_experiment_urn}").json() + experiment_list = client.get("/api/v1/experiments/").json() + + expected = {public_badge["urn"]: ([published["urn"]], [published_experiment_urn])} + assert members(anonymous_view["officialCollections"]) == expected + assert members(anonymous_view["experiment"]["officialCollections"]) == expected + assert members(experiment_view["officialCollections"]) == expected + assert [members(e["officialCollections"]) for e in experiment_list] == [expected] diff --git a/tests/scripts/test_export_public_data.py b/tests/scripts/test_export_public_data.py index 3af75f583..8d651d7b1 100644 --- a/tests/scripts/test_export_public_data.py +++ b/tests/scripts/test_export_public_data.py @@ -970,6 +970,30 @@ def test_supersession_is_named_only_when_the_archive_carries_the_successor( assert _metadata_supersessions(metadata)[urn] == (successor_urn if expect_named else None) + @pytest.mark.parametrize("meta_analysis_published", [False, True]) + def test_names_only_readable_meta_analyses( + self, session, make_dump_score_set, anonymous_principal, meta_analysis_published + ): + """Anyone may start a meta-analysis of a published score set, so a private one must never be named + in an archive that can't be recalled. The loader's published+CC0 filter withholds it, and + `public_experiment_set` narrows by READ as a second line.""" + analyzed = make_dump_score_set() + meta_analysis = make_dump_score_set(published=meta_analysis_published) + meta_analysis.meta_analyzes_score_sets = [analyzed] + session.commit() + urn, meta_analysis_urn = analyzed.urn, meta_analysis.urn + session.expunge_all() + + metadata, _ = public_dump_metadata(session, anonymous_principal) + + meta_analyzed_by = { + score_set.urn: score_set.meta_analyzed_by_score_set_urns + for experiment_set in metadata["experimentSets"] + for experiment in experiment_set.experiments + for score_set in experiment.score_sets + } + assert meta_analyzed_by[urn] == ([meta_analysis_urn] if meta_analysis_published else []) + def test_reports_the_urns_whose_artifacts_the_archive_carries( self, session, make_dump_score_set, anonymous_principal ): From a6e08fba6c5c5a42279f58befd48ec9751402b0d Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 7 Oct 2026 15:36:47 -0700 Subject: [PATCH 224/252] fix(score-sets): check UPDATE before validating a score set update PATCH /score-sets-with-variants validated the merged update before checking permission, and its validation errors echo the stored record, so any signed-in caller could confirm a private score set exists and read back fields of it. Permission is now asserted first. --- src/mavedb/routers/score_sets.py | 25 ++++++++++++++----------- tests/routers/test_score_set.py | 12 ++++++++++++ 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/src/mavedb/routers/score_sets.py b/src/mavedb/routers/score_sets.py index e39b0a756..d048b085c 100644 --- a/src/mavedb/routers/score_sets.py +++ b/src/mavedb/routers/score_sets.py @@ -2552,20 +2552,23 @@ async def update_score_set_with_variants( # get existing item from db existing_item = db.query(ScoreSet).filter(ScoreSet.urn == urn).one_or_none() - # merge existing item data with item_update data to validate against ScoreSetUpdate - - if existing_item: - existing_item_data = score_set.ScoreSet.model_validate(existing_item).model_dump() - updated_data = {**existing_item_data, **item_update_partial.model_dump(exclude_unset=True)} - try: - score_set.ScoreSetUpdate.model_validate(updated_data) - except ValidationError as e: - # format as fastapi request validation error - raise RequestValidationError(errors=e.errors()) - else: + if not existing_item: logger.info(msg="Failed to update score set; The requested score set does not exist.", extra=logging_context()) raise HTTPException(status_code=404, detail=f"score set with URN '{urn}' not found") + # Before validation, whose errors echo the existing record's values: a caller who may not update this + # score set must not learn anything about it, including that it exists. + assert_permission(user_data, existing_item, Action.UPDATE) + + # merge existing item data with item_update data to validate against ScoreSetUpdate + existing_item_data = score_set.ScoreSet.model_validate(existing_item).model_dump() + updated_data = {**existing_item_data, **item_update_partial.model_dump(exclude_unset=True)} + try: + score_set.ScoreSetUpdate.model_validate(updated_data) + except ValidationError as e: + # format as fastapi request validation error + raise RequestValidationError(errors=e.errors()) + existing_score_columns_metadata = (existing_item.dataset_columns or {}).get("score_columns_metadata", {}) existing_count_columns_metadata = (existing_item.dataset_columns or {}).get("count_columns_metadata", {}) diff --git a/tests/routers/test_score_set.py b/tests/routers/test_score_set.py index e33c49e83..03cfe8e88 100644 --- a/tests/routers/test_score_set.py +++ b/tests/routers/test_score_set.py @@ -6282,6 +6282,18 @@ def test_publish_returns_the_owners_own_private_calibration( assert [c["urn"] for c in (published.get("scoreCalibrations") or [])] == [calibration["urn"]] +def test_patch_with_invalid_data_on_unreadable_score_set_is_404(session, client, setup_router_db): + """Validation errors echo the stored record, so a caller who may not update the score set must be + refused before validation: a 422 here would confirm that a private score set exists.""" + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + change_ownership(session, score_set["urn"], ScoreSetDbModel) + + response = client.patch(f"/api/v1/score-sets-with-variants/{score_set['urn']}", data={"title": json.dumps(None)}) + + assert response.status_code == 404 + + def test_meta_analyses_are_listed_only_to_callers_who_may_read_them( session, data_provider, client, setup_router_db, data_files, anonymous_app_overrides, extra_user_app_overrides ): From cbbff52f43400df4e592e09d09fb287f563c8520 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 7 Oct 2026 15:36:47 -0700 Subject: [PATCH 225/252] fix(collections): report unreadable records as missing when adding members Adding a score set or experiment to a collection never checked READ on it, so membership could probe for private records and attach other users' private ones. Create, the add routes, and PATCH now treat an unreadable record as not found. PATCH checks only new members, so members the caller can't see survive a reorder. --- src/mavedb/routers/collections.py | 23 +++++++++++++++++++++-- tests/routers/test_collections.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/src/mavedb/routers/collections.py b/src/mavedb/routers/collections.py index 8475322d7..051d5bb3a 100644 --- a/src/mavedb/routers/collections.py +++ b/src/mavedb/routers/collections.py @@ -241,11 +241,16 @@ async def create_collection( score_set_associations = [] for position, score_set_urn in enumerate(item_create.score_set_urns or []): score_set = db.scalars(select(ScoreSet).where(ScoreSet.urn == score_set_urn)).one() + # A record the caller may not read is reported as missing, so membership can't probe for it. + if not has_permission(user_data, score_set, Action.READ).permitted: + raise NoResultFound() score_set_associations.append(CollectionScoreSetAssociation(score_set=score_set, position=position)) experiment_associations = [] for position, experiment_urn in enumerate(item_create.experiment_urns or []): experiment = db.scalars(select(Experiment).where(Experiment.urn == experiment_urn)).one() + if not has_permission(user_data, experiment, Action.READ).permitted: + raise NoResultFound() experiment_associations.append(CollectionExperimentAssociation(experiment=experiment, position=position)) except NoResultFound as e: @@ -345,6 +350,13 @@ async def update_collection( # Build new ordered associations for position, score_set_urn in enumerate(item_update.score_set_urns): score_set = db.scalars(select(ScoreSet).where(ScoreSet.urn == score_set_urn)).one() + # Only new members are checked, so existing members the caller can't see survive a reorder. + # A new member the caller may not read is reported as missing. + if ( + score_set_urn not in current_urns + and not has_permission(user_data, score_set, Action.READ).permitted + ): + raise NoResultFound() item.score_set_associations.append( CollectionScoreSetAssociation(score_set=score_set, position=position) ) @@ -372,6 +384,11 @@ async def update_collection( # Build new ordered associations for position, experiment_urn in enumerate(item_update.experiment_urns): experiment = db.scalars(select(Experiment).where(Experiment.urn == experiment_urn)).one() + if ( + experiment_urn not in current_urns + and not has_permission(user_data, experiment, Action.READ).permitted + ): + raise NoResultFound() item.experiment_associations.append( CollectionExperimentAssociation(experiment=experiment, position=position) ) @@ -450,7 +467,8 @@ async def add_score_set_to_collection( raise HTTPException(status_code=404, detail=f"collection with URN '{collection_urn}' not found") score_set = db.execute(select(ScoreSet).where(ScoreSet.urn == body.score_set_urn)).scalars().one_or_none() - if not score_set: + # A score set the caller may not read is reported as missing, so this route can't probe for one. + if not score_set or not has_permission(user_data, score_set, Action.READ).permitted: logger.info( msg="Failed to add score set to collection; The requested score set does not exist.", extra=logging_context(), @@ -608,7 +626,8 @@ async def add_experiment_to_collection( raise HTTPException(status_code=404, detail=f"collection with URN '{collection_urn}' not found") experiment = db.execute(select(Experiment).where(Experiment.urn == body.experiment_urn)).scalars().one_or_none() - if not experiment: + # An experiment the caller may not read is reported as missing, so this route can't probe for one. + if not experiment or not has_permission(user_data, experiment, Action.READ).permitted: logger.info( msg="Failed to add experiment to collection; The requested experiment does not exist.", extra=logging_context(), diff --git a/tests/routers/test_collections.py b/tests/routers/test_collections.py index b3df63538..899a30fba 100644 --- a/tests/routers/test_collections.py +++ b/tests/routers/test_collections.py @@ -1116,3 +1116,33 @@ def test_only_users_who_can_add_users_see_the_full_roster( assert orcids["admins"] == [TEST_USER["username"]] assert orcids["editors"] == [] assert orcids["viewers"] == [] + + +@pytest.mark.parametrize("route", ["add", "create", "update"]) +def test_cannot_add_unreadable_score_set_to_own_collection( + session, client, setup_router_db, extra_user_app_overrides, route +): + """Adding another user's private score set to your own collection reports it as missing, so membership + can neither probe for private records nor attach them.""" + experiment = create_experiment(client) + private_score_set = create_seq_score_set(client, experiment["urn"]) + + with DependencyOverrider(extra_user_app_overrides): + if route == "create": + response = client.post( + "/api/v1/collections/", json={**TEST_COLLECTION, "score_set_urns": [private_score_set["urn"]]} + ) + else: + collection = create_collection(client) + if route == "add": + response = client.post( + f"/api/v1/collections/{collection['urn']}/score-sets", + json={"score_set_urn": private_score_set["urn"]}, + ) + else: + response = client.patch( + f"/api/v1/collections/{collection['urn']}", json={"score_set_urns": [private_score_set["urn"]]} + ) + + assert response.status_code == 404 + assert session.query(CollectionScoreSetAssociation).count() == 0 From 45ad66a6df937fe21330743f86c7e45f89ac60de Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 7 Oct 2026 15:36:48 -0700 Subject: [PATCH 226/252] fix(permissions): report unreadable resources as missing in permission checks GET /permissions/user-is-permitted answered 404 for a missing resource but false for one the caller may not read, confirming that it exists. It now answers 404 for both. The UI already treats an error as false. --- src/mavedb/routers/permissions.py | 12 ++++---- tests/routers/test_permissions.py | 48 +++++++++++++++---------------- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/src/mavedb/routers/permissions.py b/src/mavedb/routers/permissions.py index 39833ec77..cd6349f66 100644 --- a/src/mavedb/routers/permissions.py +++ b/src/mavedb/routers/permissions.py @@ -76,9 +76,9 @@ async def check_permission( elif model_name == ModelName.score_calibration: item = db.query(ScoreCalibration).filter(ScoreCalibration.urn == urn).one_or_none() - if item: - permission = has_permission(user_data, item, action).permitted - return permission - else: - logger.debug(msg="The requested resources does not exist.", extra=logging_context()) - raise HTTPException(status_code=404, detail=f"{model_name.value} with URN '{urn}' not found") + # A resource the caller may not read is reported as missing, so this route can't confirm it exists. + if item and has_permission(user_data, item, Action.READ).permitted: + return has_permission(user_data, item, action).permitted + + logger.debug(msg="The requested resource does not exist or is not readable.", extra=logging_context()) + raise HTTPException(status_code=404, detail=f"{model_name.value} with URN '{urn}' not found") diff --git a/tests/routers/test_permissions.py b/tests/routers/test_permissions.py index b60a924e8..27a07b9e3 100644 --- a/tests/routers/test_permissions.py +++ b/tests/routers/test_permissions.py @@ -58,7 +58,7 @@ def test_contributor_gets_true_permission_from_others_experiment_set_add_experim assert response.json() -def test_get_false_permission_from_others_experiment_set_add_experiment_check(session, client, setup_router_db): +def test_get_not_found_from_others_experiment_set_add_experiment_check(session, client, setup_router_db): experiment = create_experiment(client) change_ownership(session, experiment["urn"], ExperimentDbModel) change_ownership(session, experiment["experimentSetUrn"], ExperimentSetDbModel) @@ -67,8 +67,8 @@ def test_get_false_permission_from_others_experiment_set_add_experiment_check(se f"/api/v1/permissions/user-is-permitted/experiment-set/{experiment['experimentSetUrn']}/add_experiment" ) - assert response.status_code == 200 - assert not response.json() + # Unreadable resources are reported as missing, so the check can't confirm they exist. + assert response.status_code == 404 def test_cannot_get_permission_with_wrong_action_in_experiment_set(client, setup_router_db): @@ -167,14 +167,14 @@ def test_contributor_gets_true_permission_from_others_private_experiment_add_sco assert response.json() -def test_get_false_permission_from_others_private_experiment_add_score_set_check(session, client, setup_router_db): +def test_get_not_found_from_others_private_experiment_add_score_set_check(session, client, setup_router_db): experiment = create_experiment(client) change_ownership(session, experiment["urn"], ExperimentDbModel) response = client.get(f"/api/v1/permissions/user-is-permitted/experiment/{experiment['urn']}/add_score_set") - assert response.status_code == 200 - assert not response.json() + # Unreadable resources are reported as missing, so the check can't confirm they exist. + assert response.status_code == 404 def test_get_true_permission_from_others_public_experiment_add_score_set_check( @@ -198,24 +198,24 @@ def test_get_true_permission_from_others_public_experiment_add_score_set_check( assert response.json() -def test_get_false_permission_from_others_experiment_update_check(session, client, setup_router_db): +def test_get_not_found_from_others_experiment_update_check(session, client, setup_router_db): experiment = create_experiment(client) change_ownership(session, experiment["urn"], ExperimentDbModel) response = client.get(f"/api/v1/permissions/user-is-permitted/experiment/{experiment['urn']}/update") - assert response.status_code == 200 - assert not response.json() + # Unreadable resources are reported as missing, so the check can't confirm they exist. + assert response.status_code == 404 -def test_get_false_permission_from_other_users_experiment_delete_check(session, client, setup_router_db): +def test_get_not_found_from_other_users_experiment_delete_check(session, client, setup_router_db): experiment = create_experiment(client) change_ownership(session, experiment["urn"], ExperimentDbModel) response = client.get(f"/api/v1/permissions/user-is-permitted/experiment/{experiment['urn']}/delete") - assert response.status_code == 200 - assert not response.json() + # Unreadable resources are reported as missing, so the check can't confirm they exist. + assert response.status_code == 404 def test_cannot_get_permission_with_wrong_action_in_experiment(client, setup_router_db): @@ -318,37 +318,37 @@ def test_contributor_gets_false_permission_from_others_score_set_publish_check(s assert not response.json() -def test_get_false_permission_from_others_score_set_delete_check(session, client, setup_router_db): +def test_get_not_found_from_others_score_set_delete_check(session, client, setup_router_db): experiment = create_experiment(client) score_set = create_seq_score_set(client, experiment["urn"]) change_ownership(session, score_set["urn"], ScoreSetDbModel) response = client.get(f"/api/v1/permissions/user-is-permitted/score-set/{score_set['urn']}/delete") - assert response.status_code == 200 - assert not response.json() + # Unreadable resources are reported as missing, so the check can't confirm they exist. + assert response.status_code == 404 -def test_get_false_permission_from_others_score_set_update_check(session, client, setup_router_db): +def test_get_not_found_from_others_score_set_update_check(session, client, setup_router_db): experiment = create_experiment(client) score_set = create_seq_score_set(client, experiment["urn"]) change_ownership(session, score_set["urn"], ScoreSetDbModel) response = client.get(f"/api/v1/permissions/user-is-permitted/score-set/{score_set['urn']}/update") - assert response.status_code == 200 - assert not response.json() + # Unreadable resources are reported as missing, so the check can't confirm they exist. + assert response.status_code == 404 -def test_get_false_permission_from_others_score_set_publish_check(session, client, setup_router_db): +def test_get_not_found_from_others_score_set_publish_check(session, client, setup_router_db): experiment = create_experiment(client) score_set = create_seq_score_set(client, experiment["urn"]) change_ownership(session, score_set["urn"], ScoreSetDbModel) response = client.get(f"/api/v1/permissions/user-is-permitted/score-set/{score_set['urn']}/publish") - assert response.status_code == 200 - assert not response.json() + # Unreadable resources are reported as missing, so the check can't confirm they exist. + assert response.status_code == 404 def test_cannot_get_permission_with_wrong_action_in_score_set(client, setup_router_db): @@ -464,7 +464,7 @@ def test_get_true_permission_as_score_set_owner_on_others_investigator_provided_ assert response.json() -def test_get_false_permission_as_score_set_owner_on_others_community_score_calibration_update_check( +def test_get_not_found_as_score_set_owner_on_others_community_score_calibration_update_check( session, client, setup_router_db, admin_app_overrides ): experiment = create_experiment(client) @@ -477,8 +477,8 @@ def test_get_false_permission_as_score_set_owner_on_others_community_score_calib response = client.get(f"/api/v1/permissions/user-is-permitted/score-calibration/{score_calibration['urn']}/update") - assert response.status_code == 200 - assert not response.json() + # Unreadable resources are reported as missing, so the check can't confirm they exist. + assert response.status_code == 404 def test_get_false_permission_from_others_score_calibration_delete_check(session, client, setup_router_db): From 322f52c9148dbe5a81190cb69ccf89447d216fd9 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Thu, 8 Oct 2026 09:07:44 -0700 Subject: [PATCH 227/252] fix(measurements): only fall back to an RUO call when no clinical calibration is readable A non-RUO calibration that leaves a variant unclassified is itself the answer for that score set, so the research-use-only classification no longer fills the gap. The fallback now applies only when the score set has no non-RUO calibration the caller can read. --- src/mavedb/lib/allele_measurements.py | 23 ++++++++-- src/mavedb/view_models/allele_measurement.py | 4 +- tests/lib/test_allele_measurements.py | 47 +++++++++++++++++++- 3 files changed, 67 insertions(+), 7 deletions(-) diff --git a/src/mavedb/lib/allele_measurements.py b/src/mavedb/lib/allele_measurements.py index 10eb9b1e3..8acd45f5c 100644 --- a/src/mavedb/lib/allele_measurements.py +++ b/src/mavedb/lib/allele_measurements.py @@ -77,7 +77,8 @@ class AlleleMeasurement: ``assay_level`` is the sequence level at which this measurement was assayed. ``preferred_classification`` is the readable classification the UI will default to; it never comes from a research-use-only calibration, so ranking and any consumer treating it as clinical evidence can rely on that. ``research_use_only_classification`` - is a display-only fallback: the RUO pick, set only when ``preferred_classification`` is ``None``. + is a display-only fallback: the RUO pick, set only when the score set has no readable non-RUO calibration at + all. A non-RUO calibration that leaves this variant unclassified is a call in itself and is not overridden. """ variant_urn: str @@ -152,9 +153,21 @@ def preference(pair: tuple[ScoreCalibrationFunctionalClassification, ScoreCalibr return min(candidates, key=preference)[0] +def _has_readable_clinical_calibration(score_set: ScoreSet, *, user_data: Optional[UserData]) -> bool: + """Whether the score set has any non-RUO calibration the caller can read, classifying this variant or not.""" + return any( + not calibration.research_use_only and has_permission(user_data, calibration, Action.READ).permitted + for calibration in score_set.score_calibrations + ) + + # Experiment keywords shown on the variant page's measurement table; how many are filled in breaks evidence ties. ASSAY_FACT_KEYWORD_KEYS = frozenset( - {"Phenotypic Assay Method", "Molecular Mechanism Assessed", "Phenotypic Assay Model System"} + { + "Phenotypic Assay Method", + "Molecular Mechanism Assessed", + "Phenotypic Assay Model System", + } ) @@ -311,6 +324,7 @@ def get_allele_measurements( ) measurements: list[tuple[AlleleMeasurement, Optional[date], int]] = [] + clinically_calibrated: dict[int, bool] = {} protein_consequence_count = 0 apex_only_unresolved_count = 0 for record, variant, measured_allele in rows: @@ -344,6 +358,9 @@ def get_allele_measurements( assay_level = SequenceLevel(record.assay_level) if record.assay_level else None preferred = _preferred_classification(db, variant, user_data=user_data) + if score_set.id not in clinically_calibrated: + clinically_calibrated[score_set.id] = _has_readable_clinical_calibration(score_set, user_data=user_data) + measurement = AlleleMeasurement( variant_urn=variant.urn or "", score=variant_score(variant), @@ -356,7 +373,7 @@ def get_allele_measurements( preferred_classification=preferred, research_use_only_classification=( _preferred_classification(db, variant, user_data=user_data, research_use_only=True) - if preferred is None + if not clinically_calibrated[score_set.id] else None ), is_current=is_current, diff --git a/src/mavedb/view_models/allele_measurement.py b/src/mavedb/view_models/allele_measurement.py index 33353df4e..ddcf0f8e6 100644 --- a/src/mavedb/view_models/allele_measurement.py +++ b/src/mavedb/view_models/allele_measurement.py @@ -25,8 +25,8 @@ class AlleleMeasurement(BaseModel): ``nucleotide_encoding`` (a nt measurement encoding a protein query). ``preferredClassification`` is the readable functional classification the UI defaults to (primary-first cascade, RUO excluded), omitted when absent or gated; ranking uses it alone. ``researchUseOnlyClassification`` is a display-only - fallback from research-use-only calibrations, present only when ``preferredClassification`` is absent; - it must not be treated as clinical evidence. ``isCurrent`` / + fallback from research-use-only calibrations, present only when the score set has no readable non-RUO + calibration; it must not be treated as clinical evidence. ``isCurrent`` / ``supersededByScoreSet`` let a superseded measurement (surfaced only under ``include_superseded``) self-describe; ``supersededByScoreSet`` is the superseding *score set*'s URN. """ diff --git a/tests/lib/test_allele_measurements.py b/tests/lib/test_allele_measurements.py index da5f675c7..a9f1b37d3 100644 --- a/tests/lib/test_allele_measurements.py +++ b/tests/lib/test_allele_measurements.py @@ -524,18 +524,61 @@ def test_research_use_only_classification_absent_when_a_preferred_call_exists(se assert result[0].research_use_only_classification is None +@pytest.mark.integration +def test_research_use_only_classification_absent_when_a_clinical_calibration_leaves_the_variant_unclassified( + session, setup_lib_db_with_score_set +): + """A non-RUO calibration that doesn't classify the variant still blocks the RUO fallback: the clinical + calibration's silence is its answer.""" + score_set = setup_lib_db_with_score_set + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + variant = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}) + other = _variant(session, score_set, 2, data={"score_data": {"score": 1.0}}) + _link(session, _record(session, variant, assay_level="cdna"), nt, is_authoritative=True) + + _calibration(session, score_set, variants=[variant], primary=False, oddspaths_ratio=1000.0, research_use_only=True) + _calibration(session, score_set, variants=[other], primary=True, oddspaths_ratio=2.0) + + result = get_allele_measurements(session, "CA123", user_data=_user_data(session)) + + assert result[0].preferred_classification is None + assert result[0].research_use_only_classification is None + + +@pytest.mark.integration +def test_research_use_only_classification_falls_back_past_an_unreadable_clinical_calibration( + session, setup_lib_db_with_score_set +): + """A non-RUO calibration the caller can't read doesn't count, so the RUO fallback still shows.""" + public = _second_score_set(session, setup_lib_db_with_score_set, "urn:mavedb:00000002-a-1", private=False) + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + variant = _variant(session, public, 1, data={"score_data": {"score": 1.0}}) + _link(session, _record(session, variant, assay_level="cdna"), nt, is_authoritative=True) + + _calibration(session, public, variants=[variant], primary=False, oddspaths_ratio=1000.0, research_use_only=True) + _calibration(session, public, variants=[variant], primary=False, oddspaths_ratio=2.0, private=True) + + result = get_allele_measurements(session, "CA123", user_data=None) + + assert result[0].preferred_classification is None + assert result[0].research_use_only_classification.oddspaths_ratio == 1000.0 + + @pytest.mark.integration def test_research_use_only_classification_does_not_rank(session, setup_lib_db_with_score_set): """A measurement carrying only an RUO call ranks as unclassified: below one with a non-RUO call, however strong the RUO evidence.""" score_set = setup_lib_db_with_score_set + ruo_score_set = _second_score_set(session, score_set, "urn:mavedb:00000001-a-2", private=False) nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") - ruo_only = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}) + ruo_only = _variant(session, ruo_score_set, 1, data={"score_data": {"score": 1.0}}) classified = _variant(session, score_set, 2, data={"score_data": {"score": 1.0}}) _link(session, _record(session, ruo_only, assay_level="cdna"), nt, is_authoritative=True) _link(session, _record(session, classified, assay_level="cdna"), nt, is_authoritative=True) - _calibration(session, score_set, variants=[ruo_only], primary=False, oddspaths_ratio=1000.0, research_use_only=True) + _calibration( + session, ruo_score_set, variants=[ruo_only], primary=False, oddspaths_ratio=1000.0, research_use_only=True + ) _calibration(session, score_set, variants=[classified], primary=False, oddspaths_ratio=2.0) result = get_allele_measurements(session, "CA123", user_data=_user_data(session)) From 99c54c2cb2e579cfdc69f34ebe74933b8a55e758 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Thu, 8 Oct 2026 13:06:59 -0700 Subject: [PATCH 228/252] feat(variant): score distribution summaries and opt-in research-use-only calls Score sets store a 24-bin summary of their scores in a new nullable score_distribution column, written by the variant creation job in the same transaction as the variants and served as scoreDistribution on the ScoreSet view model. Bins span the full [min, max] of the numeric scores; null and non-finite scores are counted separately. backfill_score_distributions fills existing score sets after migration b4d8f2a6c1e3. Allele measurements carry a single classification instead of separate preferred and research-use-only fields. A research-use-only call stands in only when the request sets include_research_use_only and the score set has no readable non-RUO calibration with functional classifications; classificationIsResearchUseOnly marks it. Ordering uses non-RUO calls alone. --- ...d8f2a6c1e3_score_set_score_distribution.py | 30 +++++++ src/mavedb/lib/allele_measurements.py | 57 ++++++++------ src/mavedb/lib/score_distribution.py | 62 +++++++++++++++ src/mavedb/models/score_set.py | 11 ++- src/mavedb/routers/clingen_alleles.py | 10 +++ .../scripts/backfill_score_distributions.py | 74 ++++++++++++++++++ src/mavedb/view_models/allele_measurement.py | 23 ++---- src/mavedb/view_models/score_distribution.py | 15 ++++ src/mavedb/view_models/score_set.py | 2 + .../jobs/variant_processing/creation.py | 2 + tests/lib/test_allele_measurements.py | 69 ++++++++++------ tests/lib/test_score_distribution.py | 78 +++++++++++++++++++ tests/routers/test_clingen_allele.py | 48 +++++++++++- tests/routers/test_score_set.py | 17 ++++ tests/view_models/test_allele_measurement.py | 33 ++------ .../jobs/variant_processing/test_creation.py | 71 +++++++++++++++-- 16 files changed, 508 insertions(+), 94 deletions(-) create mode 100644 alembic/versions/b4d8f2a6c1e3_score_set_score_distribution.py create mode 100644 src/mavedb/lib/score_distribution.py create mode 100644 src/mavedb/scripts/backfill_score_distributions.py create mode 100644 src/mavedb/view_models/score_distribution.py create mode 100644 tests/lib/test_score_distribution.py diff --git a/alembic/versions/b4d8f2a6c1e3_score_set_score_distribution.py b/alembic/versions/b4d8f2a6c1e3_score_set_score_distribution.py new file mode 100644 index 000000000..364018af9 --- /dev/null +++ b/alembic/versions/b4d8f2a6c1e3_score_set_score_distribution.py @@ -0,0 +1,30 @@ +"""store a binned score distribution on each score set + +Revision ID: b4d8f2a6c1e3 +Revises: a7c3e5f9b1d2 +Create Date: 2026-10-08 + +Adds a nullable score_distribution JSONB column, written by the variant creation job (see +mavedb.lib.score_distribution). Adding a nullable column without a default is a catalog-only change and +does not rewrite the table. Existing score sets stay null until the backfill runs: + + python -m mavedb.scripts.backfill_score_distributions --commit +""" + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = "b4d8f2a6c1e3" +down_revision = "a7c3e5f9b1d2" +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column("scoresets", sa.Column("score_distribution", postgresql.JSONB(astext_type=sa.Text()), nullable=True)) + + +def downgrade(): + op.drop_column("scoresets", "score_distribution") diff --git a/src/mavedb/lib/allele_measurements.py b/src/mavedb/lib/allele_measurements.py index 8acd45f5c..f56f79e09 100644 --- a/src/mavedb/lib/allele_measurements.py +++ b/src/mavedb/lib/allele_measurements.py @@ -74,11 +74,11 @@ class MeasurementRelationship(str, Enum): class AlleleMeasurement: """One measurement in the query's equivalence class. - ``assay_level`` is the sequence level at which this measurement was assayed. ``preferred_classification`` - is the readable classification the UI will default to; it never comes from a research-use-only calibration, - so ranking and any consumer treating it as clinical evidence can rely on that. ``research_use_only_classification`` - is a display-only fallback: the RUO pick, set only when the score set has no readable non-RUO calibration at - all. A non-RUO calibration that leaves this variant unclassified is a call in itself and is not overridden. + ``assay_level`` is the sequence level at which this measurement was assayed. ``classification`` is the + readable call for this variant. It comes from a non-RUO calibration unless the caller opted into research-use-only + calls and the score set has no readable non-RUO calibration with functional classifications; then it may come + from an RUO calibration, and ``classification_is_research_use_only`` says so. A non-RUO calibration that leaves + this variant unclassified is a call in itself and is not overridden. """ variant_urn: str @@ -89,8 +89,8 @@ class AlleleMeasurement: submitted_hgvs: Optional[str] score_set_urn: str score_set_title: str - preferred_classification: Optional[ScoreCalibrationFunctionalClassification] - research_use_only_classification: Optional[ScoreCalibrationFunctionalClassification] + classification: Optional[ScoreCalibrationFunctionalClassification] + classification_is_research_use_only: bool is_current: bool superseded_by_score_set: Optional[str] @@ -154,9 +154,15 @@ def preference(pair: tuple[ScoreCalibrationFunctionalClassification, ScoreCalibr def _has_readable_clinical_calibration(score_set: ScoreSet, *, user_data: Optional[UserData]) -> bool: - """Whether the score set has any non-RUO calibration the caller can read, classifying this variant or not.""" + """Whether the score set has any non-RUO calibration the caller can read, classifying this variant or not. + + A calibration without functional classifications (one that only records controls or a baseline score) can't + classify any variant, so its silence is no answer and it doesn't count. + """ return any( - not calibration.research_use_only and has_permission(user_data, calibration, Action.READ).permitted + not calibration.research_use_only + and calibration.functional_classifications + and has_permission(user_data, calibration, Action.READ).permitted for calibration in score_set.score_calibrations ) @@ -179,12 +185,16 @@ def _assay_fact_count(score_set: ScoreSet) -> int: def _ordering_key(measurement: AlleleMeasurement, published_date: Optional[date], assay_fact_count: int) -> tuple: """Sort: current before superseded; direct before related; strongest evidence (pathogenic wins ties) - first; most assay facts provided first; newest-published first; then URN for a stable tiebreak.""" - magnitude, direction = classification_evidence_strength(measurement.preferred_classification) + first; most assay facts provided first; newest-published first; then URN for a stable tiebreak. + + Only non-RUO calls rank, so opting into research-use-only calls never reorders the list. + """ + clinical = None if measurement.classification_is_research_use_only else measurement.classification + magnitude, direction = classification_evidence_strength(clinical) return ( 0 if measurement.is_current else 1, 0 if measurement.relationship == MeasurementRelationship.direct else 1, - 0 if measurement.preferred_classification is not None else 1, + 0 if clinical is not None else 1, -magnitude, direction, -assay_fact_count, @@ -231,10 +241,12 @@ def get_allele_measurements( *, user_data: Optional[UserData], include_superseded: bool = False, + include_research_use_only: bool = False, as_of: Optional[datetime] = None, ) -> list[AlleleMeasurement]: """The measurements in ``clingen_allele_id``'s equivalence class, or ``[]`` if it resolves to no live - record. + record. ``include_research_use_only`` lets a research-use-only call stand in where the score set has no + readable clinical calibration (see :class:`AlleleMeasurement`). A CA query returns ``direct`` measurements, its ``protein_consequence``, and the ``nucleotide_encoding`` members. A PA query returns its ``direct`` protein measurements and their ``nucleotide_encoding`` encodings. @@ -357,9 +369,14 @@ def get_allele_measurements( apex_only_unresolved_count += 1 assay_level = SequenceLevel(record.assay_level) if record.assay_level else None - preferred = _preferred_classification(db, variant, user_data=user_data) - if score_set.id not in clinically_calibrated: - clinically_calibrated[score_set.id] = _has_readable_clinical_calibration(score_set, user_data=user_data) + classification = _preferred_classification(db, variant, user_data=user_data) + classification_is_research_use_only = False + if include_research_use_only and classification is None: + if score_set.id not in clinically_calibrated: + clinically_calibrated[score_set.id] = _has_readable_clinical_calibration(score_set, user_data=user_data) + if not clinically_calibrated[score_set.id]: + classification = _preferred_classification(db, variant, user_data=user_data, research_use_only=True) + classification_is_research_use_only = classification is not None measurement = AlleleMeasurement( variant_urn=variant.urn or "", @@ -370,12 +387,8 @@ def get_allele_measurements( submitted_hgvs=variant.hgvs_pro if assay_level == SequenceLevel.protein.value else variant.hgvs_nt, score_set_urn=score_set.urn or "", score_set_title=score_set.title or "", - preferred_classification=preferred, - research_use_only_classification=( - _preferred_classification(db, variant, user_data=user_data, research_use_only=True) - if not clinically_calibrated[score_set.id] - else None - ), + classification=classification, + classification_is_research_use_only=classification_is_research_use_only, is_current=is_current, superseded_by_score_set=superseding.urn if superseding is not None else None, ) diff --git a/src/mavedb/lib/score_distribution.py b/src/mavedb/lib/score_distribution.py new file mode 100644 index 000000000..2c784c3a5 --- /dev/null +++ b/src/mavedb/lib/score_distribution.py @@ -0,0 +1,62 @@ +"""Precomputed summaries of a score set's score distribution. + +The variant page draws a small histogram of each measured score set beside the variant's score. Fetching +every variant to draw it is too slow for large score sets, so the summary is computed once when scores are +written and stored on the score set (``ScoreSet.score_distribution``). +""" + +import math +from typing import Any, Iterable, Optional + +from mavedb.lib.variants import score_from_variant_data + +SCORE_DISTRIBUTION_VERSION = 1 +"""Bump when the binning changes, then re-run the backfill with ``--force``.""" + +SCORE_DISTRIBUTION_BIN_COUNT = 24 + + +def summarize_scores(scores: Iterable[Optional[float]]) -> Optional[dict[str, Any]]: + """Bin scores into equal-width bins spanning their full range. + + The full ``[min, max]`` is used rather than a clipped range: percentile clipping always hides some + scores, and IQR fences drop a whole mode in bimodal genome-editing data. + + Args: + scores: Scores to summarize. ``None`` and non-finite entries are counted in ``null_count`` and not binned. + + Returns: + ``{"version", "min", "max", "counts", "null_count"}``, or ``None`` when there are no numeric scores. + The maximum falls in the last bin. When every score is the same value, all of them fall in the + first bin. + """ + values: list[float] = [] + null_count = 0 + for score in scores: + if score is None or not math.isfinite(score): + null_count += 1 + else: + values.append(score) + + if not values: + return None + + low, high = min(values), max(values) + counts = [0] * SCORE_DISTRIBUTION_BIN_COUNT + width = (high - low) / SCORE_DISTRIBUTION_BIN_COUNT + for value in values: + index = 0 if width == 0 else min(int((value - low) / width), SCORE_DISTRIBUTION_BIN_COUNT - 1) + counts[index] += 1 + + return { + "version": SCORE_DISTRIBUTION_VERSION, + "min": low, + "max": high, + "counts": counts, + "null_count": null_count, + } + + +def summarize_variant_data(variant_data: Iterable[Optional[dict[str, Any]]]) -> Optional[dict[str, Any]]: + """Summarize scores read from variants' ``data`` JSONB. See :func:`summarize_scores`.""" + return summarize_scores(score_from_variant_data(data) for data in variant_data) diff --git a/src/mavedb/models/score_set.py b/src/mavedb/models/score_set.py index b61704ffe..1c71bf6b5 100644 --- a/src/mavedb/models/score_set.py +++ b/src/mavedb/models/score_set.py @@ -101,6 +101,8 @@ class ScoreSet(Base): nullable=True, ) processing_errors = Column(JSONB, nullable=True) + # Binned summary for small distribution charts; see mavedb.lib.score_distribution. Null until computed. + score_distribution = Column(JSONB, nullable=True) data_usage_policy = Column(String, nullable=True) num_variants = Column(Integer, nullable=False, default=0) @@ -124,7 +126,14 @@ class ScoreSet(Base): # TODO Standardize on US or GB spelling for licenc/se. licence_id = Column(Integer, ForeignKey("licenses.id"), index=True, nullable=False) license: Mapped["License"] = relationship("License") - superseded_score_set_id = Column("replaces_id", Integer, ForeignKey("scoresets.id"), index=True, nullable=True, unique=True,) + superseded_score_set_id = Column( + "replaces_id", + Integer, + ForeignKey("scoresets.id"), + index=True, + nullable=True, + unique=True, + ) superseded_score_set: Mapped[Optional["ScoreSet"]] = relationship( "ScoreSet", uselist=False, diff --git a/src/mavedb/routers/clingen_alleles.py b/src/mavedb/routers/clingen_alleles.py index b3b626c02..a686f3ff7 100644 --- a/src/mavedb/routers/clingen_alleles.py +++ b/src/mavedb/routers/clingen_alleles.py @@ -50,6 +50,14 @@ def get_clingen_allele_measurements( "measurements are a deliberate power-user / citation path, never surfaced by discovery." ), ), + include_research_use_only: bool = Query( + default=False, + description=( + "Let a call from a research-use-only calibration stand in when the score set has no readable " + "calibration that is not research use only. Such calls set classificationIsResearchUseOnly and are " + "not clinical evidence. Default false; ordering never uses them." + ), + ), as_of: Optional[datetime] = Query( default=None, description=( @@ -72,6 +80,7 @@ def get_clingen_allele_measurements( "requested_resource": clingen_allele_id, "as_of": as_of, "include_superseded": include_superseded, + "include_research_use_only": include_research_use_only, } ) response.headers["X-As-Of"] = as_of.isoformat() if as_of is not None else "current" @@ -81,5 +90,6 @@ def get_clingen_allele_measurements( clingen_allele_id, user_data=user_data, include_superseded=include_superseded, + include_research_use_only=include_research_use_only, as_of=as_of, ) diff --git a/src/mavedb/scripts/backfill_score_distributions.py b/src/mavedb/scripts/backfill_score_distributions.py new file mode 100644 index 000000000..072269f51 --- /dev/null +++ b/src/mavedb/scripts/backfill_score_distributions.py @@ -0,0 +1,74 @@ +"""Fill ``ScoreSet.score_distribution`` for score sets whose variants predate the column. + +New uploads compute the summary in the variant creation job; this one-off covers existing score sets. +Score sets are processed one at a time so only one score set's variant data is in memory at once. + +Usage: + + python -m mavedb.scripts.backfill_score_distributions --dry-run + python -m mavedb.scripts.backfill_score_distributions --commit + python -m mavedb.scripts.backfill_score_distributions --commit --force +""" + +import logging + +import asyncclick as click +from sqlalchemy import select +from sqlalchemy.orm import Session + +from mavedb.lib.score_distribution import summarize_variant_data +from mavedb.models.score_set import ScoreSet +from mavedb.models.variant import Variant +from mavedb.scripts.environment import script_environment, with_database_session + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + +VARIANT_FETCH_BATCH_SIZE = 10_000 + + +def backfill(db: Session, force: bool = False) -> tuple[int, int]: + """Summarize each score set with variants; returns ``(filled, without_numeric_scores)``.""" + query = select(ScoreSet.id).where(ScoreSet.num_variants > 0).order_by(ScoreSet.id) + if not force: + query = query.where(ScoreSet.score_distribution.is_(None)) + score_set_ids = db.scalars(query).all() + logger.info(f"Summarizing scores for {len(score_set_ids)} score sets.") + + filled = 0 + empty = 0 + for index, score_set_id in enumerate(score_set_ids, start=1): + variant_data = db.scalars( + select(Variant.data) + .where(Variant.score_set_id == score_set_id) + .execution_options(yield_per=VARIANT_FETCH_BATCH_SIZE) + ) + summary = summarize_variant_data(variant_data) + + score_set = db.get_one(ScoreSet, score_set_id) + score_set.score_distribution = summary + db.flush() + db.expunge(score_set) + + if summary is None: + empty += 1 + else: + filled += 1 + + if index % 100 == 0: + logger.info(f"Processed {index} of {len(score_set_ids)} score sets.") + + return filled, empty + + +@script_environment.command() +@with_database_session +@click.option("--force", is_flag=True, help="Recompute summaries that already exist.") +def backfill_score_distributions(db: Session, force: bool) -> None: + """Compute the score distribution summary for every score set with variants.""" + filled, empty = backfill(db, force) + logger.info(f"Filled {filled} score distributions; {empty} score sets had no numeric scores.") + + +if __name__ == "__main__": + backfill_score_distributions() diff --git a/src/mavedb/view_models/allele_measurement.py b/src/mavedb/view_models/allele_measurement.py index ddcf0f8e6..ebf8ffcbe 100644 --- a/src/mavedb/view_models/allele_measurement.py +++ b/src/mavedb/view_models/allele_measurement.py @@ -7,8 +7,6 @@ from typing import Optional -from pydantic import model_validator - from mavedb.lib.allele_measurements import MeasurementRelationship from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.view_models.base.base import BaseModel @@ -22,11 +20,11 @@ class AlleleMeasurement(BaseModel): ``genomic``) — always shown, since the measured level is the clinically load-bearing fact. ``relationship`` says how the measurement relates to the queried ClinGen id: ``direct`` (assayed at this allele), ``protein_consequence`` (a protein measurement of a nt query's consequence), or - ``nucleotide_encoding`` (a nt measurement encoding a protein query). ``preferredClassification`` is the - readable functional classification the UI defaults to (primary-first cascade, RUO excluded), omitted - when absent or gated; ranking uses it alone. ``researchUseOnlyClassification`` is a display-only - fallback from research-use-only calibrations, present only when the score set has no readable non-RUO - calibration; it must not be treated as clinical evidence. ``isCurrent`` / + ``nucleotide_encoding`` (a nt measurement encoding a protein query). ``classification`` is the readable + functional classification for the variant (primary-first cascade), omitted when absent or gated. It comes + from a research-use-only calibration only when the request sets ``include_research_use_only`` and the score + set has no readable non-RUO calibration; ``classificationIsResearchUseOnly`` then marks it, and it must not + be treated as clinical evidence. Ranking uses non-RUO calls alone. ``isCurrent`` / ``supersededByScoreSet`` let a superseded measurement (surfaced only under ``include_superseded``) self-describe; ``supersededByScoreSet`` is the superseding *score set*'s URN. """ @@ -39,17 +37,10 @@ class AlleleMeasurement(BaseModel): submitted_hgvs: Optional[str] = None score_set_urn: str score_set_title: str - preferred_classification: Optional[SavedFunctionalClassification] = None - research_use_only_classification: Optional[SavedFunctionalClassification] = None + classification: Optional[SavedFunctionalClassification] = None + classification_is_research_use_only: bool = False is_current: bool superseded_by_score_set: Optional[str] = None - @model_validator(mode="after") - def classifications_are_mutually_exclusive(self) -> "AlleleMeasurement": - """The RUO fallback exists only where no preferred call does; both set means the builder broke that.""" - if self.preferred_classification is not None and self.research_use_only_classification is not None: - raise ValueError("researchUseOnlyClassification may only be set when preferredClassification is absent.") - return self - class Config: from_attributes = True diff --git a/src/mavedb/view_models/score_distribution.py b/src/mavedb/view_models/score_distribution.py new file mode 100644 index 000000000..b4272b18e --- /dev/null +++ b/src/mavedb/view_models/score_distribution.py @@ -0,0 +1,15 @@ +from mavedb.view_models.base.base import BaseModel + + +class ScoreDistribution(BaseModel): + """Binned summary of a score set's scores, built by ``mavedb.lib.score_distribution.summarize_scores``. + + ``counts`` holds equal-width bins spanning ``[min, max]``; scores that are null or not finite are + counted in ``null_count`` instead. + """ + + version: int + min: float + max: float + counts: list[int] + null_count: int diff --git a/src/mavedb/view_models/score_set.py b/src/mavedb/view_models/score_set.py index ce7a0730a..c10a5353f 100644 --- a/src/mavedb/view_models/score_set.py +++ b/src/mavedb/view_models/score_set.py @@ -38,6 +38,7 @@ ScoreCalibration, ScoreCalibrationCreate, ) +from mavedb.view_models.score_distribution import ScoreDistribution from mavedb.view_models.score_set_dataset_columns import DatasetColumns, SavedDatasetColumns from mavedb.view_models.target_gene import ( SavedTargetGene, @@ -439,6 +440,7 @@ class ScoreSet(SavedScoreSet): mapping_errors: Optional[dict] = None score_calibrations: Optional[Sequence[ScoreCalibration]] = None # type: ignore[assignment] dataset_columns: Optional[DatasetColumns] = None # type: ignore[assignment] + score_distribution: Optional[ScoreDistribution] = None class ScoreSetWithVariants(ScoreSet): diff --git a/src/mavedb/worker/jobs/variant_processing/creation.py b/src/mavedb/worker/jobs/variant_processing/creation.py index 480705ecb..333f167f0 100644 --- a/src/mavedb/worker/jobs/variant_processing/creation.py +++ b/src/mavedb/worker/jobs/variant_processing/creation.py @@ -17,6 +17,7 @@ restore_calibration_variant_links, snapshot_calibration_variant_links, ) +from mavedb.lib.score_distribution import summarize_variant_data from mavedb.lib.score_sets import columns_for_dataset, create_variants, create_variants_data from mavedb.lib.types.score_calibrations import CalibrationVariantLinkSnapshot from mavedb.lib.types.workflow import JobExecutionOutcome @@ -225,6 +226,7 @@ async def create_variants_for_score_set(ctx: dict, job_id: int, job_manager: Job job_manager.db.refresh(score_set) variants_data = create_variants_data(validated_scores, validated_counts, None) + score_set.score_distribution = summarize_variant_data(variant["data"] for variant in variants_data) create_variants(job_manager.db, score_set, variants_data) # Scoped to the replacement path above: only links the delete broke need re-establishing, so a diff --git a/tests/lib/test_allele_measurements.py b/tests/lib/test_allele_measurements.py index a9f1b37d3..4f969a670 100644 --- a/tests/lib/test_allele_measurements.py +++ b/tests/lib/test_allele_measurements.py @@ -445,11 +445,11 @@ def test_calibration_gate_withholds_classification(session, setup_lib_db_with_sc anonymous = get_allele_measurements(session, "CA123", user_data=None) assert len(anonymous) == 1 - assert anonymous[0].preferred_classification is None + assert anonymous[0].classification is None owner = get_allele_measurements(session, "CA123", user_data=_user_data(session)) - assert owner[0].preferred_classification is not None - assert owner[0].preferred_classification.functional_classification.value == "abnormal" + assert owner[0].classification is not None + assert owner[0].classification.functional_classification.value == "abnormal" @pytest.mark.integration @@ -468,7 +468,7 @@ def test_preferred_classification_cascade_beats_strength(session, setup_lib_db_w result = get_allele_measurements(session, "CA123", user_data=_user_data(session)) # Investigator-provided (weakest) is chosen over the stronger non-RUO and RUO calibrations. - assert result[0].preferred_classification.oddspaths_ratio == 2.0 + assert result[0].classification.oddspaths_ratio == 2.0 @pytest.mark.integration @@ -484,7 +484,7 @@ def test_preferred_classification_strongest_within_tier(session, setup_lib_db_wi result = get_allele_measurements(session, "CA123", user_data=_user_data(session)) - assert result[0].preferred_classification.oddspaths_ratio == 100.0 + assert result[0].classification.oddspaths_ratio == 100.0 @pytest.mark.integration @@ -500,15 +500,17 @@ def test_research_use_only_calibrations_are_excluded(session, setup_lib_db_with_ _calibration(session, score_set, variants=[variant], primary=False, oddspaths_ratio=1000.0, research_use_only=True) result = get_allele_measurements(session, "CA123", user_data=_user_data(session)) + assert result[0].classification is None - assert result[0].preferred_classification is None - # The RUO pick is still surfaced for display, by the same cascade (strongest evidence within the tier). - assert result[0].research_use_only_classification.oddspaths_ratio == 1000.0 + # Opted in, the RUO pick stands in, by the same cascade (strongest evidence within the tier). + opted = get_allele_measurements(session, "CA123", user_data=_user_data(session), include_research_use_only=True) + assert opted[0].classification.oddspaths_ratio == 1000.0 + assert opted[0].classification_is_research_use_only @pytest.mark.integration def test_research_use_only_classification_absent_when_a_preferred_call_exists(session, setup_lib_db_with_score_set): - """The RUO fallback is display-only: with any non-RUO call present it stays empty, even when the RUO call + """The RUO fallback never displaces a non-RUO call, even when the RUO call is stronger.""" score_set = setup_lib_db_with_score_set nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") @@ -518,10 +520,10 @@ def test_research_use_only_classification_absent_when_a_preferred_call_exists(se _calibration(session, score_set, variants=[variant], primary=False, oddspaths_ratio=1000.0, research_use_only=True) _calibration(session, score_set, variants=[variant], primary=False, oddspaths_ratio=2.0) - result = get_allele_measurements(session, "CA123", user_data=_user_data(session)) + result = get_allele_measurements(session, "CA123", user_data=_user_data(session), include_research_use_only=True) - assert result[0].preferred_classification.oddspaths_ratio == 2.0 - assert result[0].research_use_only_classification is None + assert result[0].classification.oddspaths_ratio == 2.0 + assert not result[0].classification_is_research_use_only @pytest.mark.integration @@ -539,10 +541,33 @@ def test_research_use_only_classification_absent_when_a_clinical_calibration_lea _calibration(session, score_set, variants=[variant], primary=False, oddspaths_ratio=1000.0, research_use_only=True) _calibration(session, score_set, variants=[other], primary=True, oddspaths_ratio=2.0) - result = get_allele_measurements(session, "CA123", user_data=_user_data(session)) + result = get_allele_measurements(session, "CA123", user_data=_user_data(session), include_research_use_only=True) + + assert result[0].classification is None + + +@pytest.mark.integration +def test_research_use_only_classification_falls_back_past_a_clinical_calibration_without_classifications( + session, setup_lib_db_with_score_set +): + """A non-RUO calibration with no functional classifications can't classify anything, so it doesn't block the + RUO fallback.""" + score_set = setup_lib_db_with_score_set + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + variant = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}) + _link(session, _record(session, variant, assay_level="cdna"), nt, is_authoritative=True) - assert result[0].preferred_classification is None - assert result[0].research_use_only_classification is None + _calibration(session, score_set, variants=[variant], primary=False, oddspaths_ratio=1000.0, research_use_only=True) + empty = _calibration(session, score_set, variants=[], primary=True) + for classification in empty.functional_classifications: + session.delete(classification) + session.commit() + session.refresh(score_set) + + result = get_allele_measurements(session, "CA123", user_data=_user_data(session), include_research_use_only=True) + + assert result[0].classification.oddspaths_ratio == 1000.0 + assert result[0].classification_is_research_use_only @pytest.mark.integration @@ -558,10 +583,10 @@ def test_research_use_only_classification_falls_back_past_an_unreadable_clinical _calibration(session, public, variants=[variant], primary=False, oddspaths_ratio=1000.0, research_use_only=True) _calibration(session, public, variants=[variant], primary=False, oddspaths_ratio=2.0, private=True) - result = get_allele_measurements(session, "CA123", user_data=None) + result = get_allele_measurements(session, "CA123", user_data=None, include_research_use_only=True) - assert result[0].preferred_classification is None - assert result[0].research_use_only_classification.oddspaths_ratio == 1000.0 + assert result[0].classification.oddspaths_ratio == 1000.0 + assert result[0].classification_is_research_use_only @pytest.mark.integration @@ -581,10 +606,10 @@ def test_research_use_only_classification_does_not_rank(session, setup_lib_db_wi ) _calibration(session, score_set, variants=[classified], primary=False, oddspaths_ratio=2.0) - result = get_allele_measurements(session, "CA123", user_data=_user_data(session)) + result = get_allele_measurements(session, "CA123", user_data=_user_data(session), include_research_use_only=True) assert [m.variant_urn for m in result] == [classified.urn, ruo_only.urn] - assert result[1].research_use_only_classification is not None + assert result[1].classification_is_research_use_only @pytest.mark.integration @@ -680,8 +705,8 @@ def _measurement(urn, *, relationship=MeasurementRelationship.direct, classifica submitted_hgvs=None, score_set_urn="", score_set_title="", - preferred_classification=classification, - research_use_only_classification=None, + classification=classification, + classification_is_research_use_only=False, is_current=is_current, superseded_by_score_set=None, ) diff --git a/tests/lib/test_score_distribution.py b/tests/lib/test_score_distribution.py new file mode 100644 index 000000000..0f26dd5cb --- /dev/null +++ b/tests/lib/test_score_distribution.py @@ -0,0 +1,78 @@ +import math + +import pytest + +from mavedb.lib.score_distribution import ( + SCORE_DISTRIBUTION_BIN_COUNT, + SCORE_DISTRIBUTION_VERSION, + summarize_scores, + summarize_variant_data, +) + + +@pytest.mark.parametrize("scores", [[], [None], [None, math.nan, math.inf]]) +def test_summarize_scores_without_numeric_scores_returns_none(scores): + assert summarize_scores(scores) is None + + +def test_summarize_scores_shape(): + summary = summarize_scores([0.0, 1.0]) + + assert summary is not None + assert summary["version"] == SCORE_DISTRIBUTION_VERSION + assert summary["min"] == 0.0 + assert summary["max"] == 1.0 + assert len(summary["counts"]) == SCORE_DISTRIBUTION_BIN_COUNT + assert summary["null_count"] == 0 + + +def test_summarize_scores_puts_min_in_first_bin_and_max_in_last_bin(): + summary = summarize_scores([-2.0, 0.0, 4.0]) + + assert summary is not None + assert summary["counts"][0] == 1 + assert summary["counts"][-1] == 1 + assert sum(summary["counts"]) == 3 + + +def test_summarize_scores_bins_by_equal_width(): + # Width is 1.0 over [0, 24]; 11.5 falls in bin 11 and 12.0 starts bin 12. + summary = summarize_scores([0.0, 11.5, 12.0, 24.0]) + + assert summary is not None + assert summary["counts"][11] == 1 + assert summary["counts"][12] == 1 + + +def test_summarize_scores_single_repeated_value_lands_in_first_bin(): + summary = summarize_scores([3.5, 3.5, 3.5]) + + assert summary is not None + assert summary["min"] == summary["max"] == 3.5 + assert summary["counts"][0] == 3 + assert sum(summary["counts"]) == 3 + + +def test_summarize_scores_counts_nulls_separately(): + summary = summarize_scores([1.0, None, 2.0, math.nan, 3.0]) + + assert summary is not None + assert summary["null_count"] == 2 + assert sum(summary["counts"]) == 3 + assert (summary["min"], summary["max"]) == (1.0, 3.0) + + +def test_summarize_variant_data_reads_canonical_score(): + variant_data = [ + {"score_data": {"score": 1.0}}, + {"score_data": {"score": "2.0"}}, + {"score_data": {"score": None}}, + {"score_data": {}}, + None, + ] + + summary = summarize_variant_data(variant_data) + + assert summary is not None + assert sum(summary["counts"]) == 2 + assert summary["null_count"] == 3 diff --git a/tests/routers/test_clingen_allele.py b/tests/routers/test_clingen_allele.py index 4157c27db..8569465f1 100644 --- a/tests/routers/test_clingen_allele.py +++ b/tests/routers/test_clingen_allele.py @@ -4,7 +4,7 @@ Exercise the HTTP surface: the equivalence-class list serializes and validates, the ``as_of`` content-time is echoed in ``X-As-Of``, an unknown id is an empty list (not a 404), superseded -measurements are opt-in, a private score set's measurement never leaks, and the response body's +measurements and research-use-only calls are opt-in, a private score set's measurement never leaks, and the response body's order reflects the ranked sort (current-then-superseded, direct-then-related, newest-published, urn). """ @@ -18,12 +18,17 @@ from sqlalchemy import select +from mavedb.lib.mondo import get_generic_disease_term from mavedb.models.allele import Allele from mavedb.models.mapping_record import MappingRecord from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.score_calibration import ScoreCalibration +from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification from mavedb.models.score_set import ScoreSet as ScoreSetDbModel +from mavedb.models.user import User from mavedb.models.variant import Variant as VariantDbModel from mavedb.view_models.allele_measurement import AlleleMeasurement +from tests.helpers.constants import TEST_USER from tests.helpers.dependency_overrider import DependencyOverrider from tests.helpers.util.experiment import create_experiment from tests.helpers.util.score_set import create_seq_score_set_with_variants @@ -81,6 +86,47 @@ def test_measurements_serialize(client, session, data_provider, data_files, setu assert response.headers["X-As-Of"] == "current" +def test_research_use_only_call_is_opt_in(client, session, data_provider, data_files, setup_router_db): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + urn = f"{score_set['urn']}#1" + _seed_cdna_measurement(session, urn) + + user = session.scalar(select(User).where(User.username == TEST_USER["username"])) + calibration = ScoreCalibration( + score_set_id=session.scalar(select(ScoreSetDbModel.id).where(ScoreSetDbModel.urn == score_set["urn"])), + title="Research calibration", + primary=False, + private=False, + research_use_only=True, + disease_term=get_generic_disease_term(session), + created_by_id=user.id, + modified_by_id=user.id, + ) + session.add(calibration) + session.commit() + classification = ScoreCalibrationFunctionalClassification( + calibration_id=calibration.id, + label="abnormal", + functional_classification="abnormal", + range=[None, 0.0], + inclusive_lower_bound=False, + ) + classification.variants = [session.scalar(select(VariantDbModel).where(VariantDbModel.urn == urn))] + session.add(classification) + session.commit() + + default = client.get("/api/v1/clingen-alleles/CA123/measurements").json() + assert "classification" not in default[0] + assert default[0]["classificationIsResearchUseOnly"] is False + + opted = client.get("/api/v1/clingen-alleles/CA123/measurements", params={"include_research_use_only": True}).json() + assert opted[0]["classification"]["functionalClassification"] == "abnormal" + assert opted[0]["classificationIsResearchUseOnly"] is True + + def test_unknown_clingen_id_returns_empty_list(client, setup_router_db): response = client.get("/api/v1/clingen-alleles/CA000/measurements") assert response.status_code == 200 diff --git a/tests/routers/test_score_set.py b/tests/routers/test_score_set.py index 03cfe8e88..9f1710587 100644 --- a/tests/routers/test_score_set.py +++ b/tests/routers/test_score_set.py @@ -23,6 +23,7 @@ from mavedb.lib.csv import build_limiter from mavedb.lib.annotation.exceptions import MappingDataDoesntExistException from mavedb.lib.exceptions import NonexistentOrcidUserError +from mavedb.lib.score_distribution import summarize_scores from mavedb.lib.validation.urn_re import MAVEDB_EXPERIMENT_URN_RE, MAVEDB_SCORE_SET_URN_RE, MAVEDB_TMP_URN_RE from mavedb.models.annotation_event import AnnotationEvent from mavedb.models.clinvar_allele_link import ClinvarAlleleLink @@ -892,6 +893,22 @@ def test_get_own_private_score_set(client, setup_router_db): assert (key, expected_response[key]) == (key, response_data[key]) +def test_get_score_set_serves_stored_score_distribution(session, client, setup_router_db): + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + assert "scoreDistribution" not in client.get(f"/api/v1/score-sets/{score_set['urn']}").json() + + item = session.scalars(select(ScoreSetDbModel).where(ScoreSetDbModel.urn == score_set["urn"])).one() + item.score_distribution = summarize_scores([-1.0, 0.5, 2.0, None]) + session.commit() + + response = client.get(f"/api/v1/score-sets/{score_set['urn']}") + assert response.status_code == 200 + distribution = response.json()["scoreDistribution"] + assert (distribution["min"], distribution["max"], distribution["nullCount"]) == (-1.0, 2.0, 1) + assert sum(distribution["counts"]) == 3 + + def test_cannot_get_other_user_private_score_set(session, client, setup_router_db): experiment = create_experiment(client) score_set = create_seq_score_set(client, experiment["urn"]) diff --git a/tests/view_models/test_allele_measurement.py b/tests/view_models/test_allele_measurement.py index b74ebfeb6..39ef233fa 100644 --- a/tests/view_models/test_allele_measurement.py +++ b/tests/view_models/test_allele_measurement.py @@ -1,8 +1,5 @@ -import pytest -from pydantic import ValidationError - from mavedb.view_models.allele_measurement import AlleleMeasurement -from tests.helpers.constants import TEST_SAVED_FUNCTIONAL_RANGE_ABNORMAL, TEST_SAVED_FUNCTIONAL_RANGE_NORMAL +from tests.helpers.constants import TEST_SAVED_FUNCTIONAL_RANGE_ABNORMAL MEASUREMENT = { "variantUrn": "urn:mavedb:00000001-a-1#1", @@ -13,26 +10,10 @@ } -@pytest.mark.parametrize( - "classifications", - [ - {}, - {"preferredClassification": TEST_SAVED_FUNCTIONAL_RANGE_NORMAL}, - {"researchUseOnlyClassification": TEST_SAVED_FUNCTIONAL_RANGE_ABNORMAL}, - ], -) -def test_allele_measurement_accepts_at_most_one_classification(classifications): - measurement = AlleleMeasurement.model_validate({**MEASUREMENT, **classifications}) - - assert (measurement.preferred_classification is None) or (measurement.research_use_only_classification is None) - +def test_allele_measurement_classification_defaults_to_clinical(): + measurement = AlleleMeasurement.model_validate( + {**MEASUREMENT, "classification": TEST_SAVED_FUNCTIONAL_RANGE_ABNORMAL} + ) -def test_allele_measurement_rejects_both_classifications(): - with pytest.raises(ValidationError, match="researchUseOnlyClassification may only be set"): - AlleleMeasurement.model_validate( - { - **MEASUREMENT, - "preferredClassification": TEST_SAVED_FUNCTIONAL_RANGE_NORMAL, - "researchUseOnlyClassification": TEST_SAVED_FUNCTIONAL_RANGE_ABNORMAL, - } - ) + assert measurement.classification is not None + assert not measurement.classification_is_research_use_only diff --git a/tests/worker/jobs/variant_processing/test_creation.py b/tests/worker/jobs/variant_processing/test_creation.py index e791e87a4..5bff5b1a5 100644 --- a/tests/worker/jobs/variant_processing/test_creation.py +++ b/tests/worker/jobs/variant_processing/test_creation.py @@ -6,7 +6,7 @@ import math from datetime import date, timedelta -from unittest.mock import ANY, MagicMock, call, patch +from unittest.mock import ANY, call, patch from mavedb.lib.mondo import get_generic_disease_term from mavedb.models.annotation_event import AnnotationEvent @@ -79,7 +79,7 @@ async def test_create_variants_for_score_set_calls_s3_client_with_correct_parame ), patch( "mavedb.worker.jobs.variant_processing.creation.create_variants_data", - return_value=[MagicMock(spec=Variant)], + return_value=[{"data": {"score_data": {"score": 1.0}}}], ), patch("mavedb.worker.jobs.variant_processing.creation.create_variants", return_value=None), ): @@ -163,7 +163,7 @@ async def test_create_variants_for_score_set_counts_file_can_be_optional( ), patch( "mavedb.worker.jobs.variant_processing.creation.create_variants_data", - return_value=[MagicMock(spec=Variant)], + return_value=[{"data": {"score_data": {"score": 1.0}}}], ), patch("mavedb.worker.jobs.variant_processing.creation.create_variants", return_value=None), ): @@ -280,7 +280,7 @@ async def test_create_variants_for_score_set_removes_existing_variants_before_cr ), patch( "mavedb.worker.jobs.variant_processing.creation.create_variants_data", - return_value=[MagicMock(spec=Variant)], + return_value=[{"data": {"score_data": {"score": 1.0}}}], ), patch("mavedb.worker.jobs.variant_processing.creation.create_variants", return_value=None), ): @@ -352,7 +352,7 @@ async def test_create_variants_for_score_set_removes_existing_mapped_variants_be ), patch( "mavedb.worker.jobs.variant_processing.creation.create_variants_data", - return_value=[MagicMock(spec=Variant)], + return_value=[{"data": {"score_data": {"score": 1.0}}}], ), patch("mavedb.worker.jobs.variant_processing.creation.create_variants", return_value=None), ): @@ -401,7 +401,7 @@ async def test_create_variants_for_score_set_updates_processing_state( ), patch( "mavedb.worker.jobs.variant_processing.creation.create_variants_data", - return_value=[MagicMock(spec=Variant)], + return_value=[{"data": {"score_data": {"score": 1.0}}}], ), patch("mavedb.worker.jobs.variant_processing.creation.create_variants", return_value=None), ): @@ -546,6 +546,12 @@ async def test_create_variants_for_score_set_independent_job( assert sample_score_set.processing_state == ProcessingState.success assert sample_score_set.mapping_state == MappingState.queued + distribution = sample_score_set.score_distribution + assert distribution is not None + assert sum(distribution["counts"]) + distribution["null_count"] == len(created_variants) + assert distribution["min"] == sample_score_dataframe["score"].min() + assert distribution["max"] == sample_score_dataframe["score"].max() + # Verify that the created variants have expected data for variant in created_variants: assert variant.data # Ensure data is not empty @@ -703,6 +709,10 @@ async def test_create_variants_for_score_set_replaces_variants( replaced_variants = session.query(Variant).filter(Variant.score_set_id == sample_score_set.id).all() assert len(replaced_variants) == sample_score_dataframe.shape[0] + session.refresh(sample_score_set) + assert sample_score_set.score_distribution["min"] == updated_score_dataframe["score"].min() + assert sample_score_set.score_distribution["max"] == updated_score_dataframe["score"].max() + # Verify that the variants have been replaced with updated data for variant in replaced_variants: assert variant.data # Ensure data is not empty @@ -954,6 +964,55 @@ async def test_create_variants_for_score_set_generic_exception_handling_during_r ) assert job_run.status == JobStatus.ERRORED + async def test_create_variants_for_score_set_keeps_score_distribution_when_creation_fails( + self, + session, + with_independent_processing_runs, + with_populated_domain_data, + mock_worker_ctx, + mock_s3_client, + sample_score_dataframe, + sample_count_dataframe, + sample_score_set, + sample_independent_variant_creation_run, + ): + with ( + patch.object(mock_s3_client, "download_fileobj", return_value=None), + patch( + "mavedb.worker.jobs.variant_processing.creation.pd.read_csv", + side_effect=[sample_score_dataframe, sample_count_dataframe], + ), + ): + await create_variants_for_score_set(mock_worker_ctx, sample_independent_variant_creation_run.id) + + session.refresh(sample_score_set) + initial_distribution = sample_score_set.score_distribution + assert initial_distribution is not None + + sample_independent_variant_creation_run.status = JobStatus.PENDING + session.commit() + + # The new scores change the summary, which is assigned before the variant insert fails. + updated_score_dataframe = sample_score_dataframe.copy() + updated_score_dataframe["score"] += 10 + with ( + patch.object(mock_s3_client, "download_fileobj", return_value=None), + patch( + "mavedb.worker.jobs.variant_processing.creation.pd.read_csv", + side_effect=[updated_score_dataframe, sample_count_dataframe], + ), + patch( + "mavedb.worker.jobs.variant_processing.creation.create_variants", + side_effect=Exception("Generic exception during variant creation"), + ), + patch("mavedb.worker.lib.decorators.job_management.send_slack_job_error"), + ): + await create_variants_for_score_set(mock_worker_ctx, sample_independent_variant_creation_run.id) + + session.refresh(sample_score_set) + assert sample_score_set.processing_state == ProcessingState.failed + assert sample_score_set.score_distribution == initial_distribution + ## Pipeline failure workflow async def test_create_variants_for_score_set_pipeline_job_generic_exception_handling( From decd6f0a89ac046d026741c220dc3485c20e6c48 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Thu, 8 Oct 2026 13:13:45 -0700 Subject: [PATCH 229/252] fix(scripts): commit the score distribution backfill per score set Run with --commit against a serving database, a single end-of-run commit held row locks on every updated score set for the whole run. Each score set now commits as it finishes, so locks are brief and an interrupted run keeps its progress. --dry-run still rolls everything back. --- .../scripts/backfill_score_distributions.py | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/src/mavedb/scripts/backfill_score_distributions.py b/src/mavedb/scripts/backfill_score_distributions.py index 072269f51..6d28221f1 100644 --- a/src/mavedb/scripts/backfill_score_distributions.py +++ b/src/mavedb/scripts/backfill_score_distributions.py @@ -1,7 +1,9 @@ """Fill ``ScoreSet.score_distribution`` for score sets whose variants predate the column. New uploads compute the summary in the variant creation job; this one-off covers existing score sets. -Score sets are processed one at a time so only one score set's variant data is in memory at once. +Score sets are processed one at a time so only one score set's variant data is in memory at once. With +``--commit``, each score set commits as it finishes, so a run against a serving database never holds row locks on +more than one score set at a time and an interrupted run keeps its progress. Usage: @@ -19,7 +21,7 @@ from mavedb.lib.score_distribution import summarize_variant_data from mavedb.models.score_set import ScoreSet from mavedb.models.variant import Variant -from mavedb.scripts.environment import script_environment, with_database_session +from mavedb.scripts.environment import DatabaseSessionAction, script_environment, with_database_session logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) @@ -27,8 +29,11 @@ VARIANT_FETCH_BATCH_SIZE = 10_000 -def backfill(db: Session, force: bool = False) -> tuple[int, int]: - """Summarize each score set with variants; returns ``(filled, without_numeric_scores)``.""" +def backfill(db: Session, force: bool = False, commit_each: bool = False) -> tuple[int, int]: + """Summarize each score set with variants; returns ``(filled, without_numeric_scores)``. + + ``commit_each`` commits after every score set instead of leaving the transaction to the caller. + """ query = select(ScoreSet.id).where(ScoreSet.num_variants > 0).order_by(ScoreSet.id) if not force: query = query.where(ScoreSet.score_distribution.is_(None)) @@ -47,7 +52,10 @@ def backfill(db: Session, force: bool = False) -> tuple[int, int]: score_set = db.get_one(ScoreSet, score_set_id) score_set.score_distribution = summary - db.flush() + if commit_each: + db.commit() + else: + db.flush() db.expunge(score_set) if summary is None: @@ -62,11 +70,11 @@ def backfill(db: Session, force: bool = False) -> tuple[int, int]: @script_environment.command() -@with_database_session +@with_database_session(pass_action=True) @click.option("--force", is_flag=True, help="Recompute summaries that already exist.") -def backfill_score_distributions(db: Session, force: bool) -> None: +def backfill_score_distributions(db: Session, action: DatabaseSessionAction, force: bool) -> None: """Compute the score distribution summary for every score set with variants.""" - filled, empty = backfill(db, force) + filled, empty = backfill(db, force, commit_each=action is DatabaseSessionAction.COMMIT) logger.info(f"Filled {filled} score distributions; {empty} score sets had no numeric scores.") From 5a06525d822110cbe5ba30ba136177424984d104 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Fri, 9 Oct 2026 06:17:05 -0700 Subject: [PATCH 230/252] fix(reverse-translation): keep projection groups stable across reruns A rerun renumbered projection groups by list position, so a measured allele's group could change while its link stayed live, pairing it with the wrong projection. Groups that fold in the measured allele keep their prior numbers where possible, and a measured-allele link whose group does change is superseded and re-created under the same timestamp. A fold-in that misses the measured allele is now logged, counted, and recorded as fold_in_missed in the candidates metadata. --- .../scripts/audit_reverse_translation.py | 4 +- .../variant_processing/reverse_translation.py | 137 +++++++++--- .../test_reverse_translation.py | 196 +++++++++++++++--- 3 files changed, 280 insertions(+), 57 deletions(-) diff --git a/src/mavedb/scripts/audit_reverse_translation.py b/src/mavedb/scripts/audit_reverse_translation.py index b1ffdbe38..7a0132aa7 100644 --- a/src/mavedb/scripts/audit_reverse_translation.py +++ b/src/mavedb/scripts/audit_reverse_translation.py @@ -215,8 +215,8 @@ class SqlCheck: severity=ERROR, description=( "RT attached derived nucleotide alleles to a record, but the measured nucleotide allele is in no " - "projection group: the fold-in missed it, so its convergent siblings read as projections. Only " - "length-preserving changes, which RT's codon-level candidates always include." + "projection group: the fold-in missed it, so serving labels every sibling convergent, its own c/g " + "twin included. Only length-preserving changes, which RT's codon-level candidates always include." ), sql=_MEASURED_OUTSIDE_GROUP_SQL.replace("{length_condition}", f"NOT {_LENGTH_CHANGING}"), ), diff --git a/src/mavedb/worker/jobs/variant_processing/reverse_translation.py b/src/mavedb/worker/jobs/variant_processing/reverse_translation.py index e3988920a..46d18d28e 100644 --- a/src/mavedb/worker/jobs/variant_processing/reverse_translation.py +++ b/src/mavedb/worker/jobs/variant_processing/reverse_translation.py @@ -12,8 +12,10 @@ links of a projection pair with a shared per-record ``projection_group`` id; the protein apex is shared across all projection pairs and carries no group. Where a projection pair member equals the record's authoritative (measured) allele, its group is folded onto the -existing authoritative link rather than duplicated, so the canonical c/g projection -is resolvable from the authoritative allele's group downstream. +authoritative link rather than duplicated, so the canonical c/g projection +is resolvable from the authoritative allele's group downstream. An authoritative link +whose group changes is superseded rather than updated, so ``as_of`` reads keep the +group that was live at the time. """ import asyncio @@ -25,8 +27,8 @@ from typing import Any, Callable, NamedTuple, Sequence from ga4gh.vrs.extras.translator import AlleleTranslator -from sqlalchemy import select -from sqlalchemy.orm import Session +from sqlalchemy import or_, select +from sqlalchemy.orm import Session, joinedload from variant_annotation import __version__ as variant_annotation_version from variant_annotation.lib.accessions import looks_like_refseq_protein_accession from variant_annotation.lib.translation import construct_equivalent_variants @@ -43,7 +45,7 @@ from mavedb.lib.variant_translations import get_or_create_allele from mavedb.lib.vrs_utils import translate_hgvs_to_variation, verify_allele_refget from mavedb.models.allele import Allele as AlleleDbModel -from mavedb.models.enums.sequence_level import SequenceLevel +from mavedb.models.enums.sequence_level import NUCLEOTIDE_LEVELS, SequenceLevel from mavedb.models.enums.annotation_type import AnnotationType from mavedb.models.enums.disposition import Disposition from mavedb.models.enums.event_reason import EventReason @@ -278,7 +280,15 @@ async def reverse_translate_variants_for_score_set( .all() ) - annotation_counts: Counter[str] = Counter({"translated": 0, "failed": 0, "skipped": 0, "alleles_created": 0}) + annotation_counts: Counter[str] = Counter( + { + "translated": 0, + "failed": 0, + "skipped": 0, + "alleles_created": 0, + "fold_in_missed": 0, + } + ) if not rows: logger.warning( @@ -410,14 +420,11 @@ async def reverse_translate_variants_for_score_set( # superseded atomically — retire and insert share a timestamp with no gap. new_links: list[MappingRecordAllele] = [] - # The live authoritative links for the current records, keyed by (record_id, allele_id). - # Held as ORM objects (not a bare pair set) because the authoritative fold-in updates them in - # place: when an RT candidate's allele equals a record's authoritative (measured) allele, that - # link already exists (is_authoritative=True, projection_group NULL, written by the mapping job) - # and uq_mapping_record_alleles_live forbids a derived duplicate — so the candidate's group is - # stamped onto this existing link instead of inserting a new one. Modifying the loaded object - # emits an UPDATE on flush; supersede_live_where at the end only retires derived links, leaving - # the authoritative link (and its freshly stamped group) intact. + # The live authoritative links for the current records, keyed by (record_id, allele_id). When an + # RT candidate's allele equals a record's authoritative (measured) allele, that link already exists + # and uq_mapping_record_alleles_live forbids a derived duplicate, so the candidate's group is + # recorded in folded_groups instead. After the loop, each link whose group changed is superseded by + # a copy carrying the new one. authoritative_links: dict[tuple[int, int], MappingRecordAllele] = { (link.mapping_record_id, link.allele_id): link for link in job_manager.db.scalars( @@ -425,12 +432,20 @@ async def reverse_translate_variants_for_score_set( .where(MappingRecordAllele.is_authoritative.is_(True)) .where(MappingRecordAllele.current) .where(MappingRecordAllele.mapping_record_id.in_(current_record_ids)) + .options(joinedload(MappingRecordAllele.allele)) ).all() } + # record_id -> the projection group this run folded onto its authoritative allele, and the group its + # authoritative link already holds. + folded_groups: dict[int, int | None] = {} + prior_groups = {record_id: link.projection_group for (record_id, _), link in authoritative_links.items()} + measured_alleles = {record_id: link.allele for (record_id, _), link in authoritative_links.items()} for result in results: rec, variant = variant_input_map[id(result.input)] candidate_count = 0 + record_links_start = len(new_links) + linked_nucleotide = False # Equivalence generation may surface the same VRS object more than once. # Dedup by vrs_digest per mapping record to avoid duplicate links. @@ -440,10 +455,10 @@ async def reverse_translate_variants_for_score_set( # Flatten the projection pairs into a work list of (hgvs, level, field, projection_group) members, # preserving the coding↔genomic pairing the library emits. Each ProjectionPair's coding and # genomic members share the pair's per-record group id — its index in the equivalence class - # (0..N-1). hgvs_c is always present (the pair key); hgvs_g is None when the c→g projection - # failed, yielding a well-formed one-member (coding only) group rather than a desync. The - # group id is assigned once per pair here, so the two members are guaranteed to carry the - # same id even though they translate and link independently below. + # (0..N-1), until the fold-in renumbering below. hgvs_c is always present (the pair key); hgvs_g + # is None when the c→g projection failed, yielding a well-formed one-member (coding only) group + # rather than a desync. The group id is assigned once per pair here, so the two members are + # guaranteed to carry the same id even though they translate and link independently below. members: list[tuple[str, SequenceLevel, str, int | None]] = [] for group_id, pair in enumerate(result.projection_pairs): members.append((pair.hgvs_c, SequenceLevel.cdna, "hgvs_c", group_id)) @@ -492,12 +507,10 @@ async def reverse_translate_variants_for_score_set( allele = get_or_create_allele(job_manager.db, draft_allele) job_manager.db.flush() - authoritative_link = authoritative_links.get((rec.id, allele.id)) - if authoritative_link is not None: - # Authoritative fold-in: this member is the record's measured allele, already - # linked authoritatively. Stamp it's projection group to preserve the c↔g pairing, - # and skip creating a new derived link. - authoritative_link.projection_group = projection_group + # Authoritative fold-in: this member is the record's measured allele, already linked + # authoritatively. Keep its group to preserve the c↔g pairing; no derived link. + if (rec.id, allele.id) in authoritative_links: + folded_groups[rec.id] = projection_group continue new_links.append( @@ -510,18 +523,48 @@ async def reverse_translate_variants_for_score_set( ) ) candidate_count += 1 + linked_nucleotide |= level in NUCLEOTIDE_LEVELS + + # Group ids only pair links within this record, so when the measured allele folds into a different + # position than the group its link already holds, swap the two ids rather than supersede the link. + folded, prior = folded_groups.get(rec.id), prior_groups.get(rec.id) + renumber: dict[int | None, int | None] = ( + {folded: prior, prior: folded} if folded is not None and prior is not None and folded != prior else {} + ) + for link in new_links[record_links_start:]: + link.projection_group = renumber.get(link.projection_group, link.projection_group) + if renumber: + folded_groups[rec.id] = prior + + # A nucleotide measured allele that RT linked nucleotide siblings to but never folded in has no projection + # partner, so serving labels every sibling convergent. Expected for an indel RT's codon-level candidates + # can't express; otherwise the mapper and RT produced different VRS objects for one change. + measured_allele = measured_alleles.get(rec.id) + fold_in_missed = ( + linked_nucleotide + and rec.id not in folded_groups + and measured_allele is not None + and measured_allele.level in NUCLEOTIDE_LEVELS + ) + annotation_counts["fold_in_missed"] += fold_in_missed annotation_counts["alleles_created"] += candidate_count annotation_metadata = { "hgvs_input": result.input.hgvs, - # Serializable projection of the projection pairs (the pairing the links now encode). + # Serializable projection of the projection pairs and the group each one's links carry. "candidates": [ - {"hgvs_c": pair.hgvs_c, "hgvs_g": pair.hgvs_g, "variant_type": pair.variant_type} - for pair in result.projection_pairs + { + "hgvs_c": pair.hgvs_c, + "hgvs_g": pair.hgvs_g, + "variant_type": pair.variant_type, + "projection_group": renumber.get(position, position), + } + for position, pair in enumerate(result.projection_pairs) ], "hgvs_p": result.hgvs_p, "alleles_created": candidate_count, "failed_candidates": failed_candidates, + "fold_in_missed": fold_in_missed, } # No translatable candidates and failures mean the variant failed reverse translation. No @@ -550,19 +593,42 @@ async def reverse_translate_variants_for_score_set( # its inputs are unchanged since the run that produced them, so they are still that run's answer. # Every other outcome, including a skip or a failed translation, is this run's answer and replaces # the prior set. - unavailable_record_ids = [ + unavailable_record_ids = { variant_input_map[id(error.input)][0].id for error in errors if error.reason is TranslationErrorReason.UPSTREAM_UNAVAILABLE - ] + } + + # An authoritative link takes this run's fold-in group, or NULL when nothing folded in. Group ids + # are positions in this run's pair list, so a group left from a prior run would name an unrelated + # pair. Records whose links are kept (unavailable) keep their group too. + regrouped_link_ids: list[int] = [] + for (record_id, _), link in authoritative_links.items(): + group = folded_groups.get(record_id) + if record_id in unavailable_record_ids or link.projection_group == group: + continue - # Supersede prior live derived links atomically. + regrouped_link_ids.append(link.id) + new_links.append( + MappingRecordAllele( + mapping_record_id=link.mapping_record_id, + score_set_id=link.score_set_id, + allele_id=link.allele_id, + is_authoritative=True, + projection_group=group, + ) + ) + + # Supersede prior live derived links, and the regrouped authoritative links, under one timestamp. # TODO#765: re-runs retire and recreate the whole derived set because re-mapping re-mints # records; idempotent records would allow unchanged links to stay live. MappingRecordAllele.supersede_live_where( job_manager.db, new_links, - MappingRecordAllele.is_authoritative.is_(False), + or_( + MappingRecordAllele.is_authoritative.is_(False), + MappingRecordAllele.id.in_(regrouped_link_ids), + ), MappingRecordAllele.mapping_record_id.in_(current_record_ids), MappingRecordAllele.mapping_record_id.not_in(unavailable_record_ids), ) @@ -640,6 +706,15 @@ async def reverse_translate_variants_for_score_set( ) job_manager.db.flush() + if annotation_counts["fold_in_missed"]: + logger.warning( + msg=( + f"{annotation_counts['fold_in_missed']} measured alleles matched none of their reverse translation " + "candidates; their events carry fold_in_missed in metadata." + ), + extra=job_manager.logging_context(), + ) + if annotation_counts["translated"] == 0 and annotation_counts["failed"] > 0: logger.error( msg="All variant reverse translations failed.", diff --git a/tests/worker/jobs/variant_processing/test_reverse_translation.py b/tests/worker/jobs/variant_processing/test_reverse_translation.py index 152133ff0..22f904f63 100644 --- a/tests/worker/jobs/variant_processing/test_reverse_translation.py +++ b/tests/worker/jobs/variant_processing/test_reverse_translation.py @@ -307,7 +307,7 @@ async def test_no_mapping_records_is_a_noop( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - assert result.data == {"translated": 0, "failed": 0, "skipped": 0, "alleles_created": 0} + assert result.data == {"translated": 0, "failed": 0, "skipped": 0, "alleles_created": 0, "fold_in_missed": 0} # No candidate alleles, links, or annotations were produced. assert session.query(Allele).count() == 0 @@ -412,7 +412,7 @@ def _fail_if_engine_sees_it(inputs, *, transcripts, coordinates, config): result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) assert result.status == JobStatus.SUCCEEDED - assert result.data == {"translated": 0, "failed": 0, "skipped": 1, "alleles_created": 0} + assert result.data == {"translated": 0, "failed": 0, "skipped": 1, "alleles_created": 0, "fold_in_missed": 0} assert _non_authoritative_links(session) == [] events = _cross_level_events(session, sample_score_set.id, reason="cis_phased_unsupported") @@ -464,7 +464,7 @@ async def test_falls_back_to_mapped_date_when_the_record_has_no_job_run_id( # The transcript resolved via the fallback, so the variant translated instead of skipping. assert result.status == JobStatus.SUCCEEDED - assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 2} + assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 2, "fold_in_missed": 1} async def test_creates_genomic_and_coding_candidate_alleles( self, @@ -503,7 +503,7 @@ async def test_creates_genomic_and_coding_candidate_alleles( result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) assert result.status == JobStatus.SUCCEEDED - assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 2} + assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 2, "fold_in_missed": 1} # Two non-authoritative links were created, both for our one mapping record. non_auth_links = _non_authoritative_links(session) @@ -535,7 +535,7 @@ async def test_creates_genomic_and_coding_candidate_alleles( assert events[0].disposition == "present" # The event metadata carries the paired projection pair shape, not the removed flat lists. assert events[0].event_metadata["candidates"] == [ - {"hgvs_c": c_candidate, "hgvs_g": g_candidate, "variant_type": None} + {"hgvs_c": c_candidate, "hgvs_g": g_candidate, "variant_type": None, "projection_group": 0} ] async def test_transcript_is_queryable_via_derived_expression( @@ -677,7 +677,7 @@ async def test_cis_phased_multivariant_candidate_is_persisted_as_a_block( result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) assert result.status == JobStatus.SUCCEEDED - assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 2} + assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 2, "fold_in_missed": 1} block_allele = session.query(Allele).filter(Allele.vrs_digest == "ga4gh:CPB.block").one() assert block_allele.level == SequenceLevel.genomic.value @@ -727,7 +727,7 @@ async def test_duplicate_candidate_digests_are_deduped_per_record( result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) assert result.status == JobStatus.SUCCEEDED - assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 1} + assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 1, "fold_in_missed": 1} non_auth_links = _non_authoritative_links(session) assert len(non_auth_links) == 1 assert non_auth_links[0].projection_group == 0 @@ -779,7 +779,7 @@ async def test_authoritative_allele_is_folded_into_its_projection_group( assert result.status == JobStatus.SUCCEEDED # Only the genomic sibling is a newly created derived allele; the coding member folds into # the existing authoritative link rather than creating a second link/allele. - assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 1} + assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 1, "fold_in_missed": 0} record = _record_for(session, variant.id) auth_link = _authoritative_link(session, record.id) @@ -798,6 +798,7 @@ async def test_authoritative_allele_is_folded_into_its_projection_group( assert {link.is_authoritative for link in group_links} == {True, False} assert auth_link.allele.vrs_digest == CANONICAL_AUTHORITATIVE_DIGEST assert derived_links[0].allele.vrs_digest == "ga4gh:VA.genomic" + assert _cross_level_events(session, sample_score_set.id)[0].event_metadata["fold_in_missed"] is False async def test_authoritative_fold_in_when_measured_at_genomic_level( self, @@ -840,7 +841,7 @@ async def test_authoritative_fold_in_when_measured_at_genomic_level( result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) assert result.status == JobStatus.SUCCEEDED - assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 1} + assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 1, "fold_in_missed": 0} record = _record_for(session, variant.id) auth_link = _authoritative_link(session, record.id) @@ -850,6 +851,47 @@ async def test_authoritative_fold_in_when_measured_at_genomic_level( assert auth_link.allele.vrs_digest == CANONICAL_AUTHORITATIVE_DIGEST assert derived_links[0].allele.vrs_digest == "ga4gh:VA.coding" + async def test_measured_allele_matching_no_candidate_is_flagged_as_a_missed_fold_in( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """Nucleotide assay whose measured allele has a different digest from every RT candidate: the + authoritative link stays ungrouped, and the run counts the miss and flags it on the variant's event.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn=f"{sample_score_set.urn}#11", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + assay_hgvs = "NM_000000.1:c.1A>G" + c_candidate = "NM_000001.1:c.5A>G" + g_sibling = "NC_000001.11:g.1000A>G" + construct = fake_construct({assay_hgvs: [(c_candidate, g_sibling)]}) + # The coding member is the measured change, but it identifies differently from the stored allele. + translate = fake_translate({c_candidate: "ga4gh:VA.divergent", g_sibling: "ga4gh:VA.genomic"}) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 2, "fold_in_missed": 1} + assert _authoritative_link(session, _record_for(session, variant.id).id).projection_group is None + assert _cross_level_events(session, sample_score_set.id)[0].event_metadata["fold_in_missed"] is True + async def test_projection_failed_candidate_is_a_one_member_group( self, session, @@ -894,7 +936,7 @@ async def test_projection_failed_candidate_is_a_one_member_group( result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) assert result.status == JobStatus.SUCCEEDED - assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 3} + assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 3, "fold_in_missed": 1} non_auth_links = _non_authoritative_links(session) groups: dict[int, set[str]] = {} @@ -954,7 +996,7 @@ async def test_protein_assay_leaves_authoritative_group_null_over_a_fan_out( result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) assert result.status == JobStatus.SUCCEEDED - assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 6} + assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 6, "fold_in_missed": 0} record = _record_for(session, variant.id) # The protein authoritative allele is not part of any pair: its group stays NULL. @@ -1059,6 +1101,112 @@ async def test_independent_rerun_retires_prior_links_and_regroups( assert len(events) == 2 assert max(events, key=lambda e: e.id).disposition == "present" + @pytest.mark.parametrize( + "rerun_pairs, rerun_unavailable, expected_group, regrouped, derived_groups", + [ + # The measured allele's pair moves to a new position: the ids are swapped so its link keeps group 0. + ( + [("NM_000001.1:c.9C>T", "NC_000001.11:g.1004C>T"), ("NM_000001.1:c.5A>G", None)], + False, + 0, + False, + {1}, + ), + # The measured allele no longer folds in: its prior group would name an unrelated pair. + ([("NM_000001.1:c.9C>T", "NC_000001.11:g.1004C>T")], False, None, True, {0}), + # Same fold-in as before: the link is left alone rather than re-versioned. + ([("NM_000001.1:c.5A>G", None)], False, 0, False, set()), + # UTA unavailable: the record's prior links, its group included, are still this run's answer. + ([], True, 0, False, set()), + ], + ) + async def test_rerun_supersedes_the_authoritative_link_when_its_group_changes( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + rerun_pairs, + rerun_unavailable, + expected_group, + regrouped, + derived_groups, + ): + """Group ids only pair links within one run, so a rerun that skips re-mapping must not leave the measured + allele holding a group that now names another pair. A run renumbers to keep the link's group where the + measured allele still folds in, and otherwise supersedes the link under the run's timestamp, keeping the + old group readable through ``as_of``.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn=f"{sample_score_set.urn}#11", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + assay_hgvs = "NM_000000.1:c.1A>G" + measured_candidate = "NM_000001.1:c.5A>G" + translate = fake_translate({measured_candidate: CANONICAL_AUTHORITATIVE_DIGEST}) + with ( + patch( + f"{RT_MODULE}.construct_equivalent_variants", fake_construct({assay_hgvs: [(measured_candidate, None)]}) + ), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + first = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + assert first.status == JobStatus.SUCCEEDED + + record = _record_for(session, variant.id) + assert _authoritative_link(session, record.id).projection_group == 0 + + rerun = JobRun( + urn="test:reverse_translate_variants_for_score_set:rerun", + job_type="reverse_translate_variants_for_score_set", + job_function="reverse_translate_variants_for_score_set", + max_retries=3, + retry_count=0, + job_params=dict(sample_independent_reverse_translation_run.job_params), + ) + session.add(rerun) + session.commit() + + second_construct = ( + fake_construct({}, upstream_unavailable_by_hgvs={assay_hgvs: "could not connect to server"}) + if rerun_unavailable + else fake_construct({assay_hgvs: rerun_pairs}) + ) + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", second_construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + await _reverse_translate(session, mock_worker_ctx, rerun) + + versions = ( + session.query(MappingRecordAllele) + .filter( + MappingRecordAllele.mapping_record_id == record.id, + MappingRecordAllele.is_authoritative.is_(True), + ) + .order_by(MappingRecordAllele.id) + .all() + ) + live = _authoritative_link(session, record.id) + assert live.projection_group == expected_group + # The mapping job's link (group NULL) plus the first run's version (group 0), and one more if regrouped. + assert len(versions) == (3 if regrouped else 2) + prior = versions[-2] + assert prior.projection_group == (0 if regrouped else None) + assert prior.valid_to == live.valid_from + assert live.allele_id == prior.allele_id + live_derived = [link for link in _non_authoritative_links(session) if link.valid_to is None] + assert {link.projection_group for link in live_derived} == derived_groups + @pytest.mark.parametrize( "rerun_outcome, keeps_prior_link", [ @@ -1178,7 +1326,7 @@ async def test_all_failures_fail_the_job( result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) assert result.status == JobStatus.FAILED - assert result.data == {"translated": 0, "failed": 1, "skipped": 0, "alleles_created": 0} + assert result.data == {"translated": 0, "failed": 1, "skipped": 0, "alleles_created": 0, "fold_in_missed": 0} assert _non_authoritative_links(session) == [] failed_events = _cross_level_events(session, sample_score_set.id, disposition="failed") @@ -1231,7 +1379,7 @@ async def test_partial_success_succeeds_with_mixed_annotations( result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) assert result.status == JobStatus.SUCCEEDED - assert result.data == {"translated": 1, "failed": 1, "skipped": 0, "alleles_created": 1} + assert result.data == {"translated": 1, "failed": 1, "skipped": 0, "alleles_created": 1, "fold_in_missed": 1} assert len(_cross_level_events(session, sample_score_set.id, disposition="present")) == 1 assert len(_cross_level_events(session, sample_score_set.id, disposition="failed")) == 1 @@ -1278,7 +1426,7 @@ async def test_non_translatable_edit_is_skipped_not_failed( # A whole score set of non-translatable variants is a benign no-op, not a job failure. assert result.status == JobStatus.SUCCEEDED - assert result.data == {"translated": 0, "failed": 0, "skipped": 1, "alleles_created": 0} + assert result.data == {"translated": 0, "failed": 0, "skipped": 1, "alleles_created": 0, "fold_in_missed": 0} # No alleles/links, and no failed event at all. assert _non_authoritative_links(session) == [] @@ -1331,7 +1479,7 @@ async def test_non_translatable_skip_coexists_with_success_and_failure( result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) assert result.status == JobStatus.SUCCEEDED - assert result.data == {"translated": 1, "failed": 1, "skipped": 1, "alleles_created": 1} + assert result.data == {"translated": 1, "failed": 1, "skipped": 1, "alleles_created": 1, "fold_in_missed": 1} # The genuine error is FAILED with reason translation_error — distinct from the benign skip. failed = _cross_level_events(session, sample_score_set.id, disposition="failed") @@ -1380,7 +1528,7 @@ async def test_upstream_outage_is_failed_as_api_error_not_translation_error( result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) assert result.status == JobStatus.SUCCEEDED - assert result.data == {"translated": 1, "failed": 1, "skipped": 0, "alleles_created": 1} + assert result.data == {"translated": 1, "failed": 1, "skipped": 0, "alleles_created": 1, "fold_in_missed": 1} failed = _cross_level_events(session, sample_score_set.id, disposition="failed") assert len(failed) == 1 @@ -1427,7 +1575,7 @@ async def test_partial_candidate_translation_failure_keeps_success_with_metadata result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) assert result.status == JobStatus.SUCCEEDED - assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 1} + assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 1, "fold_in_missed": 1} assert len(_non_authoritative_links(session)) == 1 events = _cross_level_events(session, sample_score_set.id, disposition="present") @@ -1470,7 +1618,7 @@ async def test_all_candidates_failing_translation_marks_variant_failed( result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) assert result.status == JobStatus.FAILED - assert result.data == {"translated": 0, "failed": 1, "skipped": 0, "alleles_created": 0} + assert result.data == {"translated": 0, "failed": 1, "skipped": 0, "alleles_created": 0, "fold_in_missed": 0} assert _non_authoritative_links(session) == [] failed_events = _cross_level_events(session, sample_score_set.id, disposition="failed") @@ -1517,7 +1665,7 @@ async def test_unresolved_transcript_is_tallied_as_skip_but_recorded_as_failed( result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) assert result.status == JobStatus.SUCCEEDED - assert result.data == {"translated": 0, "failed": 0, "skipped": 1, "alleles_created": 0} + assert result.data == {"translated": 0, "failed": 0, "skipped": 1, "alleles_created": 0, "fold_in_missed": 0} # No translation attempted: no candidate alleles or links, and the variant is # recorded as SKIPPED rather than FAILED. @@ -1575,7 +1723,7 @@ async def test_genomic_accession_coding_target_is_reverse_translated( result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) assert result.status == JobStatus.SUCCEEDED - assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 2} + assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 2, "fold_in_missed": 1} events = _cross_level_events(session, sample_score_set.id) assert events and all(e.disposition == "present" for e in events) @@ -1702,7 +1850,7 @@ async def test_ignores_stale_cdna_row_from_a_different_run( ): result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) - assert result.data == {"translated": 0, "failed": 0, "skipped": 1, "alleles_created": 0} + assert result.data == {"translated": 0, "failed": 0, "skipped": 1, "alleles_created": 0, "fold_in_missed": 0} skipped = _cross_level_events(session, sample_score_set.id, reason="transcript_unresolved") assert len(skipped) == 1 assert skipped[0].disposition == "failed" @@ -1740,7 +1888,7 @@ async def test_coding_target_skip_is_classified_recoverable( ): result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) - assert result.data == {"translated": 0, "failed": 0, "skipped": 1, "alleles_created": 0} + assert result.data == {"translated": 0, "failed": 0, "skipped": 1, "alleles_created": 0, "fold_in_missed": 0} skipped = _cross_level_events(session, sample_score_set.id, reason="transcript_unresolved") assert len(skipped) == 1 assert skipped[0].disposition == "failed" @@ -1780,7 +1928,7 @@ async def test_regulatory_target_skip_is_classified_correct( ): result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) - assert result.data == {"translated": 0, "failed": 0, "skipped": 1, "alleles_created": 0} + assert result.data == {"translated": 0, "failed": 0, "skipped": 1, "alleles_created": 0, "fold_in_missed": 0} skipped = _cross_level_events(session, sample_score_set.id, reason="no_coding_transcript") assert len(skipped) == 1 assert skipped[0].disposition == "absent" @@ -1930,6 +2078,6 @@ def capturing_construct(inputs, *, transcripts, coordinates, config): result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) assert result.status == JobStatus.SUCCEEDED - assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 1} + assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 1, "fold_in_missed": 0} # The NP_ accession was resolved to its NM_ transcript and passed as the hint. assert captured_hints == {assay_hgvs: "NM_000111.1"} From b425481accc107160ff7495709268c660d8a7d86 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Fri, 9 Oct 2026 06:17:06 -0700 Subject: [PATCH 231/252] feat(logging): count database statements per request observe_statements (formerly log_slow_statements) also increments db_statement_count in the request logging context, so a route's query count shows up in the request log. CSV tests count statements through the shared captured_statements helper. --- src/mavedb/db/timeouts.py | 12 ++++++++---- src/mavedb/lib/logging/context.py | 6 ++++++ src/mavedb/server_main.py | 4 ++-- tests/db/test_timeouts.py | 26 +++++++++++++++++++++----- tests/lib/csv/test_fetch.py | 19 +++++-------------- tests/lib/csv/test_variant.py | 26 +++++--------------------- 6 files changed, 47 insertions(+), 46 deletions(-) diff --git a/src/mavedb/db/timeouts.py b/src/mavedb/db/timeouts.py index 540a2b812..5ecc1011f 100644 --- a/src/mavedb/db/timeouts.py +++ b/src/mavedb/db/timeouts.py @@ -13,7 +13,7 @@ from sqlalchemy.engine import Engine from sqlalchemy.orm import Session -from mavedb.lib.logging.context import logging_context +from mavedb.lib.logging.context import increment_logging_context, logging_context logger = logging.getLogger(__name__) @@ -67,10 +67,13 @@ def allow_long_statements(db: Session, seconds: int = CSV_STATEMENT_TIMEOUT_SECO db.execute(text("SELECT set_config('statement_timeout', :ms, true)"), {"ms": str(seconds * 1000)}) -def log_slow_statements(engine: Engine, threshold_seconds: float) -> None: - """Log any statement on ``engine`` that runs longer than ``threshold_seconds``, including ones that error. +def observe_statements(engine: Engine, threshold_seconds: float) -> None: + """Count each request's statements on ``engine``, and log any statement slower than ``threshold_seconds``. - A statement cancelled by ``statement_timeout`` is logged too, since those are the ones worth tuning. + The count lands in the request's canonical log as ``db_statement_count``, so a route whose query count + grows with its data shows up on real data, where test fixtures are too small to show it. + + A slow statement cancelled by ``statement_timeout`` is logged too, since those are the ones worth tuning. Logs the duration and the start of the SQL text, never the bound parameters, which can hold user data. """ @@ -87,6 +90,7 @@ def _log(statement: str, started_at: float) -> None: @event.listens_for(engine, "before_cursor_execute") def _start_timer(_conn, _cursor, _statement, _parameters, context, _executemany): + increment_logging_context("db_statement_count") setattr(context, _STARTED_AT_ATTR, time.perf_counter()) @event.listens_for(engine, "after_cursor_execute") diff --git a/src/mavedb/lib/logging/context.py b/src/mavedb/lib/logging/context.py index ad3e84413..8e05e7706 100644 --- a/src/mavedb/lib/logging/context.py +++ b/src/mavedb/lib/logging/context.py @@ -63,6 +63,12 @@ def save_to_logging_context(ctx: dict) -> dict: return context.data +def increment_logging_context(key: str, by: int = 1) -> None: + """Add ``by`` to a counter in the request's logging context. A no-op outside a request (worker, scripts).""" + if context.exists(): + context[key] = context.get(key, 0) + by + + def logging_context() -> dict: if not context.exists(): logger.debug("Could not access logging context. Context does not exist.") diff --git a/src/mavedb/server_main.py b/src/mavedb/server_main.py index 26a93c11e..1270a47ac 100644 --- a/src/mavedb/server_main.py +++ b/src/mavedb/server_main.py @@ -21,7 +21,7 @@ from mavedb import __version__ from mavedb.db.session import engine -from mavedb.db.timeouts import SLOW_STATEMENT_SECONDS, apply_api_timeouts, log_slow_statements +from mavedb.db.timeouts import SLOW_STATEMENT_SECONDS, apply_api_timeouts, observe_statements from mavedb.lib.exceptions import ( AmbiguousIdentifierError, HGNCGeneNotFoundError, @@ -83,7 +83,7 @@ configure_mappers() apply_api_timeouts(engine) -log_slow_statements(engine, SLOW_STATEMENT_SECONDS) +observe_statements(engine, SLOW_STATEMENT_SECONDS) # forward_retired_urns is applied to every route, so that one implementation forwards a read of a URN # publication has retired wherever that URN points: a record, or any of its sub-resources. diff --git a/tests/db/test_timeouts.py b/tests/db/test_timeouts.py index fb82e5663..8b5e5d911 100644 --- a/tests/db/test_timeouts.py +++ b/tests/db/test_timeouts.py @@ -10,13 +10,14 @@ from sqlalchemy.exc import OperationalError from sqlalchemy.orm import Session from sqlalchemy.pool import NullPool +from starlette_context import context, request_cycle_context from mavedb.db import timeouts from mavedb.db.timeouts import ( CSV_STATEMENT_TIMEOUT_SECONDS, allow_long_statements, apply_api_timeouts, - log_slow_statements, + observe_statements, ) @@ -75,9 +76,9 @@ def test_other_engines_are_unbounded(self, engine): @pytest.mark.unit -class TestLogSlowStatements: +class TestObserveStatements: def test_logs_a_slow_statement_without_its_parameters(self, session, engine, caplog): - log_slow_statements(engine, threshold_seconds=0.05) + observe_statements(engine, threshold_seconds=0.05) with caplog.at_level(logging.WARNING, logger="mavedb.db.timeouts"): session.execute(text("SELECT pg_sleep(0.1), :secret"), {"secret": "hunter2"}) @@ -87,15 +88,30 @@ def test_logs_a_slow_statement_without_its_parameters(self, session, engine, cap assert not any("hunter2" in message for message in messages) def test_does_not_log_a_fast_statement(self, session, engine, caplog): - log_slow_statements(engine, threshold_seconds=5) + observe_statements(engine, threshold_seconds=5) with caplog.at_level(logging.WARNING, logger="mavedb.db.timeouts"): session.execute(text("SELECT 1")) assert not caplog.records + def test_counts_each_statement_in_the_request_context(self, session, engine): + observe_statements(engine, threshold_seconds=5) + + with request_cycle_context({}): + session.execute(text("SELECT 1")) + session.execute(text("SELECT 2")) + assert context["db_statement_count"] == 2 + + def test_counting_outside_a_request_is_a_no_op(self, session, engine): + observe_statements(engine, threshold_seconds=5) + + session.execute(text("SELECT 1")) + + assert not context.exists() + def test_logs_a_statement_cancelled_by_the_timeout(self, session, engine, caplog): - log_slow_statements(engine, threshold_seconds=0.05) + observe_statements(engine, threshold_seconds=0.05) session.execute(text("SET LOCAL statement_timeout = '100ms'")) with caplog.at_level(logging.WARNING, logger="mavedb.db.timeouts"): diff --git a/tests/lib/csv/test_fetch.py b/tests/lib/csv/test_fetch.py index 84b8a1a84..b7febd43f 100644 --- a/tests/lib/csv/test_fetch.py +++ b/tests/lib/csv/test_fetch.py @@ -6,26 +6,19 @@ pytest.importorskip("psycopg2") -from sqlalchemy import event, text +from sqlalchemy import text from mavedb.lib.csv.columns import plan_csv_columns from mavedb.lib.csv.fetch import fetch_variant_csv_data +from tests.helpers.util.query_plan import captured_statements VARIANT_NUMBER_INDEX = "ix_variants_scoreset_number" def _captured_variant_query(session, namespaces, start=None, limit=None): """Run the score set CSV fetch and return the statement and parameters it sent for the variant rows.""" - captured = [] - engine = session.get_bind() - - def capture(_conn, _cursor, statement, parameters, _context, _executemany): - if "FROM variants" in statement and not captured: - captured.append((statement, parameters)) - - event.listen(engine, "before_cursor_execute", capture) - try: - plan = plan_csv_columns({"score_columns": ["score"]}, namespaces) + plan = plan_csv_columns({"score_columns": ["score"]}, namespaces) + with captured_statements(session) as captured: fetch_variant_csv_data( session, plan.namespaced_columns, @@ -34,10 +27,8 @@ def capture(_conn, _cursor, statement, parameters, _context, _executemany): start=start, limit=limit, ) - finally: - event.remove(engine, "before_cursor_execute", capture) - return captured[0] + return next((statement, parameters) for statement, parameters in captured if "FROM variants" in statement) @pytest.mark.unit diff --git a/tests/lib/csv/test_variant.py b/tests/lib/csv/test_variant.py index a2e30c313..71ecbc31a 100644 --- a/tests/lib/csv/test_variant.py +++ b/tests/lib/csv/test_variant.py @@ -5,6 +5,7 @@ from unittest.mock import Mock from tests.helpers.util.annotation import AlleleSpec, seed_mapping_record +from tests.helpers.util.query_plan import captured_statements from sqlalchemy import select from sqlalchemy.exc import IntegrityError from unittest.mock import patch @@ -13,7 +14,6 @@ pytest.importorskip("psycopg2") -from sqlalchemy import event from mavedb.lib.csv.namespaces import calibration_namespace_for_urn, is_valid_csv_namespace from mavedb.lib.csv.score_set import available_score_set_csv_namespaces, get_score_set_variants_as_csv @@ -776,17 +776,9 @@ def test_does_not_load_whole_score_range_variant_collections(self, session, setu _add_pathogenicity_calibration(session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="First") _add_pathogenicity_calibration(session, variant.score_set, [], urn=CALIBRATION_URN_2, title="Second") - statements: list[str] = [] - - def record(conn, cursor, statement, parameters, context, executemany): - statements.append(statement) - - bind = session.get_bind() - event.listen(bind, "before_cursor_execute", record) - try: + with captured_statements(session) as captured: get_variant_csv(session, variant.urn) - finally: - event.remove(bind, "before_cursor_execute", record) + statements = [statement for statement, _ in captured] collection_loads = [ statement @@ -808,17 +800,9 @@ def test_namespace_discovery_does_not_scan_score_set_variants(self, session, set variant = setup_lib_db_with_mapped_variant.variant _add_pathogenicity_calibration(session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="First") - statements: list[str] = [] - - def record(conn, cursor, statement, parameters, context, executemany): - statements.append(statement) - - bind = session.get_bind() - event.listen(bind, "before_cursor_execute", record) - try: + with captured_statements(session) as captured: available_variant_csv_namespaces(session, variant.urn) - finally: - event.remove(bind, "before_cursor_execute", record) + statements = [statement for statement, _ in captured] calibration_scans = [ statement From bbbad589561c9031ce25b88945bde2562d8c4400 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Fri, 9 Oct 2026 06:17:19 -0700 Subject: [PATCH 232/252] fix(variants): batch variant detail and annotation reads, label members with one rule The variant-details stream, annotation streams, and public exports loaded each variant's records, links, cross-references, and classifications separately. They now load per chunk of 500 variants through with_chunk_inputs, a fixed number of queries per chunk, and the single variant routes are chunks of one. Cat-VRS, variant detail, and allele detail each labelled members with their own rule and disagreed. cat_vrs.member_label is now the one labeller. A nucleotide member outside the measured allele's projection group reads as co_encodes/convergent rather than a coordinate representation, so a fold-in miss no longer serves convergent siblings as exact matches. test_query_scaling walks every parameterized GET route and fails when a route runs more queries on a larger score set than a smaller one. --- src/mavedb/lib/allele_detail.py | 49 +---- src/mavedb/lib/allele_identity.py | 4 +- src/mavedb/lib/allele_measurements.py | 45 ++-- src/mavedb/lib/alleles.py | 97 ++++++--- src/mavedb/lib/annotation/context.py | 126 +++++++---- src/mavedb/lib/cat_vrs.py | 115 ++++------ src/mavedb/lib/score_calibrations.py | 23 ++ src/mavedb/lib/variant_detail.py | 204 ++++++++++-------- src/mavedb/models/mapping_record.py | 2 +- src/mavedb/routers/score_sets.py | 48 +++-- src/mavedb/scripts/export_public_data.py | 25 ++- src/mavedb/scripts/export_sweep.py | 13 +- tests/helpers/util/annotation.py | 29 +-- tests/helpers/util/variant.py | 2 +- tests/lib/test_alleles.py | 46 +++- tests/lib/test_cat_vrs.py | 107 +++++++--- tests/lib/test_variant_detail.py | 97 ++++++++- tests/routers/test_query_scaling.py | 256 +++++++++++++++++++++++ tests/routers/test_score_set.py | 26 ++- 19 files changed, 919 insertions(+), 395 deletions(-) create mode 100644 tests/routers/test_query_scaling.py diff --git a/src/mavedb/lib/allele_detail.py b/src/mavedb/lib/allele_detail.py index 4853f1e19..5aa70f193 100644 --- a/src/mavedb/lib/allele_detail.py +++ b/src/mavedb/lib/allele_detail.py @@ -33,12 +33,12 @@ from sqlalchemy.orm import Session from mavedb.lib.allele_annotations import AlleleAnnotations, get_allele_annotations -from mavedb.lib.allele_identity import AlleleDerivation, AlleleIdentity +from mavedb.lib.allele_identity import AlleleIdentity from mavedb.lib.alleles import get_allele_translations -from mavedb.lib.cat_vrs import CatVrsRelation +from mavedb.lib.cat_vrs import member_label from mavedb.lib.vrs import vrs_object_from_mapped_variant from mavedb.models.allele import Allele -from mavedb.models.enums.sequence_level import NUCLEOTIDE_LEVELS, SequenceLevel +from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.models.mapping_record_allele import MappingRecordAllele @@ -120,34 +120,6 @@ def _focus_projection_pairs( return pairs -def _member_label( - *, focus_is_protein: bool, member_level: Optional[str], is_projection: bool -) -> tuple[Optional[CatVrsRelation], Optional[AlleleDerivation]]: - """The (relation, derivation) for a non-focus member, read *relative to the focus*. - - - **protein focus** → every nt member is a reverse-translation ``candidate`` that ``encodes`` it. - - **nt focus** → The protein consequence is ``translation_of`` (a deterministic ``projection`` of the focus). - The focus's projection partner is ``coordinate_representation_of`` (also a deterministic ``projection``). - Any other nt is a convergent encoding: ``co_encodes`` and ``convergent`` (a distinct change which shares the - ultimate protein consequence of the focus). - """ - if focus_is_protein: - if member_level in NUCLEOTIDE_LEVELS: - return CatVrsRelation.ENCODES, AlleleDerivation.CANDIDATE - - return None, None - - if member_level == SequenceLevel.protein.value: - return CatVrsRelation.TRANSLATION_OF, AlleleDerivation.PROJECTION - if member_level in NUCLEOTIDE_LEVELS: - if is_projection: - return CatVrsRelation.COORDINATE_REPRESENTATION_OF, AlleleDerivation.PROJECTION - - return CatVrsRelation.CO_ENCODES, AlleleDerivation.CONVERGENT - - return None, None - - def get_allele_detail( db: Session, focus: Allele, @@ -162,14 +134,19 @@ def get_allele_detail( a by-digest fetch, or the (genomic + coding) representations of a CAID for a by-CAID fetch. The equivalence class is every allele co-linked to a live record touching the ``focus`` (:func:`get_allele_translations`). An orphan allele falls back to ``focus`` alone. Non-focus members - are labelled by :func:`_member_label`. ``score_set_ids``, when given, restricts the equivalence + are labelled by :func:`~mavedb.lib.cat_vrs.member_label`. ``score_set_ids``, when given, restricts the equivalence class and projection pairing to records of those score sets. """ equivalence_class = get_allele_translations(db, focus.id, as_of=as_of, score_set_ids=score_set_ids) or [focus] annotations = get_allele_annotations(db, equivalence_class, as_of=as_of) focus_alleles = [a for a in equivalence_class if a.vrs_digest in focus_digests] or [focus] - focus_is_protein = any(a.level == SequenceLevel.protein.value for a in focus_alleles) + # A PAID focuses the protein allele; a CAID or digest focuses nucleotide allele(s). + focus_level = ( + SequenceLevel.protein.value + if any(a.level == SequenceLevel.protein.value for a in focus_alleles) + else focus_alleles[0].level + ) pairs = _focus_projection_pairs(db, focus_alleles, as_of=as_of, score_set_ids=score_set_ids) # The focus's projection(s) that are not themselves focus (for a CAID fetch the projection pair is @@ -186,11 +163,7 @@ class and projection pairing to records of those score sets. # members have no relation or derivation relative to themselves. relation, derivation = None, None if digest not in focus_digests: - rel, der = _member_label( - focus_is_protein=focus_is_protein, - member_level=member.level, - is_projection=digest in projection_digests, - ) + rel, der = member_label(focus_level, member.level, is_projection=digest in projection_digests) relation = rel.value if rel is not None else None derivation = der diff --git a/src/mavedb/lib/allele_identity.py b/src/mavedb/lib/allele_identity.py index 1b065b223..6ae589cf9 100644 --- a/src/mavedb/lib/allele_identity.py +++ b/src/mavedb/lib/allele_identity.py @@ -36,8 +36,8 @@ Attempting to encode that on a molecular-relationship axis would misstate what the molecule does. Today's one-to-one mapping between the two vocabularies is **a fact about today's pipeline, not about -these axes. Never infer one from the other.** They are emitted together by -``variant_detail._derivation_for`` and ``allele_detail._member_label``, the single source of truth. +these axes. Never infer one from the other.** They are emitted together by ``cat_vrs.member_label``, the +single source of truth for every view. """ from dataclasses import dataclass diff --git a/src/mavedb/lib/allele_measurements.py b/src/mavedb/lib/allele_measurements.py index f56f79e09..df1b37b92 100644 --- a/src/mavedb/lib/allele_measurements.py +++ b/src/mavedb/lib/allele_measurements.py @@ -42,7 +42,11 @@ from mavedb.lib.alleles import readable_score_set_ids_by_allele from mavedb.lib.logging.context import logging_context, save_to_logging_context from mavedb.lib.permissions import Action, has_permission -from mavedb.lib.score_calibrations import calibration_preference_key, classification_evidence_strength +from mavedb.lib.score_calibrations import ( + calibration_preference_key, + classification_evidence_strength, + classifications_by_variant, +) from mavedb.lib.types.authentication import UserData from mavedb.lib.variants import variant_score from mavedb.models.allele import Allele @@ -53,9 +57,6 @@ from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification -from mavedb.models.score_calibration_functional_classification_variant_association import ( - score_calibration_functional_classification_variants_association_table as classification_variants, -) from mavedb.models.score_set import ScoreSet from mavedb.models.variant import Variant @@ -113,9 +114,13 @@ class _ProteinApex: def _preferred_classification( - db: Session, variant: Variant, *, user_data: Optional[UserData], research_use_only: bool = False + classifications: list[tuple[ScoreCalibrationFunctionalClassification, ScoreCalibration]], + *, + user_data: Optional[UserData], + research_use_only: bool = False, ) -> Optional[ScoreCalibrationFunctionalClassification]: - """The variant's preferred *readable* functional classification, or ``None``. + """The variant's preferred *readable* functional classification among ``classifications`` (its + classification/calibration pairs, from :func:`classifications_by_variant`), or ``None``. Mirrors the UI's calibration cascade — ``primary`` then ``investigator_provided``, strongest evidence within a tier, then ``id`` for determinism. Calibrations the caller can't read are skipped rather than @@ -126,17 +131,9 @@ def _preferred_classification( """ candidates = [ (classification, calibration) - for classification, calibration in db.execute( - select(ScoreCalibrationFunctionalClassification, ScoreCalibration) - .join( - classification_variants, - classification_variants.c.functional_classification_id == ScoreCalibrationFunctionalClassification.id, - ) - .join(ScoreCalibration, ScoreCalibration.id == ScoreCalibrationFunctionalClassification.calibration_id) - .where(classification_variants.c.variant_id == variant.id) - .where(ScoreCalibration.research_use_only.is_(research_use_only)) - ).all() - if has_permission(user_data, calibration, Action.READ).permitted + for classification, calibration in classifications + if calibration.research_use_only is research_use_only + and has_permission(user_data, calibration, Action.READ).permitted ] if not candidates: return None @@ -328,13 +325,18 @@ def get_allele_measurements( joinedload(Variant.score_set) .joinedload(ScoreSet.experiment) .selectinload(Experiment.keyword_objs) - .joinedload(ExperimentControlledKeywordAssociation.controlled_keyword) + .joinedload(ExperimentControlledKeywordAssociation.controlled_keyword), + # Read per measurement below; loaded once per score set rather than lazily per row. + joinedload(Variant.score_set).selectinload(ScoreSet.score_calibrations), + joinedload(Variant.score_set).selectinload(ScoreSet.superseding_score_set), ) ) .tuples() .all() ) + classifications = classifications_by_variant(db, [variant.id for _, variant, _ in rows]) + measurements: list[tuple[AlleleMeasurement, Optional[date], int]] = [] clinically_calibrated: dict[int, bool] = {} protein_consequence_count = 0 @@ -369,13 +371,16 @@ def get_allele_measurements( apex_only_unresolved_count += 1 assay_level = SequenceLevel(record.assay_level) if record.assay_level else None - classification = _preferred_classification(db, variant, user_data=user_data) + variant_classifications = classifications.get(variant.id, []) + classification = _preferred_classification(variant_classifications, user_data=user_data) classification_is_research_use_only = False if include_research_use_only and classification is None: if score_set.id not in clinically_calibrated: clinically_calibrated[score_set.id] = _has_readable_clinical_calibration(score_set, user_data=user_data) if not clinically_calibrated[score_set.id]: - classification = _preferred_classification(db, variant, user_data=user_data, research_use_only=True) + classification = _preferred_classification( + variant_classifications, user_data=user_data, research_use_only=True + ) classification_is_research_use_only = classification is not None measurement = AlleleMeasurement( diff --git a/src/mavedb/lib/alleles.py b/src/mavedb/lib/alleles.py index b76a572ee..29ce9a758 100644 --- a/src/mavedb/lib/alleles.py +++ b/src/mavedb/lib/alleles.py @@ -8,15 +8,17 @@ ties all three together. """ -from collections.abc import Collection +from collections.abc import Callable, Collection, Iterator, Sequence +from dataclasses import dataclass from datetime import datetime -from typing import Optional +from typing import Optional, TypeVar from sqlalchemy import select from sqlalchemy.orm import Session, joinedload from mavedb.lib.permissions import Action, has_permission from mavedb.lib.types.authentication import UserData +from mavedb.lib.utils import batched from mavedb.models.allele import Allele from mavedb.models.mapping_record import MappingRecord from mavedb.models.mapping_record_allele import MappingRecordAllele @@ -24,6 +26,24 @@ from mavedb.models.variant import Variant +# How many variants a whole-set builder loads per round of batched queries. Bounds both the size of each +# query's IN list (a variant's record links can fan out to dozens of reverse-translation members) and the +# number of ORM rows held at once. +VARIANT_CHUNK_SIZE = 500 + +VariantT = TypeVar("VariantT") +InputsT = TypeVar("InputsT") + + +@dataclass(frozen=True) +class LiveRecordLinks: + """A variant's live (or as-of) ``MappingRecord`` and that record's live ``MappingRecordAllele`` links, + each with its ``allele`` eagerly loaded.""" + + record: MappingRecord + links: list[MappingRecordAllele] + + def readable_score_set_ids_by_allele( db: Session, user_data: Optional[UserData], allele_ids: Collection[int] ) -> dict[int, set[int]]: @@ -52,36 +72,53 @@ def readable_score_set_ids_by_allele( return by_allele +def with_chunk_inputs( + variants: Sequence[VariantT], load: Callable[[Sequence[VariantT]], InputsT] +) -> Iterator[tuple[VariantT, InputsT]]: + """Pair each variant with the inputs ``load`` builds for its chunk, one :data:`VARIANT_CHUNK_SIZE` chunk at + a time, so a whole-set builder runs a fixed number of queries per chunk rather than per variant.""" + for chunk in batched(variants, VARIANT_CHUNK_SIZE): + inputs = load(chunk) + for variant in chunk: + yield variant, inputs + + def get_live_record_allele_links( - db: Session, variant_id: int, *, as_of: Optional[datetime] = None -) -> list[MappingRecordAllele]: - """Return the live ``MappingRecordAllele`` links of a variant's single live ``MappingRecord``, - each with its ``allele`` eagerly loaded and its ``is_authoritative`` flag. - - This is **record-scoped**, deliberately unlike :func:`get_allele_translations`: it stays within - one variant's own mapping record rather than taking the cross-record union an anchor allele can - belong to. That scope is what the per-variant Cat-VRS transit needs — the variant's measured - (authoritative) allele as the defining representation and exactly its co-linked members, not the - equivalence class assembled from every record that happens to share a deduplicated allele. - - Temporal: defaults to the currently-live record and links (``valid_to IS NULL``). ``as_of`` - applies the same half-open predicate to both the record and the links, so the set is evaluated at - one instant. Returns ``[]`` when the variant has no live record. + db: Session, variant_ids: Collection[int], *, as_of: Optional[datetime] = None +) -> dict[int, LiveRecordLinks]: + """Return each variant's live ``MappingRecord`` and its live allele links, keyed by variant id. + + Two queries whatever the number of variants, so whole-set builders load a chunk of variants at a time + rather than one variant per round trip. A variant with no live record is absent from the result. + + This is **record-scoped**, deliberately unlike :func:`get_allele_translations`: it stays within each + variant's own mapping record rather than taking the cross-record union an anchor allele can belong to. + That scope is what the per-variant Cat-VRS transit needs — the variant's measured (authoritative) + allele as the defining representation and exactly its co-linked members, not the equivalence class + assembled from every record that happens to share a deduplicated allele. + + Temporal: defaults to the currently-live records and links (``valid_to IS NULL``). ``as_of`` applies + the same half-open predicate to both, so each set is evaluated at one instant. """ - record_id = db.scalar( - select(MappingRecord.id).where(MappingRecord.variant_id == variant_id).where(MappingRecord.live_at(as_of)) - ) - if record_id is None: - return [] + if not variant_ids: + return {} - return list( - db.scalars( - select(MappingRecordAllele) - .where(MappingRecordAllele.mapping_record_id == record_id) - .where(MappingRecordAllele.live_at(as_of)) - .options(joinedload(MappingRecordAllele.allele)) - ).all() - ) + records = db.scalars( + select(MappingRecord).where(MappingRecord.variant_id.in_(variant_ids)).where(MappingRecord.live_at(as_of)) + ).all() + if not records: + return {} + + links_by_record: dict[int, list[MappingRecordAllele]] = {record.id: [] for record in records} + for link in db.scalars( + select(MappingRecordAllele) + .where(MappingRecordAllele.mapping_record_id.in_(links_by_record)) + .where(MappingRecordAllele.live_at(as_of)) + .options(joinedload(MappingRecordAllele.allele)) + ): + links_by_record[link.mapping_record_id].append(link) + + return {record.variant_id: LiveRecordLinks(record, links_by_record[record.id]) for record in records} def find_variants_by_vrs_identifier( @@ -107,6 +144,8 @@ def find_variants_by_vrs_identifier( .where(Allele.vrs_digest == identifier) .where(MappingRecordAllele.live_at(as_of)) .where(MappingRecord.live_at(as_of)) + # Callers check read permission on each match's score set; load them here rather than one per match. + .options(joinedload(Variant.score_set)) ) return [(row[0], row[1]) for row in db.execute(stmt).unique().all()] diff --git a/src/mavedb/lib/annotation/context.py b/src/mavedb/lib/annotation/context.py index f824dc82f..eca672cd7 100644 --- a/src/mavedb/lib/annotation/context.py +++ b/src/mavedb/lib/annotation/context.py @@ -7,15 +7,14 @@ from dataclasses import dataclass from datetime import datetime -from typing import Optional +from typing import Optional, Sequence from ga4gh.cat_vrs.models import CategoricalVariant from ga4gh.vrs.models import MolecularVariation -from sqlalchemy import select from sqlalchemy.orm import Session -from mavedb.lib.allele_annotations import get_allele_cross_references -from mavedb.lib.alleles import get_live_record_allele_links +from mavedb.lib.allele_annotations import AlleleCrossReferences, get_allele_cross_references +from mavedb.lib.alleles import LiveRecordLinks, get_live_record_allele_links from mavedb.lib.cat_vrs import build_categorical_variant, categorical_member_links from mavedb.lib.vrs import vrs_object_from_mapped_variant from mavedb.models.allele import Allele @@ -41,48 +40,87 @@ class VariantAnnotationContext: as_of: Optional[datetime] -def variant_annotation_context( - db: Session, variant: Variant, *, as_of: Optional[datetime] = None -) -> Optional[VariantAnnotationContext]: - """Assemble the annotation context for ``variant`` from the live (or as-of) mapping substrate. +@dataclass(frozen=True) +class AnnotationContextInputs: + """The database inputs for a chunk of variants' annotation contexts, loaded up front. - Returns ``None`` when the variant is unmapped at ``as_of`` or an authoritative allele that carries - no ``post_mapped`` VRS (nothing to annotate). One record fetch and one allele-link fetch; a subject with - projection members adds one gnomAD and one ClinVar identifier fetch, scoped to those members, for its - ``mappings``. The Cat-VRS transit is built from the same links. + ``record_links`` holds each variant's live (or as-of) record and links; ``cross_references`` holds the + gnomAD and ClinVar identifiers of every allele a proposition subject in the chunk draws on, keyed by + digest. :meth:`context_for` builds one variant's context from these without touching the database, so a + caller can isolate a failure to that variant. """ - record = db.scalar( - select(MappingRecord).where(MappingRecord.variant_id == variant.id).where(MappingRecord.live_at(as_of)) - ) - if record is None: - return None - - links = get_live_record_allele_links(db, variant.id, as_of=as_of) - measured_allele = next((link.allele for link in links if link.is_authoritative), None) - if measured_allele is None or measured_allele.post_mapped is None: - return None - - # The proposition subject follows the measured as anchor rule: the categorical variant when it carries a - # projection member, else the concrete measured variation. The VA subject is deliberately *narrow* - # (include_convergent=False): the convergent encodings are dropped, because VA-Spec carries no per-member - # provenance to mark them as unmeasured, and StudyResult.focus already pins the concrete measured - # allele. A lone measured allele is served bare, so it skips the cross-reference fetch and the build. - member_links = categorical_member_links(links, include_convergent=False) - transit = None - if len(member_links) > 1: - cross_references = get_allele_cross_references(db, [link.allele for link in member_links], as_of=as_of) - transit = build_categorical_variant( - links, name=variant.urn or "", include_convergent=False, cross_references=cross_references + + record_links: dict[int, LiveRecordLinks] + cross_references: dict[str, AlleleCrossReferences] + as_of: Optional[datetime] + + def context_for(self, variant: Variant) -> Optional[VariantAnnotationContext]: + """Build ``variant``'s context, or ``None`` when it is unmapped at ``as_of`` or its authoritative + allele carries no ``post_mapped`` VRS (nothing to annotate).""" + record_links = self.record_links.get(variant.id) + if record_links is None: + return None + + links = record_links.links + measured_allele = next((link.allele for link in links if link.is_authoritative), None) + if measured_allele is None or measured_allele.post_mapped is None: + return None + + # The proposition subject follows the measured as anchor rule: the categorical variant when it carries a + # projection member, else the concrete measured variation. The VA subject is deliberately *narrow* + # (include_convergent=False): the convergent encodings are dropped, because VA-Spec carries no per-member + # provenance to mark them as unmeasured, and StudyResult.focus already pins the concrete measured + # allele. A lone measured allele is served bare. + transit = None + if len(categorical_member_links(links, include_convergent=False)) > 1: + transit = build_categorical_variant( + links, name=variant.urn or "", include_convergent=False, cross_references=self.cross_references + ) + if transit is not None and len(transit.categorical_variant.members or []) > 1: + subject_variant: MolecularVariation | CategoricalVariant = transit.categorical_variant + else: + subject_variant = vrs_object_from_mapped_variant(measured_allele.post_mapped) + + return VariantAnnotationContext( + variant=variant, + record=record_links.record, + measured_allele=measured_allele, + subject_variant=subject_variant, + as_of=self.as_of, ) - if transit is not None and len(transit.categorical_variant.members or []) > 1: - subject_variant: MolecularVariation | CategoricalVariant = transit.categorical_variant - else: - subject_variant = vrs_object_from_mapped_variant(measured_allele.post_mapped) - - return VariantAnnotationContext( - variant=variant, - record=record, - measured_allele=measured_allele, - subject_variant=subject_variant, + + +def load_annotation_context_inputs( + db: Session, variants: Sequence[Variant], *, as_of: Optional[datetime] = None +) -> AnnotationContextInputs: + """Load the annotation-context inputs for ``variants`` in at most four queries in total, however many + variants there are (where building each context alone would cost three or four per variant). + + Two for the records and links (:func:`get_live_record_allele_links`), and two for the cross-references + of the alleles in subjects with a projection member, skipped when no subject has one. Whole-set callers pass a chunk of variants + (:data:`~mavedb.lib.alleles.VARIANT_CHUNK_SIZE`). + """ + record_links = get_live_record_allele_links(db, [variant.id for variant in variants], as_of=as_of) + + subject_alleles: list[Allele] = [] + for entry in record_links.values(): + member_links = categorical_member_links(entry.links, include_convergent=False) + if len(member_links) > 1: + subject_alleles.extend(link.allele for link in member_links) + + return AnnotationContextInputs( + record_links=record_links, + cross_references=get_allele_cross_references(db, subject_alleles, as_of=as_of) if subject_alleles else {}, as_of=as_of, ) + + +def variant_annotation_context( + db: Session, variant: Variant, *, as_of: Optional[datetime] = None +) -> Optional[VariantAnnotationContext]: + """Assemble the annotation context for one variant from the live (or as-of) mapping substrate. + + The single-variant form of :func:`load_annotation_context_inputs`; see + :meth:`AnnotationContextInputs.context_for` for when it returns ``None``. + """ + return load_annotation_context_inputs(db, [variant], as_of=as_of).context_for(variant) diff --git a/src/mavedb/lib/cat_vrs.py b/src/mavedb/lib/cat_vrs.py index 097403ccc..61dcaf5e4 100644 --- a/src/mavedb/lib/cat_vrs.py +++ b/src/mavedb/lib/cat_vrs.py @@ -22,6 +22,7 @@ from ga4gh.vrs.models import CisPhasedBlock from mavedb.lib.allele_annotations import AlleleCrossReferences +from mavedb.lib.allele_identity import AlleleDerivation from mavedb.lib.clingen.allele_registry import clingen_allele_url from mavedb.lib.clinvar.utils import clinvar_variation_url from mavedb.lib.gnomad import gnomad_variant_url @@ -65,8 +66,8 @@ class CatVrsRelation(str, Enum): COORDINATE_REPRESENTATION_OF = "coordinate_representation_of" # defining is nt; member is the protein consequence. Consequence, no independent score. TRANSLATION_OF = "translation_of" - # defining is nt; member is a *convergent encoding* — an nt allele in a different projection group that - # encodes the same protein consequence as the measured change. + # defining is nt; member is a *convergent encoding* — an nt allele not paired with the measured change that + # encodes the same protein consequence. CO_ENCODES = "co_encodes" @@ -94,72 +95,46 @@ class CategoricalVariantTransit: categorical_variant: CategoricalVariant mode: CatVrsMode - member_relations: dict[str, CatVrsRelation] - """vrs_digest keyed dict encoding digest: relation for the member -> defining relationship. - Excludes the defining allele itself. - """ -def is_convergent_encoding( - member_level: Optional[str], member_group: Optional[int], *, defining_group: Optional[int] -) -> bool: - """Whether a projection-mode member is a *convergent encoding* of the measured change. +def is_projection_partner(member_group: Optional[int], *, focus_group: Optional[int]) -> bool: + """Whether a member is the focus allele's own c↔g projection: the two share a projection group. - Concretely: - - The defining (measured) allele is nt (genomic or cdna). - - The member is nt (genomic or cdna). - - The member is in a different projection group than the defining (measured) allele. + A member or focus with no group can't be shown to pair, so it counts as no projection. That includes a + measured allele RT's fold-in missed (``fold_in_missed`` on its ``cross_level_translation`` event), whose + own twin then reads as a convergent encoding: under-claiming one exact match rather than presenting + distinct encodings, and their ClinVar/gnomAD records, as the measured change. """ - return ( - member_level in NUCLEOTIDE_LEVELS - and member_group is not None - and defining_group is not None - and member_group != defining_group - ) + return member_group is not None and member_group == focus_group -def _relation_for( - defining_level: Optional[str], - member_level: Optional[str], - *, - defining_group: Optional[int] = None, - member_group: Optional[int] = None, -) -> Optional[CatVrsRelation]: - """Derive the member -> defining relation from the two levels, or None for the defining itself. - - Star model: every member relates to the single defining allele, so the relation depends only on the - (defining, member) level pair. Note that in projection mode relation also depends on whether the nt - member shares the measured change's ``projection_group`` (its projection) or lives in another group - (a convergent encoding). - """ - # Protein measured. Every nt member encodes it, the protein member is the defining. Grouping is irrelevant. - if defining_level == SequenceLevel.protein.value: - return CatVrsRelation.ENCODES if member_level in NUCLEOTIDE_LEVELS else None - - # Nt measured (genomic or cdna). - if member_level == SequenceLevel.protein.value: - return CatVrsRelation.TRANSLATION_OF - if member_level in NUCLEOTIDE_LEVELS: - if is_convergent_encoding(member_level, member_group, defining_group=defining_group): - return CatVrsRelation.CO_ENCODES - return CatVrsRelation.COORDINATE_REPRESENTATION_OF +def member_label( + focus_level: Optional[str], member_level: Optional[str], *, is_projection: bool +) -> tuple[Optional[CatVrsRelation], Optional[AlleleDerivation]]: + """A member allele's ``(relation, derivation)`` relative to the focus allele, ``(None, None)`` for none. - # Unreachable in practice. Levels are constrained by the DB and the mapping job. - return None # pragma: no cover + The single source of both axes for every view (Cat-VRS, variant detail, allele detail), so they cannot + label the same pair differently. ``is_projection`` is whether the member is the focus's own c↔g pair + (:func:`is_projection_partner` within a record). - -def _is_precise_projection_member( - member_level: Optional[str], member_group: Optional[int], *, defining_group: Optional[int] -) -> bool: - """In projection mode, whether a member is a *precise* representation of the measured change. - - True for the protein consequence (the apex, shared by the whole equivalence class) and the measured - change's own projection. False for convergent encodings in other projection groups. + - Protein focus: every nucleotide member ``encodes`` it, a reverse-translation ``candidate``. + - Nucleotide focus: the protein consequence is its ``translation_of`` and its pair its + ``coordinate_representation_of``, both deterministic ``projection``s; any other nucleotide member + ``co_encodes`` the consequence, a distinct ``convergent`` change. """ + if focus_level == SequenceLevel.protein.value: + if member_level in NUCLEOTIDE_LEVELS: + return CatVrsRelation.ENCODES, AlleleDerivation.CANDIDATE + return None, None + if member_level == SequenceLevel.protein.value: - return True + return CatVrsRelation.TRANSLATION_OF, AlleleDerivation.PROJECTION + if member_level in NUCLEOTIDE_LEVELS: + if is_projection: + return CatVrsRelation.COORDINATE_REPRESENTATION_OF, AlleleDerivation.PROJECTION + return CatVrsRelation.CO_ENCODES, AlleleDerivation.CONVERGENT - return not is_convergent_encoding(member_level, member_group, defining_group=defining_group) + return None, None def _relation_concept(relation: CatVrsRelation) -> MappableConcept: @@ -269,13 +244,12 @@ def categorical_member_links( if link is defining_link: continue - if ( - mode is CatVrsMode.PROJECTION - and not include_convergent - and not _is_precise_projection_member( - link.allele.level, link.projection_group, defining_group=defining_link.projection_group - ) - ): + relation, _ = member_label( + defining_link.allele.level, + link.allele.level, + is_projection=is_projection_partner(link.projection_group, focus_group=defining_link.projection_group), + ) + if mode is CatVrsMode.PROJECTION and not include_convergent and relation is CatVrsRelation.CO_ENCODES: continue members.append(link) @@ -324,8 +298,8 @@ def build_categorical_variant( - ``True`` (default; the detail envelope): the encodings are kept as members wearing the :attr:`CatVrsRelation.CO_ENCODES` relation, so the object is the full closure. The measured change's projection and protein consequence stay ``coordinate_representation_of`` / ``translation_of``. - - ``False`` (the VA-Spec subject): the encodings are dropped (see :func:`_is_precise_projection_member`) - and only the measured change's precise projection and protein consequence remain. + - ``False`` (the VA-Spec subject): the encodings are dropped (see :func:`member_label`) and only the + measured change's precise projection and protein consequence remain. ``mappings`` cross-references the included members' external records (see :func:`_external_mappings`). CAIDs come from the alleles themselves; gnomAD and ClinVar identifiers come from ``cross_references``, @@ -357,7 +331,6 @@ def build_categorical_variant( members: list[VrsAllele | CisPhasedBlock | iriReference] = [defining_vrs] mapped_members: list[tuple[Allele, Optional[CatVrsRelation]]] = [(defining_allele, None)] - member_relations: dict[str, CatVrsRelation] = {} relations_present: dict[CatVrsRelation, None] = {} # insertion-ordered set of relation kinds for link in other_links: @@ -375,16 +348,13 @@ def build_categorical_variant( continue members.append(member_vrs) - relation = _relation_for( + relation, _ = member_label( defining_level, allele.level, - defining_group=defining_link.projection_group, - member_group=link.projection_group, + is_projection=is_projection_partner(link.projection_group, focus_group=defining_link.projection_group), ) if relation is not None: mapped_members.append((allele, relation)) - if relation is not None and allele.vrs_digest is not None: - member_relations[allele.vrs_digest] = relation relations_present[relation] = None # The defining allele anchors the DefiningAlleleConstraint. Cat-VRS 1.0.0 requires a bare @@ -409,5 +379,4 @@ def build_categorical_variant( return CategoricalVariantTransit( categorical_variant=categorical_variant, mode=mode, - member_relations=member_relations, ) diff --git a/src/mavedb/lib/score_calibrations.py b/src/mavedb/lib/score_calibrations.py index a50ade34c..afa4723ac 100644 --- a/src/mavedb/lib/score_calibrations.py +++ b/src/mavedb/lib/score_calibrations.py @@ -236,6 +236,29 @@ def classification_evidence_strength( return (0.0, 2) +def classifications_by_variant( + db: Session, variant_ids: Sequence[int] +) -> dict[int, list[tuple[ScoreCalibrationFunctionalClassification, ScoreCalibration]]]: + """Each variant's functional classifications with their calibrations, in one query for all of + ``variant_ids``. Unordered and unfiltered: callers apply their own visibility rule and preference.""" + if not variant_ids: + return {} + + association = score_calibration_functional_classification_variants_association_table + rows = db.execute( + select(association.c.variant_id, ScoreCalibrationFunctionalClassification, ScoreCalibration) + .join(association, association.c.functional_classification_id == ScoreCalibrationFunctionalClassification.id) + .join(ScoreCalibration, ScoreCalibration.id == ScoreCalibrationFunctionalClassification.calibration_id) + .where(association.c.variant_id.in_(variant_ids)) + ).tuples() + + by_variant: dict[int, list[tuple[ScoreCalibrationFunctionalClassification, ScoreCalibration]]] = {} + for variant_id, classification, calibration in rows: + by_variant.setdefault(variant_id, []).append((classification, calibration)) + + return by_variant + + def create_functional_classification( db: Session, functional_range_create: Union[ diff --git a/src/mavedb/lib/variant_detail.py b/src/mavedb/lib/variant_detail.py index 1454cb9c4..c334ec155 100644 --- a/src/mavedb/lib/variant_detail.py +++ b/src/mavedb/lib/variant_detail.py @@ -14,23 +14,17 @@ from dataclasses import dataclass from datetime import datetime -from typing import Any, Optional +from typing import Any, Optional, Sequence -from sqlalchemy import select from sqlalchemy.orm import Session from mavedb.lib.allele_annotations import AlleleAnnotations, get_allele_annotations -from mavedb.lib.allele_identity import AlleleDerivation, AlleleIdentity -from mavedb.lib.alleles import get_live_record_allele_links -from mavedb.lib.cat_vrs import build_categorical_variant, is_convergent_encoding -from mavedb.lib.score_calibrations import calibration_preference_key +from mavedb.lib.allele_identity import AlleleIdentity +from mavedb.lib.alleles import LiveRecordLinks, get_live_record_allele_links +from mavedb.lib.cat_vrs import build_categorical_variant, is_projection_partner, member_label +from mavedb.lib.score_calibrations import calibration_preference_key, classifications_by_variant from mavedb.models.enums.sequence_level import SequenceLevel -from mavedb.models.mapping_record import MappingRecord -from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification -from mavedb.models.score_calibration_functional_classification_variant_association import ( - score_calibration_functional_classification_variants_association_table as classification_variants, -) from mavedb.models.score_set import ScoreSet from mavedb.models.variant import Variant @@ -49,6 +43,34 @@ class VariantClassificationRecord: classification: ScoreCalibrationFunctionalClassification +@dataclass(frozen=True) +class VariantDetailInputs: + """The database inputs for a chunk of variants' detail envelopes, loaded up front. + + ``annotations`` is keyed by digest across the whole chunk; each envelope takes the entries for its own + links. :meth:`detail_for` assembles one variant's envelope without touching the database. + """ + + record_links: dict[int, LiveRecordLinks] + annotations: dict[str, AlleleAnnotations] + classifications: dict[int, list[VariantClassificationRecord]] + + def detail_for(self, variant: Variant, *, superseding_score_set: Optional[ScoreSet] = None) -> "VariantDetail": + """Assemble the variant-detail envelope for ``variant``. + + ``superseding_score_set`` is the newer version the caller already resolved for visibility + (``None`` if unreadable, mirroring ``fetch_score_set_by_urn``); its presence drives + ``is_current`` / ``superseded_by_score_set``. + """ + return _assemble_variant_detail( + variant, + self.record_links.get(variant.id), + self.annotations, + self.classifications.get(variant.id, []), + superseding_score_set=superseding_score_set, + ) + + @dataclass(frozen=True) class VariantDetail: """The assembled variant-detail envelope (transit; serialized by ``view_models.variant``).""" @@ -89,62 +111,27 @@ class VariantDetail: superseded_by_score_set: Optional[str] -def _classifications_for_variant( - db: Session, variant: Variant, *, visible_calibration_ids: Optional[set[int]] -) -> list[VariantClassificationRecord]: - """Functional classifications the variant belongs to, one row per calibration that classifies it. - - Membership is read from the calibration→classification→variant association table, ordered by - ``calibration_preference_key`` (+ id) so the first entry matches the UI's default. - ``visible_calibration_ids`` restricts to readable calibrations (``None`` = no restriction, for - lib-level callers). - """ - statement = ( - select(ScoreCalibrationFunctionalClassification, ScoreCalibration) - .join( - classification_variants, - classification_variants.c.functional_classification_id == ScoreCalibrationFunctionalClassification.id, - ) - .join(ScoreCalibration, ScoreCalibration.id == ScoreCalibrationFunctionalClassification.calibration_id) - .where(classification_variants.c.variant_id == variant.id) - ) - if visible_calibration_ids is not None: - statement = statement.where(ScoreCalibration.id.in_(visible_calibration_ids)) - - rows = sorted( - db.execute(statement).tuples().all(), key=lambda row: (*calibration_preference_key(row[1]), row[1].id) - ) - return [ - VariantClassificationRecord( - calibration_id=calibration.id, primary=calibration.primary, classification=classification - ) - for classification, calibration in rows - ] - - -def _derivation_for(*, assay_level: Optional[SequenceLevel], is_convergent: bool) -> AlleleDerivation: - """Provenance of a *non-focus* linked allele, derived from the assay level. - - The measured allele is the focus (no derivation). For every other allele: - - - **Protein assay** → ``CANDIDATE``: reverse translation is ambiguous, so the derived nucleotide - fan-out is only a candidate (which codon was actually measured is unknown). - - **Nucleotide assay, convergent encoding** (``is_convergent``) → ``CONVERGENT``: an nt member in - a different projection group. A distinct, precisely-known variant that happens to converge on - the same protein consequence, not a projection of the measured change. - - **Nucleotide assay, otherwise** → ``PROJECTION``: the deterministic class derived from the - measured change itself. +def _classifications_by_variant( + db: Session, variant_ids: Sequence[int], *, visible_calibration_ids: Optional[set[int]] +) -> dict[int, list[VariantClassificationRecord]]: + """Functional classifications each variant belongs to, one row per calibration that classifies it. - ``is_convergent`` comes from :func:`cat_vrs.is_convergent_encoding`, keeping this ``derivation`` - axis aligned with the Cat-VRS ``relation`` axis (``co_encodes`` ↔ ``convergent``, - ``coordinate_representation_of`` / ``translation_of`` ↔ ``projection``, ``encodes`` ↔ - ``candidate``). + Each variant's rows are ordered by ``calibration_preference_key`` (+ id), so the first entry matches the + UI's default. ``visible_calibration_ids`` restricts to readable calibrations (``None`` = no restriction, + for lib-level callers). """ - if assay_level == SequenceLevel.protein.value: - return AlleleDerivation.CANDIDATE - if is_convergent: - return AlleleDerivation.CONVERGENT - return AlleleDerivation.PROJECTION + return { + variant_id: [ + VariantClassificationRecord( + calibration_id=calibration.id, primary=calibration.primary, classification=classification + ) + for classification, calibration in sorted( + pairs, key=lambda pair: (*calibration_preference_key(pair[1]), pair[1].id) + ) + if visible_calibration_ids is None or calibration.id in visible_calibration_ids + ] + for variant_id, pairs in classifications_by_variant(db, variant_ids).items() + } def _submitted_assay_level_hgvs(variant: Variant, assay_level: Optional[SequenceLevel]) -> Optional[str]: @@ -156,6 +143,31 @@ def _submitted_assay_level_hgvs(variant: Variant, assay_level: Optional[Sequence return variant.hgvs_nt +def load_variant_detail_inputs( + db: Session, + variants: Sequence[Variant], + *, + visible_calibration_ids: Optional[set[int]] = None, + as_of: Optional[datetime] = None, +) -> VariantDetailInputs: + """Load the detail-envelope inputs for ``variants`` in six queries in total, however many variants there + are (where building each envelope alone would cost six per variant). + + Two for the records and links, three for the annotations of every linked allele, and one for the + classifications. Whole-set callers pass a chunk of variants (:data:`~mavedb.lib.alleles.VARIANT_CHUNK_SIZE`). + ``visible_calibration_ids`` restricts classifications to readable calibrations. ``as_of`` reconstructs + the molecular layer only — see the module docstring. + """ + variant_ids = [variant.id for variant in variants] + record_links = get_live_record_allele_links(db, variant_ids, as_of=as_of) + alleles = [link.allele for entry in record_links.values() for link in entry.links] + return VariantDetailInputs( + record_links=record_links, + annotations=get_allele_annotations(db, alleles, as_of=as_of), + classifications=_classifications_by_variant(db, variant_ids, visible_calibration_ids=visible_calibration_ids), + ) + + def get_variant_detail( db: Session, variant: Variant, @@ -164,36 +176,45 @@ def get_variant_detail( visible_calibration_ids: Optional[set[int]] = None, as_of: Optional[datetime] = None, ) -> VariantDetail: - """Assemble the variant-detail envelope for ``variant``. + """Assemble the variant-detail envelope for one variant: the single-variant form of + :func:`load_variant_detail_inputs` and :meth:`VariantDetailInputs.detail_for`.""" + inputs = load_variant_detail_inputs(db, [variant], visible_calibration_ids=visible_calibration_ids, as_of=as_of) + return inputs.detail_for(variant, superseding_score_set=superseding_score_set) - ``superseding_score_set`` is the newer version the caller already resolved for visibility - (``None`` if unreadable, mirroring ``fetch_score_set_by_urn``); its presence drives - ``is_current`` / ``superseded_by_score_set``. ``visible_calibration_ids`` restricts - classifications to readable calibrations. ``as_of`` reconstructs the molecular layer only — - see the module docstring. - """ + +def _assemble_variant_detail( + variant: Variant, + record_links: Optional[LiveRecordLinks], + chunk_annotations: dict[str, AlleleAnnotations], + classifications: list[VariantClassificationRecord], + *, + superseding_score_set: Optional[ScoreSet], +) -> VariantDetail: + """Build one variant's envelope from its loaded record links, the chunk's annotations and its + classifications.""" data = variant.data if isinstance(variant.data, dict) else {} scores = data.get("score_data") counts = data.get("count_data") # The live mapping record supplies the assay level and the mapped (reference-frame) assay HGVS. - record = db.scalar( - select(MappingRecord).where(MappingRecord.variant_id == variant.id).where(MappingRecord.live_at(as_of)) - ) + record = record_links.record if record_links is not None else None assay_level = SequenceLevel(record.assay_level) if record is not None else None reference_hgvs = record.hgvs_assay_level if record is not None else None # The live allele links: the authoritative allele gives the assay-level digest + ClinGen id; all # linked alleles seed the digest-keyed annotation map. - links = get_live_record_allele_links(db, variant.id, as_of=as_of) + links = record_links.links if record_links is not None else [] authoritative = next((link for link in links if link.is_authoritative), None) assay_level_digest = authoritative.allele.vrs_digest if authoritative is not None else None clingen_allele_id = authoritative.allele.clingen_allele_id if authoritative is not None else None - annotations = get_allele_annotations(db, [link.allele for link in links], as_of=as_of) + annotations = { + link.allele.vrs_digest: chunk_annotations[link.allele.vrs_digest] + for link in links + if link.allele.vrs_digest in chunk_annotations + } - # Spec-pure Cat-VRS built on the fly, plus mode + per-member relations. The defining allele is - # deliberately absent from `relations`, so it gets relation=None below. + # Spec-pure Cat-VRS built on the fly, plus its mode. transit = build_categorical_variant( links, name=variant.urn or "", @@ -202,11 +223,9 @@ def get_variant_detail( if transit is not None: molecular_representation = transit.categorical_variant.model_dump(mode="json", exclude_none=True) mode: Optional[str] = transit.mode.value - relations = transit.member_relations else: molecular_representation = None mode = None - relations = {} # Per-allele identity sidecar: one entry per linked allele, keyed by VRS digest (the same link # set that seeds `annotations`, so the two maps share keys). Carries level, reference-frame HGVS @@ -220,9 +239,9 @@ def get_variant_detail( if link.projection_group is not None and link.allele.vrs_digest is not None: group_digests.setdefault(link.projection_group, []).append(link.allele.vrs_digest) - # Anchor for the convergence test: an nt member in a *different* projection group than the - # measured change is a convergent encoding (co_encodes / convergent), not a projection of it. - defining_group = authoritative.projection_group if authoritative is not None else None + # Members are labelled relative to the measured (authoritative) allele. + focus_level = authoritative.allele.level if authoritative is not None else None + focus_group = authoritative.projection_group if authoritative is not None else None alleles: dict[str, AlleleIdentity] = {} for link in links: @@ -236,17 +255,22 @@ def get_variant_detail( None, ) - relation = relations.get(allele.vrs_digest) - is_convergent = is_convergent_encoding(allele.level, link.projection_group, defining_group=defining_group) + relation, derivation = ( + (None, None) + if link.is_authoritative + else member_label( + focus_level, + allele.level, + is_projection=is_projection_partner(link.projection_group, focus_group=focus_group), + ) + ) alleles[allele.vrs_digest] = AlleleIdentity( level=allele.level, hgvs=allele.hgvs, clingen_allele_id=allele.clingen_allele_id, is_focus=link.is_authoritative, relation=relation.value if relation is not None else None, - derivation=( - None if link.is_authoritative else _derivation_for(assay_level=assay_level, is_convergent=is_convergent) - ), + derivation=derivation, projection_of=projection_of, ) @@ -255,7 +279,7 @@ def get_variant_detail( urn=variant.urn or "", scores=scores, counts=counts, - classifications=_classifications_for_variant(db, variant, visible_calibration_ids=visible_calibration_ids), + classifications=classifications, assay_level=assay_level, target_hgvs=_submitted_assay_level_hgvs(variant, assay_level), reference_hgvs=reference_hgvs, diff --git a/src/mavedb/models/mapping_record.py b/src/mavedb/models/mapping_record.py index e0d952788..78ad5f8d6 100644 --- a/src/mavedb/models/mapping_record.py +++ b/src/mavedb/models/mapping_record.py @@ -22,7 +22,7 @@ class MappingRecord(ValidTime, Base): id: Mapped[int] = Column(Integer, primary_key=True) - variant_id = Column(Integer, nullable=False) + variant_id: Mapped[int] = Column(Integer, nullable=False) # Denormalized from the variant so an RLS policy can check the score set directly; delegating # through variants costs ~700 ms per lookup at production size (#833). The composite foreign key # below keeps it equal to the variant's score set. diff --git a/src/mavedb/routers/score_sets.py b/src/mavedb/routers/score_sets.py index d048b085c..425b1f49c 100644 --- a/src/mavedb/routers/score_sets.py +++ b/src/mavedb/routers/score_sets.py @@ -24,14 +24,14 @@ from mavedb import deps from mavedb.data_providers.services import CSV_UPLOAD_S3_BUCKET_NAME, s3_client from mavedb.db.timeouts import allow_long_statements +from mavedb.lib.alleles import with_chunk_inputs from mavedb.lib.annotation.annotate import ( variant_functional_impact_statement, variant_pathogenicity_statement, variant_study_result, ) from mavedb.lib.annotation.conformance import serialize_annotation -from mavedb.lib.annotation.context import variant_annotation_context -from mavedb.lib.deprecation import MAPPED_VARIANT_SUNSET, deprecation_headers, record_deprecated_usage +from mavedb.lib.annotation.context import AnnotationContextInputs, load_annotation_context_inputs from mavedb.lib.annotation.exceptions import EXPECTED_ABSENCE_EXCEPTIONS from mavedb.lib.authorization import ( get_current_user, @@ -40,6 +40,7 @@ require_current_user_with_email, ) from mavedb.lib.clinical_controls import get_clinical_control_options, get_clinical_controls_with_variant_urns +from mavedb.lib.collections import readable_official_collections from mavedb.lib.contributors import find_or_create_contributor from mavedb.lib.csv.build_limiter import csv_build_slot, rows_to_build from mavedb.lib.csv.columns import variants_to_csv_rows @@ -51,6 +52,7 @@ ) from mavedb.lib.csv.namespaces import CSV_NAMESPACES_PARAM_DESCRIPTION, CsvNamespaceStr from mavedb.lib.csv.score_set import available_score_set_csv_namespaces, get_score_set_variants_as_csv +from mavedb.lib.deprecation import MAPPED_VARIANT_SUNSET, deprecation_headers, record_deprecated_usage from mavedb.lib.exceptions import MixedTargetError, NonexistentOrcidUserError from mavedb.lib.experiments import enrich_experiment_with_num_score_sets from mavedb.lib.gnomad import get_gnomad_variants_with_variant_urns @@ -66,7 +68,6 @@ from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer from mavedb.lib.score_calibrations import create_score_calibration from mavedb.lib.score_set_variants import get_lean_score_set_variants -from mavedb.lib.collections import readable_official_collections from mavedb.lib.score_sets import ( csv_data_to_df, fetch_score_set_search_filter_options, @@ -84,14 +85,14 @@ from mavedb.lib.taxonomies import find_or_create_taxonomy from mavedb.lib.types.authentication import UserData from mavedb.lib.urn_redirects import record_urn_redirect -from mavedb.lib.validation.exceptions import ValidationError as MaveDBValidationError from mavedb.lib.urns import ( generate_experiment_set_urn, generate_experiment_urn, generate_score_set_urn, lock_urn_assignment, ) -from mavedb.lib.variant_detail import get_variant_detail +from mavedb.lib.validation.exceptions import ValidationError as MaveDBValidationError +from mavedb.lib.variant_detail import load_variant_detail_inputs from mavedb.lib.workflow.kickoff import enqueue_pipeline_for_score_set from mavedb.models.contributor import Contributor from mavedb.models.enums.processing_state import ProcessingState @@ -1018,21 +1019,20 @@ def _stream_score_set_variant_details( ): """Serialize the whole-set variant-detail export as NDJSON — one :class:`VariantDetail` per line. - Assembles each variant's detail envelope (the flat assay fields + ``preMapped``/``postMapped`` VRS - pair + the spec-pure GA4GH CategoricalVariant + the digest-keyed VEP/gnomAD/ClinVar annotation map) - one at a time, so a large score set streams rather than building every envelope up front. - ``superseding_score_set`` / ``visible_calibration_ids`` are resolved once for the whole set and - threaded into every per-variant build. + Each variant's detail envelope (the flat assay fields + ``preMapped``/``postMapped`` VRS pair + the + spec-pure GA4GH CategoricalVariant + the digest-keyed VEP/gnomAD/ClinVar annotation map) is assembled + from inputs loaded a chunk of variants at a time, so a large score set streams in a fixed number of + queries per chunk rather than building every envelope up front. ``superseding_score_set`` / + ``visible_calibration_ids`` are resolved once for the whole set. """ - for variant in variants: - detail = get_variant_detail( - db, - variant, - superseding_score_set=superseding_score_set, - visible_calibration_ids=visible_calibration_ids, - as_of=as_of, - ) - # get_variant_detail returns the lib transit dataclass; coerce it through the view model. + for variant, inputs in with_chunk_inputs( + variants, + lambda chunk: load_variant_detail_inputs( + db, chunk, visible_calibration_ids=visible_calibration_ids, as_of=as_of + ), + ): + detail = inputs.detail_for(variant, superseding_score_set=superseding_score_set) + # detail_for returns the lib transit dataclass; coerce it through the view model. # exclude_none=False keeps a stable key set per line for programmatic consumers. yield VariantDetail.model_validate(detail).model_dump_json(by_alias=True, exclude_none=False) + "\n" @@ -1442,7 +1442,7 @@ def get_score_set_counts_csv( def _annotation_stream_record( - db, variant, annotation_function, as_of=None + inputs: AnnotationContextInputs, variant, annotation_function ) -> tuple[dict, Literal["annotated", "unannotated", "errored"]]: """ Build the NDJSON record for one variant, and classify its outcome. @@ -1462,7 +1462,7 @@ def _annotation_stream_record( variant_urn = variant.urn try: - context = variant_annotation_context(db, variant, as_of=as_of) + context = inputs.context_for(variant) annotation = annotation_function(context) if context is not None else None annotation_data = serialize_annotation(annotation) if annotation else None except EXPECTED_ABSENCE_EXCEPTIONS: @@ -1510,8 +1510,10 @@ def _stream_generated_annotations(db, variants, annotation_function, as_of=None) outcome_counts = {"annotated": 0, "unannotated": 0, "errored": 0} logger.info(f"Starting streaming processing of {total_variants} variants") - for variant in variants: - result, outcome = _annotation_stream_record(db, variant, annotation_function, as_of=as_of) + for variant, inputs in with_chunk_inputs( + variants, lambda chunk: load_annotation_context_inputs(db, chunk, as_of=as_of) + ): + result, outcome = _annotation_stream_record(inputs, variant, annotation_function) outcome_counts[outcome] += 1 yield json.dumps(result, default=str) + "\n" diff --git a/src/mavedb/scripts/export_public_data.py b/src/mavedb/scripts/export_public_data.py index e19a6fd68..dd20f8fb1 100644 --- a/src/mavedb/scripts/export_public_data.py +++ b/src/mavedb/scripts/export_public_data.py @@ -31,9 +31,9 @@ from mavedb.lib.annotation.annotate import variant_highest_level_annotation from mavedb.lib.annotation.conformance import serialize_annotation -from mavedb.lib.annotation.context import variant_annotation_context +from mavedb.lib.annotation.context import load_annotation_context_inputs from mavedb.lib.allele_annotations import get_allele_cross_references -from mavedb.lib.alleles import get_live_record_allele_links +from mavedb.lib.alleles import get_live_record_allele_links, with_chunk_inputs from mavedb.lib.cat_vrs import build_categorical_variant from mavedb.lib.csv.entries import score_sets_have_current_mappings from mavedb.lib.csv.namespaces import CsvNamespace @@ -208,8 +208,10 @@ def va_ndjson(db: Session, score_set: ScoreSet, principal: Principal) -> str: VA-Spec layer. """ lines = [] - for variant in get_annotatable_variants(db, score_set): - context = variant_annotation_context(db, variant) + for variant, inputs in with_chunk_inputs( + get_annotatable_variants(db, score_set), lambda chunk: load_annotation_context_inputs(db, chunk) + ): + context = inputs.context_for(variant) annotation = variant_highest_level_annotation(context, principal=principal) if context is not None else None record = { "variant_urn": variant.urn, @@ -236,18 +238,23 @@ def vrs_ndjson(db: Session, score_set: ScoreSet) -> str: calibration-derived. """ lines = [] - for variant in get_annotatable_variants(db, score_set): - links = get_live_record_allele_links(db, variant.id) + + def load(chunk): + record_links = get_live_record_allele_links(db, [variant.id for variant in chunk]) + alleles = [link.allele for entry in record_links.values() for link in entry.links] + return record_links, get_allele_cross_references(db, alleles) + + for variant, (record_links, cross_references) in with_chunk_inputs(get_annotatable_variants(db, score_set), load): + entry = record_links.get(variant.id) + links = entry.links if entry is not None else [] authoritative = next((link.allele for link in links if link.is_authoritative), None) - record = next((link.mapping_record for link in links), None) - cross_references = get_allele_cross_references(db, [link.allele for link in links]) transit = build_categorical_variant(links, name=variant.urn or "", cross_references=cross_references) lines.append( json.dumps( { "variant_urn": variant.urn, - "pre_mapped": record.pre_mapped if record is not None else None, + "pre_mapped": entry.record.pre_mapped if entry is not None else None, "post_mapped": authoritative.post_mapped if authoritative is not None else None, "categorical_variant": ( transit.categorical_variant.model_dump(mode="json", exclude_none=True) diff --git a/src/mavedb/scripts/export_sweep.py b/src/mavedb/scripts/export_sweep.py index 6ba5bf604..fb04b55a7 100644 --- a/src/mavedb/scripts/export_sweep.py +++ b/src/mavedb/scripts/export_sweep.py @@ -43,7 +43,8 @@ variant_study_result, ) from mavedb.lib.annotation.conformance import AnnotationRoundTripError, round_trip_annotation -from mavedb.lib.annotation.context import VariantAnnotationContext, variant_annotation_context +from mavedb.lib.alleles import with_chunk_inputs +from mavedb.lib.annotation.context import VariantAnnotationContext, load_annotation_context_inputs from mavedb.lib.annotation.exceptions import EXPECTED_ABSENCE_EXCEPTIONS from mavedb.lib.csv.score_set import available_score_set_csv_namespaces, get_score_set_variants_as_csv from mavedb.lib.permissions.principal import Principal @@ -153,9 +154,9 @@ def sweep_annotation_surface( ) -> SurfaceResult: """Attempt one annotation surface across every annotatable variant of one score set. - The context is built per variant per surface, rather than once and reused, because that is what the - streaming endpoints do per request — a defect in context construction is a defect in what the API - serves, and the sweep only sees it by taking the same path. + The contexts are loaded a chunk at a time and built per variant per surface, rather than once and reused, + because that is what the streaming endpoints do per request — a defect in context construction is a defect + in what the API serves, and the sweep only sees it by taking the same path. Nothing raises out of here. A sweep that aborted on the first bad variant would report the corpus as far healthier than it is. @@ -166,11 +167,11 @@ def sweep_annotation_surface( variants_attempted=len(variants), ) - for variant in variants: + for variant, inputs in with_chunk_inputs(variants, lambda chunk: load_annotation_context_inputs(db, chunk)): variant_urn = getattr(variant, "urn", "") or "" try: - context = variant_annotation_context(db, variant) + context = inputs.context_for(variant) annotation = annotate(context) if context is not None else None except EXPECTED_ABSENCE_EXCEPTIONS: # An expected absence, drawn from the same definition the streaming endpoints use so the two diff --git a/tests/helpers/util/annotation.py b/tests/helpers/util/annotation.py index 7f92067cc..d8ee5ca31 100644 --- a/tests/helpers/util/annotation.py +++ b/tests/helpers/util/annotation.py @@ -33,7 +33,8 @@ class AlleleSpec: """One allele on a mapping record, plus the annotations that hang off it. - Digests are the allele's identity in the serving layer, so they must be distinct within a test. + Digests are the allele's identity in the serving layer: a digest already seeded reuses that allele, so give + a shared allele's annotations on one spec only. ``clinvar_control_ids`` / ``gnomad_variant_ids`` reference rows seeded elsewhere (the router fixtures seed ClinVar controls with ids 1 and 2); each id becomes a live link on this allele. """ @@ -89,18 +90,22 @@ def seed_mapping_record( session.add(record) session.commit() + # Alleles are deduplicated by digest, so one seeded under the same digest before is reused (an allele + # several records share, like a protein consequence). for spec in alleles: - allele = Allele( - vrs_digest=spec.digest, - level=spec.level, - clingen_allele_id=spec.clingen_allele_id, - hgvs_c=spec.hgvs_c, - hgvs_p=spec.hgvs_p, - hgvs_g=spec.hgvs_g, - post_mapped=spec.post_mapped if spec.post_mapped is not None else {"type": "Allele"}, - ) - session.add(allele) - session.commit() + allele = session.scalar(select(Allele).where(Allele.vrs_digest == spec.digest)) + if allele is None: + allele = Allele( + vrs_digest=spec.digest, + level=spec.level, + clingen_allele_id=spec.clingen_allele_id, + hgvs_c=spec.hgvs_c, + hgvs_p=spec.hgvs_p, + hgvs_g=spec.hgvs_g, + post_mapped=spec.post_mapped if spec.post_mapped is not None else {"type": "Allele"}, + ) + session.add(allele) + session.commit() links: list[Any] = [ MappingRecordAllele( diff --git a/tests/helpers/util/variant.py b/tests/helpers/util/variant.py index eede1e610..f2ef94890 100644 --- a/tests/helpers/util/variant.py +++ b/tests/helpers/util/variant.py @@ -112,7 +112,7 @@ def mock_worker_variant_insertion( ) variants = create_variants_data(scores, counts, None) num_variants = create_variants(db, item, variants) - assert num_variants == 3 + assert num_variants == len(scores) item.processing_state = ProcessingState.success item.dataset_columns = DatasetColumnsCreate( diff --git a/tests/lib/test_alleles.py b/tests/lib/test_alleles.py index f01f69378..ed8c277d5 100644 --- a/tests/lib/test_alleles.py +++ b/tests/lib/test_alleles.py @@ -1,7 +1,7 @@ # ruff: noqa: E402 """Integration tests for the record-scoped allele-link query backing Cat-VRS transit. -``get_live_record_allele_links`` stays within one variant's own live ``MappingRecord`` — unlike +``get_live_record_allele_links`` stays within each variant's own live ``MappingRecord`` — unlike ``get_allele_translations``, which takes the cross-record union an anchor allele can belong to. These tests pin that scoping, the ValidTime live/retired filtering, and the ``as_of`` reconstruction. """ @@ -70,6 +70,12 @@ def _digests(links): return {link.allele.vrs_digest for link in links} +def _links(session, variant_id, **kwargs): + """One variant's live links through the batched loader; empty when it has no live record.""" + entry = get_live_record_allele_links(session, [variant_id], **kwargs).get(variant_id) + return entry.links if entry is not None else [] + + @pytest.mark.integration def test_returns_live_links_with_alleles(session, setup_lib_db_with_score_set): """The variant's live record yields its authoritative + derived links, each allele eagerly loaded.""" @@ -81,7 +87,7 @@ def test_returns_live_links_with_alleles(session, setup_lib_db_with_score_set): _link(session, record, authoritative, is_authoritative=True) _link(session, record, derived, is_authoritative=False) - links = get_live_record_allele_links(session, variant.id) + links = _links(session, variant.id) assert _digests(links) == {"auth", "deriv"} # Exactly one authoritative (the defining allele); the eager-loaded allele is reachable. @@ -90,7 +96,7 @@ def test_returns_live_links_with_alleles(session, setup_lib_db_with_score_set): @pytest.mark.integration def test_no_mapping_record_returns_empty(session, setup_lib_db_with_score_set): - assert get_live_record_allele_links(session, 1) == [] + assert get_live_record_allele_links(session, [1]) == {} @pytest.mark.integration @@ -105,7 +111,7 @@ def test_excludes_a_retired_link(session, setup_lib_db_with_score_set): retired.retire(at=T1) session.commit() - assert _digests(get_live_record_allele_links(session, variant.id)) == {"live"} + assert _digests(_links(session, variant.id)) == {"live"} @pytest.mark.integration @@ -123,7 +129,7 @@ def test_excludes_a_superseded_record(session, setup_lib_db_with_score_set): new_record = _record(session, variant, valid_from=T1) _link(session, new_record, new_allele, is_authoritative=True, valid_from=T1) - assert _digests(get_live_record_allele_links(session, variant.id)) == {"new"} + assert _digests(_links(session, variant.id)) == {"new"} @pytest.mark.integration @@ -146,7 +152,7 @@ def test_record_scoped_not_cross_record_union(session, setup_lib_db_with_score_s _link(session, record_b, only_b) # Variant A's record sees the shared allele and its own member, never variant B's. - assert _digests(get_live_record_allele_links(session, variant_a.id)) == {"shared", "only_a"} + assert _digests(_links(session, variant_a.id)) == {"shared", "only_a"} @pytest.mark.integration @@ -165,8 +171,28 @@ def test_as_of_returns_the_historical_link_set(session, setup_lib_db_with_score_ _link(session, new_record, current_allele, is_authoritative=True, valid_from=T1) # Inside the old window: the historical set. - assert _digests(get_live_record_allele_links(session, variant.id, as_of=T0)) == {"past"} - assert _digests(get_live_record_allele_links(session, variant.id, as_of=T1 - timedelta(days=1))) == {"past"} + assert _digests(_links(session, variant.id, as_of=T0)) == {"past"} + assert _digests(_links(session, variant.id, as_of=T1 - timedelta(days=1))) == {"past"} # At/after the handoff and current: the new set. - assert _digests(get_live_record_allele_links(session, variant.id, as_of=T2)) == {"current"} - assert _digests(get_live_record_allele_links(session, variant.id)) == {"current"} + assert _digests(_links(session, variant.id, as_of=T2)) == {"current"} + assert _digests(_links(session, variant.id)) == {"current"} + + +@pytest.mark.integration +def test_one_call_returns_each_variants_own_record_and_links(session, setup_lib_db_with_score_set): + """The batched form keys each variant to its own record and links, and omits a variant with none.""" + score_set = setup_lib_db_with_score_set + variant_a = _variant(session, score_set, 1) + variant_b = _variant(session, score_set, 2) + unmapped = _variant(session, score_set, 3) + record_a = _record(session, variant_a) + record_b = _record(session, variant_b) + _link(session, record_a, _allele(session, "a"), is_authoritative=True) + _link(session, record_b, _allele(session, "b"), is_authoritative=True) + + result = get_live_record_allele_links(session, [variant_a.id, variant_b.id, unmapped.id]) + + assert set(result) == {variant_a.id, variant_b.id} + assert result[variant_a.id].record.id == record_a.id + assert _digests(result[variant_a.id].links) == {"a"} + assert _digests(result[variant_b.id].links) == {"b"} diff --git a/tests/lib/test_cat_vrs.py b/tests/lib/test_cat_vrs.py index 041f14dce..4c3fe193c 100644 --- a/tests/lib/test_cat_vrs.py +++ b/tests/lib/test_cat_vrs.py @@ -14,6 +14,7 @@ from ga4gh.cat_vrs.relations import Relation from ga4gh.core.models import Relation as MappingRelation +from mavedb.lib.allele_identity import AlleleDerivation from mavedb.lib.cat_vrs import ( _SPEC_EQUIVALENT, _relation_concept, @@ -21,6 +22,8 @@ CatVrsRelation, build_categorical_variant, categorical_member_links, + is_projection_partner, + member_label, ) from mavedb.lib.allele_annotations import AlleleCrossReferences, GnomadReference from mavedb.models.allele import Allele @@ -79,6 +82,46 @@ def _link( return MappingRecordAllele(is_authoritative=is_authoritative, allele=allele, projection_group=projection_group) +def _member_digests(links, *, include_convergent=True): + """The digests of the links a categorical variant draws its members from.""" + return {link.allele.vrs_digest for link in categorical_member_links(links, include_convergent=include_convergent)} + + +@pytest.mark.unit +@pytest.mark.parametrize( + "focus_level, member_level, is_projection, expected", + [ + ("protein", "cdna", False, (CatVrsRelation.ENCODES, AlleleDerivation.CANDIDATE)), + ("protein", "genomic", True, (CatVrsRelation.ENCODES, AlleleDerivation.CANDIDATE)), + ("protein", "protein", False, (None, None)), + ("cdna", "protein", False, (CatVrsRelation.TRANSLATION_OF, AlleleDerivation.PROJECTION)), + ("cdna", "genomic", True, (CatVrsRelation.COORDINATE_REPRESENTATION_OF, AlleleDerivation.PROJECTION)), + ("genomic", "cdna", True, (CatVrsRelation.COORDINATE_REPRESENTATION_OF, AlleleDerivation.PROJECTION)), + ("cdna", "cdna", False, (CatVrsRelation.CO_ENCODES, AlleleDerivation.CONVERGENT)), + ("genomic", "genomic", False, (CatVrsRelation.CO_ENCODES, AlleleDerivation.CONVERGENT)), + ], +) +def test_member_label(focus_level, member_level, is_projection, expected): + """Every view labels a member from this one function, relative to the focus allele.""" + assert member_label(focus_level, member_level, is_projection=is_projection) == expected + + +@pytest.mark.unit +@pytest.mark.parametrize( + "member_group, focus_group, expected", + [ + (0, 0, True), + (1, 0, False), + # Ungrouped either side can't be shown to pair: a pre-RT link, or a measured allele the fold-in missed. + (None, 0, False), + (0, None, False), + (None, None, False), + ], +) +def test_is_projection_partner(member_group, focus_group, expected): + assert is_projection_partner(member_group, focus_group=focus_group) is expected + + @pytest.mark.unit def test_mode_2_protein_measured_reverse_translation(): """Protein measured: defining is the protein allele; both nt members `encodes` it (star model).""" @@ -92,11 +135,6 @@ def test_mode_2_protein_measured_reverse_translation(): assert transit is not None assert transit.mode == CatVrsMode.REVERSE_TRANSLATION - # Per-member relations exclude the defining allele; both nt siblings encode the protein. - assert transit.member_relations == { - "cdna": CatVrsRelation.ENCODES, - "gen": CatVrsRelation.ENCODES, - } cv = transit.categorical_variant assert isinstance(cv, CategoricalVariant) @@ -122,10 +160,6 @@ def test_mode_1_coding_measured_projection(): assert transit is not None assert transit.mode == CatVrsMode.PROJECTION - assert transit.member_relations == { - "gen": CatVrsRelation.COORDINATE_REPRESENTATION_OF, - "prot": CatVrsRelation.TRANSLATION_OF, - } constraint = transit.categorical_variant.constraints[0].root codes = {str(r.primaryCoding.code.root) for r in constraint.relations} @@ -150,14 +184,6 @@ def test_mode_1_projection_includes_sibling_encoders(): transit = build_categorical_variant(links, name="urn:mavedb:test#2b") assert transit is not None - # The coordinate partner + protein consequence keep their faithful relations; the two cousins ride as - # co_encodes (distinct, unmeasured synonymous variants). - assert transit.member_relations == { - "gen": CatVrsRelation.COORDINATE_REPRESENTATION_OF, - "prot": CatVrsRelation.TRANSLATION_OF, - "sibling_cdna": CatVrsRelation.CO_ENCODES, - "sibling_gen": CatVrsRelation.CO_ENCODES, - } # members = defining cdna + gen partner + protein apex + the two cousins (full closure). assert len(transit.categorical_variant.members) == 5 # The distinct relation kinds surface on the constraint, including co_encodes. @@ -183,10 +209,7 @@ def test_mode_1_projection_narrow_object_drops_sibling_encoders(): assert transit is not None # Only the measured change's coordinate partner + the protein consequence; the cousins are excluded. - assert transit.member_relations == { - "gen": CatVrsRelation.COORDINATE_REPRESENTATION_OF, - "prot": CatVrsRelation.TRANSLATION_OF, - } + assert _member_digests(links, include_convergent=False) == {"cdna", "gen", "prot"} # members = defining cdna + gen partner + protein apex (the two cousins are dropped). assert len(transit.categorical_variant.members) == 3 @@ -207,13 +230,8 @@ def test_mode_2_reverse_translation_keeps_the_full_encoder_class(): assert transit is not None assert transit.mode == CatVrsMode.REVERSE_TRANSLATION - # All four nt encoders `encodes` the defining protein; none dropped. - assert transit.member_relations == { - "cdna_a": CatVrsRelation.ENCODES, - "gen_a": CatVrsRelation.ENCODES, - "cdna_b": CatVrsRelation.ENCODES, - "gen_b": CatVrsRelation.ENCODES, - } + # All four nt encoders stay, even in the narrow object; none dropped. + assert _member_digests(links, include_convergent=False) == {"prot", "cdna_a", "gen_a", "cdna_b", "gen_b"} assert len(transit.categorical_variant.members) == 5 @@ -267,8 +285,7 @@ def test_unhydratable_member_allele_is_skipped(): transit = build_categorical_variant(links, name="urn:mavedb:test#7") assert transit is not None - # The un-hydratable genomic member is excluded from both the members and the relation map. - assert transit.member_relations == {"good": CatVrsRelation.ENCODES} + # The un-hydratable genomic member is excluded from the members. assert len(transit.categorical_variant.members) == 2 @@ -340,6 +357,36 @@ def test_narrow_object_omits_mappings_for_dropped_convergent_encodings(): assert _mappings(transit) == {(_GNOMAD, "1-100-A-G"): MappingRelation.EXACT_MATCH} +@pytest.mark.unit +def test_unfolded_measured_allele_treats_every_grouped_member_as_convergent(): + """A measured nt allele RT's fold-in missed has no group, so no member can be shown to be its projection. + Every grouped nt member is co_encodes rather than a coordinate representation, its records are + relatedMatch, and the VA subject keeps only the protein consequence.""" + links = [ + _link(level="genomic", digest="gen", is_authoritative=True, caid="CA1"), + _link(level="cdna", digest="twin_cdna", is_authoritative=False, projection_group=0, caid="CA1"), + _link(level="cdna", digest="sibling_cdna", is_authoritative=False, projection_group=1, caid="CA2"), + _link(level="genomic", digest="sibling_gen", is_authoritative=False, projection_group=1, caid="CA2"), + _link(level="protein", digest="prot", is_authoritative=False, caid="PA1"), + ] + references = {"gen": _clinvar("100"), "sibling_gen": _clinvar("200")} + + wide = build_categorical_variant(links, name="urn:mavedb:test#unfolded", cross_references=references) + narrow = build_categorical_variant( + links, name="urn:mavedb:test#unfolded", include_convergent=False, cross_references=references + ) + + assert wide is not None and narrow is not None + assert _mappings(wide) == { + (_CLINGEN, "CA1"): MappingRelation.EXACT_MATCH, + (_CLINVAR, "100"): MappingRelation.EXACT_MATCH, + (_CLINGEN, "CA2"): MappingRelation.RELATED_MATCH, + (_CLINVAR, "200"): MappingRelation.RELATED_MATCH, + (_CLINGEN, "PA1"): MappingRelation.RELATED_MATCH, + } + assert _member_digests(links, include_convergent=False) == {"gen", "prot"} + + @pytest.mark.unit def test_reverse_translation_maps_each_encoding_as_related(): """Protein measured: the protein's own CAID is exact; every nt encoding is a distinct variant.""" diff --git a/tests/lib/test_variant_detail.py b/tests/lib/test_variant_detail.py index 2c9970c07..60151cb84 100644 --- a/tests/lib/test_variant_detail.py +++ b/tests/lib/test_variant_detail.py @@ -16,7 +16,10 @@ pytest.importorskip("psycopg2") -from mavedb.lib.variant_detail import get_variant_detail +from mavedb.lib.annotation.context import load_annotation_context_inputs, variant_annotation_context +from mavedb.lib.allele_detail import get_allele_detail +from mavedb.lib.allele_identity import AlleleDerivation +from mavedb.lib.variant_detail import get_variant_detail, load_variant_detail_inputs from mavedb.lib.mondo import get_generic_disease_term from mavedb.models.allele import Allele from mavedb.models.clinical_control import ClinvarControl @@ -31,6 +34,7 @@ from mavedb.models.variant import Variant from mavedb.models.vep_allele_consequence import VepAlleleConsequence from tests.helpers.constants import TEST_MINIMAL_VARIANT, TEST_USER +from tests.helpers.util.query_plan import captured_statements # A spec-valid post_mapped VRS Allele — the Cat-VRS builder hydrates this, so it must parse. _VALID_DIGEST = "0123456789abcdefghijABCDEFGHIJ_-" @@ -457,3 +461,94 @@ def test_superseded_variant_self_describes(session, setup_lib_db_with_score_set) assert detail.is_current is False assert detail.superseded_by_score_set == "urn:mavedb:00000001-a-2" + + +def _seed_mapped_variants(session, score_set, count): + """``count`` coding-measured variants, each with a genomic projection, a protein apex shared by all of + them, VEP on the measured allele, and membership in one primary calibration's classification.""" + protein = _allele(session, "shared-prot", level="protein", hgvs_p="NP_000537.3:p.Ala406Thr") + variants = [] + for index in range(count): + variant = _variant(session, score_set, index + 1, data={"score_data": {"score": float(index)}}) + record = _record(session, variant, hgvs_assay_level=f"NM_000546.6:c.{1216 + index}G>A") + measured = _allele(session, f"cdna-{index}", level="cdna", hgvs_c=f"NM_000546.6:c.{1216 + index}G>A") + genomic = _allele(session, f"gen-{index}", level="genomic", hgvs_g=f"NC_000017.11:g.{7676154 + index}C>T") + _link(session, record, measured, is_authoritative=True, projection_group=0) + _link(session, record, genomic, projection_group=0) + _link(session, record, protein) + _vep(session, measured, "missense_variant") + variants.append(variant) + _calibration(session, score_set, primary=True, classifications=[("abnormal", variants)]) + return variants + + +@pytest.mark.integration +def test_batched_details_equal_single_variant_details(session, setup_lib_db_with_score_set): + """A chunk's envelopes are identical to building each variant on its own, unmapped variants included.""" + score_set = setup_lib_db_with_score_set + variants = [*_seed_mapped_variants(session, score_set, 3), _variant(session, score_set, 99)] + + inputs = load_variant_detail_inputs(session, variants) + + for variant in variants: + assert inputs.detail_for(variant) == get_variant_detail(session, variant) + + +@pytest.mark.integration +def test_batched_contexts_equal_single_variant_contexts(session, setup_lib_db_with_score_set): + """A chunk's annotation contexts are identical to building each variant on its own.""" + score_set = setup_lib_db_with_score_set + variants = [*_seed_mapped_variants(session, score_set, 3), _variant(session, score_set, 99)] + + inputs = load_annotation_context_inputs(session, variants) + + for variant in variants: + assert inputs.context_for(variant) == variant_annotation_context(session, variant) + + +@pytest.mark.integration +@pytest.mark.parametrize( + "load", [load_variant_detail_inputs, load_annotation_context_inputs], ids=["detail", "annotation_context"] +) +def test_batched_loads_run_a_fixed_number_of_queries(session, setup_lib_db_with_score_set, load): + """Loading a chunk costs the same number of queries however many variants it holds.""" + score_set = setup_lib_db_with_score_set + variants = _seed_mapped_variants(session, score_set, 6) + # Seeding commits, which expires the variants; reload them so only the loader's own queries are counted. + for variant in variants: + session.refresh(variant) + + with captured_statements(session) as one: + load(session, variants[:1]) + with captured_statements(session) as six: + load(session, variants) + + assert len(six) == len(one) + + +@pytest.mark.integration +@pytest.mark.parametrize("measured_group", [0, None], ids=["folded", "fold_in_missed"]) +def test_variant_and_allele_detail_label_members_alike(session, setup_lib_db_with_score_set, measured_group): + """Both views label each member relative to the measured allele through one function, so they agree, + including when the measured allele has no projection group and an ungrouped member sits beside it.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1) + record = _record(session, variant) + measured = _allele(session, "measured", level="cdna", hgvs_c="NM_000546.6:c.1216G>A") + twin = _allele(session, "twin", level="genomic", hgvs_g="NC_000017.11:g.7676154C>T") + ungrouped = _allele(session, "ungrouped", level="genomic", hgvs_g="NC_000017.11:g.7676160C>T") + convergent = _allele(session, "convergent", level="cdna", hgvs_c="NM_000546.6:c.1218C>T") + protein = _allele(session, "protein", level="protein", hgvs_p="NP_000537.3:p.Ala406Thr") + _link(session, record, measured, is_authoritative=True, projection_group=measured_group) + _link(session, record, twin, projection_group=0) + _link(session, record, ungrouped) + _link(session, record, convergent, projection_group=1) + _link(session, record, protein) + + variant_alleles = get_variant_detail(session, variant).alleles + allele_alleles = get_allele_detail(session, measured, focus_digests={"measured"}).alleles + + labels = {digest: (identity.relation, identity.derivation) for digest, identity in variant_alleles.items()} + assert labels == {digest: (identity.relation, identity.derivation) for digest, identity in allele_alleles.items()} + assert labels["ungrouped"] == ("co_encodes", AlleleDerivation.CONVERGENT) + assert labels["twin"][0] == ("coordinate_representation_of" if measured_group == 0 else "co_encodes") diff --git a/tests/routers/test_query_scaling.py b/tests/routers/test_query_scaling.py new file mode 100644 index 000000000..9c31b9f17 --- /dev/null +++ b/tests/routers/test_query_scaling.py @@ -0,0 +1,256 @@ +# ruff: noqa: E402 +"""Query-count canary for every GET route with path parameters. + +Each route is called against two seeded score sets, one small and one four times larger, and must run no +more queries against the larger. A route whose count grows with its data is an N+1, which is invisible at +fixture sizes until it reaches a score set in production. + +Routes are found in the OpenAPI schema, so a new route is covered without a test of its own. Path +parameters are resolved by ``(resource segment, parameter)``: the segment before the parameter in the +path, since ``{urn}`` alone names a dozen different resources. A route under a resource the map doesn't +know fails, naming the pair to add to ``_RESOLVERS``, or a reason to add to ``_EXEMPT``. +""" + +import base64 +import hashlib +import re +from dataclasses import dataclass +from unittest.mock import patch +from urllib.parse import quote + +import pytest + +arq = pytest.importorskip("arq") +cdot = pytest.importorskip("cdot") +fastapi = pytest.importorskip("fastapi") + +from sqlalchemy import select + +from mavedb.models.score_set import ScoreSet as ScoreSetDbModel +from mavedb.models.variant import Variant as VariantDbModel +from mavedb.server_main import app +from tests.helpers.constants import ( + TEST_BIORXIV_IDENTIFIER, + TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED, + TEST_PUBMED_IDENTIFIER, + TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X, +) +from tests.helpers.util.annotation import AlleleSpec, seed_mapping_record +from tests.helpers.util.common import deepcamelize +from tests.helpers.util.experiment import create_experiment +from tests.helpers.util.query_plan import captured_statements +from tests.helpers.util.score_calibration import create_publish_and_promote_score_calibration +from tests.helpers.util.score_set import create_seq_score_set, publish_score_set +from tests.helpers.util.variant import mock_worker_variant_insertion + +# Resource segments whose routes don't read the score set's variant data, or reach a service the test +# environment doesn't run. Each needs a reason. +_EXEMPT = { + "hgvs": "resolves through cdot, an external service", + "refget": "reads SeqRepo, not the variant data", + "seqrepo": "reads SeqRepo, not the variant data", + "orcid": "calls ORCID", + "publication-identifiers": "calls the publication registries", + "genes": "reads the gene normalizer, not the variant data", + "taxonomies": "reference data, not variant data", + "licenses": "reference data, not variant data", + "target-genes": "one target gene; not variant data", + "controlled-keywords": "reference data, not variant data", + "users": "one user; not variant data", + "collections": "one collection's metadata; not variant data", + "statistics": "site-wide counts, tracked in api#900", + "user-is-permitted": "one permission check; not variant data", + "job-runs": "one job run; not variant data", + "pipelines": "one pipeline; not variant data", +} + + +@dataclass(frozen=True) +class _World: + """The identifiers of one seeded score set and the records hanging off it.""" + + experiment_set_urn: str + experiment_urn: str + score_set_urn: str + variant_urn: str + measured_digest: str + protein_clingen_allele_id: str + calibration_urn: str + classification_id: int + + +_RESOLVERS = { + ("experiment-sets", "urn"): lambda world: world.experiment_set_urn, + ("experiments", "urn"): lambda world: world.experiment_urn, + ("score-sets", "urn"): lambda world: world.score_set_urn, + ("score-set", "score_set_urn"): lambda world: world.score_set_urn, + ("variants", "urn"): lambda world: world.variant_urn, + ("mapped-variants", "urn"): lambda world: world.variant_urn, + ("vrs", "identifier"): lambda world: world.measured_digest, + ("alleles", "identifier"): lambda world: world.measured_digest, + # The protein consequence every variant in the world shares, so its measurements grow with the world. + ("clingen-alleles", "clingen_allele_id"): lambda world: world.protein_clingen_allele_id, + ("score-calibrations", "urn"): lambda world: world.calibration_urn, + ("functional-classifications", "classification_id"): lambda world: world.classification_id, +} + +# Routes that answer every request with a fixed status by design, so there is nothing to scale. +_RETIRED = { + "/api/v1/score-sets/{urn}/mapped-variants", +} + +_PARAMETER = re.compile(r"{([^}]+)}") + +_PUBLICATION_FETCH = [ + {"dbName": "PubMed", "identifier": TEST_PUBMED_IDENTIFIER}, + {"dbName": "bioRxiv", "identifier": TEST_BIORXIV_IDENTIFIER}, +] + + +def _parameterized_get_routes() -> list[str]: + return sorted( + path + for path, operations in app.openapi()["paths"].items() + if "get" in operations and _PARAMETER.search(path) and path not in _RETIRED + ) + + +def _resolve(path: str, world: _World) -> str: + """The concrete URL for ``path`` in ``world``. Raises ``KeyError`` naming the unresolved pair.""" + segments = path.split("/") + + def substitute(match: re.Match) -> str: + parameter = match.group(1) + index = segments.index(match.group(0)) + resolver = _RESOLVERS[(segments[index - 1], parameter)] + return quote(str(resolver(world)), safe=":") + + return _PARAMETER.sub(substitute, path) + + +def _exemption(path: str) -> str | None: + return next((reason for segment, reason in _EXEMPT.items() if f"/{segment}/" in path), None) + + +def _digest(text: str) -> str: + """A well-formed GA4GH VRS identifier, unique to ``text``.""" + return "ga4gh:VA." + base64.urlsafe_b64encode(hashlib.sha512(text.encode()).digest()[:24]).decode() + + +def _seed_world(client, session, data_provider, scores_csv, label: str) -> _World: + """A published score set whose variants each carry a coding measurement with VEP, gnomAD and ClinVar + records, its genomic projection, a convergent encoding and a protein consequence they all share, under a + published primary calibration that classifies every variant.""" + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + mock_worker_variant_insertion(client, session, data_provider, score_set, scores_csv) + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + score_set = publish_score_set(client, score_set["urn"]) + calibration = create_publish_and_promote_score_calibration( + client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) + ) + + variants = session.scalars( + select(VariantDbModel) + .join(ScoreSetDbModel) + .where(ScoreSetDbModel.urn == score_set["urn"]) + .order_by(VariantDbModel.id) + ).all() + vrs = TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X + for index, variant in enumerate(variants): + seed_mapping_record( + session, + variant, + hgvs_assay_level=f"NM_000546.6:c.{1000 + index}G>A", + alleles=[ + AlleleSpec( + digest=_digest(f"{label}-measured-{index}"), + is_authoritative=True, + clingen_allele_id=f"CA{label}{index}", + projection_group=0, + vep_consequence="missense_variant", + clinvar_control_ids=(1,), + gnomad_variant_ids=(1,), + post_mapped=vrs, + ), + AlleleSpec( + digest=_digest(f"{label}-genomic-{index}"), level="genomic", projection_group=0, post_mapped=vrs + ), + AlleleSpec(digest=_digest(f"{label}-convergent-{index}"), projection_group=1, post_mapped=vrs), + AlleleSpec( + digest=_digest(f"{label}-protein"), level="protein", clingen_allele_id=f"PA{label}", post_mapped=vrs + ), + ], + ) + + return _World( + experiment_set_urn=experiment["experimentSetUrn"], + experiment_urn=experiment["urn"], + score_set_urn=score_set["urn"], + variant_urn=variants[0].urn, + measured_digest=_digest(f"{label}-measured-0"), + protein_clingen_allele_id=f"PA{label}", + calibration_urn=calibration["urn"], + classification_id=calibration["functionalClassifications"][0]["id"], + ) + + +def _scores_csv(tmp_path, count: int): + """A scores file of ``count`` single-base substitutions against the minimal target (``ACGTTT``), scored + in the calibration's abnormal range so each variant carries pathogenicity evidence.""" + reference = "ACGTTT" + substitutions = [(position, base, alt) for position, base in enumerate(reference) for alt in "ACGT" if alt != base][ + :count + ] + rows = [ + f"n.{position + 1}{base}>{alt},{-2.0 - index / 10}" for index, (position, base, alt) in enumerate(substitutions) + ] + path = tmp_path / f"scores_{count}.csv" + path.write_text("hgvs_nt,score\n" + "\n".join(rows) + "\n") + return path + + +@pytest.fixture +def worlds(client, session, data_provider, setup_router_db, mock_publication_fetch, tmp_path): + small = _seed_world(client, session, data_provider, _scores_csv(tmp_path, 3), "small") + large = _seed_world(client, session, data_provider, _scores_csv(tmp_path, 12), "large") + return small, large + + +def _query_count(client, session, url: str) -> tuple[int, int]: + with captured_statements(session) as statements: + response = client.get(url) + return response.status_code, len(statements) + + +def test_every_parameterized_get_route_is_resolved_or_exempt(): + """A new route under an unknown resource needs a resolver or an exemption reason, not a silent skip.""" + unresolved = set() + for path in _parameterized_get_routes(): + if _exemption(path) is not None: + continue + segments = path.split("/") + for match in _PARAMETER.finditer(path): + key = (segments[segments.index(match.group(0)) - 1], match.group(1)) + if key not in _RESOLVERS: + unresolved.add((path, key)) + + assert not unresolved, f"Add a resolver to _RESOLVERS or a reason to _EXEMPT for: {sorted(unresolved)}" + + +@pytest.mark.parametrize("mock_publication_fetch", [_PUBLICATION_FETCH], indirect=True) +def test_no_route_runs_more_queries_on_a_larger_score_set(client, session, worlds): + small, large = worlds + failures = [] + for path in _parameterized_get_routes(): + if _exemption(path) is not None: + continue + + small_status, small_count = _query_count(client, session, _resolve(path, small)) + large_status, large_count = _query_count(client, session, _resolve(path, large)) + if small_status >= 400 or large_status >= 400: + failures.append(f"{path}: status {small_status}/{large_status}, so its query count isn't comparable") + elif large_count > small_count: + failures.append(f"{path}: {small_count} queries on the small score set, {large_count} on the large one") + + assert not failures, "\n".join(failures) diff --git a/tests/routers/test_score_set.py b/tests/routers/test_score_set.py index 9f1710587..b29d2ef4f 100644 --- a/tests/routers/test_score_set.py +++ b/tests/routers/test_score_set.py @@ -8,7 +8,7 @@ from io import StringIO import threading from types import SimpleNamespace -from unittest.mock import MagicMock, patch +from unittest.mock import patch import jsonschema import pytest @@ -5859,13 +5859,27 @@ def stream_variant(): def _record_for(variant, annotation_function, context=_STUB_CONTEXT): - """Drive one stream record with the context factory stubbed out. + """Drive one stream record with the chunk's loaded inputs stubbed out. - ``context`` is what the factory returns; pass None for the variant whose mapping substrate yields - nothing to annotate. + ``context`` is what the inputs build for the variant; pass None for the variant whose mapping substrate + yields nothing to annotate. """ - with patch("mavedb.routers.score_sets.variant_annotation_context", return_value=context): - return _annotation_stream_record(MagicMock(), variant, annotation_function) + return _annotation_stream_record(SimpleNamespace(context_for=lambda _: context), variant, annotation_function) + + +def test_annotation_stream_record_reports_a_context_build_failure_as_an_error(stream_variant): + """Contexts are built per variant from a chunk's loaded inputs, so one variant failing to build stays + that variant's error record rather than ending the stream.""" + + def _broken(_): + raise ValueError("Unsupported VRS state type") + + record, outcome = _annotation_stream_record( + SimpleNamespace(context_for=_broken), stream_variant, lambda ctx: _StubAnnotation() + ) + + assert outcome == "errored" + assert record["error"] == {"type": "ValueError", "detail": "Unsupported VRS state type"} def test_annotation_stream_record_serializes_a_successful_annotation(stream_variant): From ecbce89b806177581a8f6228f3ca71e19275cc18 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Fri, 9 Oct 2026 10:24:20 -0700 Subject: [PATCH 233/252] fix(clingen): restart cache expiry when warming reads a cached allele A cache hit kept the expiry its first writer set, so data a sibling score set cached nearly a day earlier could expire minutes after the warming job read it, leaving the downstream jobs a cold cache. The warming job now restarts each allele's expiry after fetching it. --- src/mavedb/lib/clingen/allele_registry.py | 17 +++++++++++ .../jobs/external_services/clingen_cache.py | 6 ++-- tests/lib/clingen/test_allele_registry.py | 28 +++++++++++++++++++ .../external_services/test_clingen_cache.py | 17 ++++++++--- 4 files changed, 62 insertions(+), 6 deletions(-) diff --git a/src/mavedb/lib/clingen/allele_registry.py b/src/mavedb/lib/clingen/allele_registry.py index 62e3bf9bf..d9e121bcb 100644 --- a/src/mavedb/lib/clingen/allele_registry.py +++ b/src/mavedb/lib/clingen/allele_registry.py @@ -49,6 +49,23 @@ async def get_clingen_allele_data(clingen_allele_id: str) -> Optional[dict]: return response.json() +async def refresh_clingen_allele_data_expiry(clingen_allele_id: str) -> bool: + """Restart the cache expiry of an allele's ClinGen data. + + A cache hit leaves the expiry set when the entry was written. Data a sibling score set cached + nearly a day earlier can therefore expire minutes after a warming step reads it, and the jobs + the warming step exists for find the cache cold. + + Args: + clingen_allele_id: ClinGen allele ID whose cached data to keep (e.g., CA123456). + + Returns: + True if the entry was cached and its expiry restarted, False if it was not cached. + """ + key = clingen_cache_key_builder(get_clingen_allele_data, clingen_allele_id) + return await get_clingen_allele_data.cache.expire(key, CACHE_TTL_SECONDS) # type: ignore[attr-defined] + + async def get_canonical_pa_ids(clingen_allele_id: str) -> list[str]: """Retrieve canonical PA IDs from the ClinGen API for a given ClinGen allele ID. diff --git a/src/mavedb/worker/jobs/external_services/clingen_cache.py b/src/mavedb/worker/jobs/external_services/clingen_cache.py index d530d881f..11545ec05 100644 --- a/src/mavedb/worker/jobs/external_services/clingen_cache.py +++ b/src/mavedb/worker/jobs/external_services/clingen_cache.py @@ -14,7 +14,7 @@ from sqlalchemy import select -from mavedb.lib.clingen.allele_registry import get_clingen_allele_data +from mavedb.lib.clingen.allele_registry import get_clingen_allele_data, refresh_clingen_allele_data_expiry from mavedb.lib.clingen.alleles import get_alleles_for_score_set from mavedb.lib.clingen.constants import CLINGEN_CACHE_WARMING_CONCURRENCY from mavedb.lib.types.workflow import JobExecutionOutcome @@ -74,13 +74,15 @@ async def warm_clingen_cache(ctx: dict, job_id: int, job_manager: JobManager) -> return JobExecutionOutcome.succeeded(data={"warmed": 0, "failed": 0}) # Fetch alleles concurrently up to CLINGEN_CACHE_WARMING_CONCURRENCY in-flight at a time. - # get_clingen_allele_data() is decorated with @cached, so each call populates Redis. + # get_clingen_allele_data() is decorated with @cached, so each call populates Redis. A hit keeps + # the expiry its first writer set, so restart it: the downstream jobs need it warm, not just present. semaphore = asyncio.Semaphore(CLINGEN_CACHE_WARMING_CONCURRENCY) async def fetch_one(allele_id: str) -> tuple[str, bool, BaseException | None]: async with semaphore: try: await get_clingen_allele_data(allele_id) + await refresh_clingen_allele_data_expiry(allele_id) return allele_id, True, None except Exception as exc: return allele_id, False, exc diff --git a/tests/lib/clingen/test_allele_registry.py b/tests/lib/clingen/test_allele_registry.py index 68cbf4327..c143dbbc1 100644 --- a/tests/lib/clingen/test_allele_registry.py +++ b/tests/lib/clingen/test_allele_registry.py @@ -14,7 +14,9 @@ get_canonical_pa_ids, get_clingen_allele_data, get_matching_registered_ca_ids, + refresh_clingen_allele_data_expiry, ) +from mavedb.lib.clingen.cache import CACHE_TTL_SECONDS @pytest.mark.unit @@ -455,6 +457,32 @@ async def test_different_functions_share_raw_data_cache(self, mock_request, clea assert mock_request.call_count == 1 # Only one API call for both functions +@pytest.mark.unit +@mock.patch("mavedb.lib.clingen.allele_registry.requests.get") +class TestRefreshClingenAlleleDataExpiry: + @pytest.mark.asyncio + async def test_restarts_the_expiry_of_cached_data(self, mock_request, clear_cache): + mock_response = mock.Mock() + mock_response.status_code = 200 + mock_response.json.return_value = {"@id": "CA_TTL"} + mock_request.return_value = mock_response + await get_clingen_allele_data("CA_TTL") + + with mock.patch.object( + get_clingen_allele_data.cache, # type: ignore[attr-defined] + "expire", + wraps=get_clingen_allele_data.cache.expire, # type: ignore[attr-defined] + ) as expire: + assert await refresh_clingen_allele_data_expiry("CA_TTL") is True + + expire.assert_awaited_once_with("v1:get_clingen_allele_data:CA_TTL", CACHE_TTL_SECONDS) + + @pytest.mark.asyncio + async def test_reports_uncached_data(self, mock_request, clear_cache): + assert await refresh_clingen_allele_data_expiry("CA_NOT_CACHED") is False + mock_request.assert_not_called() + + @pytest.mark.unit @mock.patch("mavedb.lib.clingen.allele_registry.requests.get") class TestCacheBackendFailure: diff --git a/tests/worker/jobs/external_services/test_clingen_cache.py b/tests/worker/jobs/external_services/test_clingen_cache.py index 6bff657b9..aa744e9c8 100644 --- a/tests/worker/jobs/external_services/test_clingen_cache.py +++ b/tests/worker/jobs/external_services/test_clingen_cache.py @@ -112,10 +112,17 @@ async def test_warms_cache_for_variants_with_caids( session.commit() mock_get_allele_data = AsyncMock(return_value={"some": "data"}) - - with patch( - "mavedb.worker.jobs.external_services.clingen_cache.get_clingen_allele_data", - mock_get_allele_data, + mock_refresh_expiry = AsyncMock(return_value=True) + + with ( + patch( + "mavedb.worker.jobs.external_services.clingen_cache.get_clingen_allele_data", + mock_get_allele_data, + ), + patch( + "mavedb.worker.jobs.external_services.clingen_cache.refresh_clingen_allele_data_expiry", + mock_refresh_expiry, + ), ): result = await warm_clingen_cache( mock_worker_ctx, @@ -124,6 +131,8 @@ async def test_warms_cache_for_variants_with_caids( ) assert result.status == JobStatus.SUCCEEDED + # A hit keeps its first writer's expiry, so every warmed entry's expiry is restarted. + assert {call.args[0] for call in mock_refresh_expiry.call_args_list} == {"CA111111", "CA222222"} # Should be called exactly 2 times (CA111111 and CA222222, deduplicated) assert mock_get_allele_data.call_count == 2 called_ids = {call.args[0] for call in mock_get_allele_data.call_args_list} From 5a30c3287b9b0570ed5d9adfcfe8d196bedb4b35 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Fri, 9 Oct 2026 10:24:20 -0700 Subject: [PATCH 234/252] fix(clinvar): report progress within a version during a refresh The first version of a refresh can make thousands of uncached ClinGen lookups without updating progress, so the stall reaper cancelled it at 30 minutes. Progress is now reported every 500 alleles. --- .../worker/jobs/external_services/clinvar.py | 13 ++++++- .../jobs/external_services/test_clinvar.py | 34 +++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/src/mavedb/worker/jobs/external_services/clinvar.py b/src/mavedb/worker/jobs/external_services/clinvar.py index d2482c6db..5aadfd8ca 100644 --- a/src/mavedb/worker/jobs/external_services/clinvar.py +++ b/src/mavedb/worker/jobs/external_services/clinvar.py @@ -38,6 +38,9 @@ logger = logging.getLogger(__name__) +# Alleles between progress checkpoints within a version. The first version of a run can fetch +# thousands of uncached ClinGen lookups; without checkpoints the stall reaper cancels it at 30 min. +CLINVAR_PROGRESS_EVERY_ALLELES = 500 # ClinVar archived data starts from February 2015, then January of each # subsequent year. This list is used to generate the date range for refreshing. CLINVAR_START_YEAR = 2015 @@ -208,7 +211,15 @@ def alleles_linked_at_version(clinvar_version: str) -> set[int]: annotation_manager = AnnotationStatusManager( job_manager.db, job_run_id=job_manager.job_id, score_set_id=score_set.id ) - for allele_id, caid in allele_data.items(): + for allele_index, (allele_id, caid) in enumerate(allele_data.items(), start=1): + if allele_index % CLINVAR_PROGRESS_EVERY_ALLELES == 0: + job_manager.update_progress( + int((version_index + allele_index / len(allele_data)) / len(versions) * 100), + 100, + f"Processing ClinVar version {clinvar_version} ({version_index + 1}/{len(versions)}): " + f"{allele_index}/{len(allele_data)} alleles.", + ) + # ClinVar is a nucleotide-level database, skip any protein level alleles. if allele_levels.get(allele_id) == SequenceLevel.protein.value: annotation_counts["skipped_link_count"] += 1 diff --git a/tests/worker/jobs/external_services/test_clinvar.py b/tests/worker/jobs/external_services/test_clinvar.py index f7698d737..caa7e3f1f 100644 --- a/tests/worker/jobs/external_services/test_clinvar.py +++ b/tests/worker/jobs/external_services/test_clinvar.py @@ -394,6 +394,40 @@ async def resolve(caid): assert by_allele[authoritative_allele.id].disposition == Disposition.ABSENT assert by_allele[authoritative_allele.id].reason == EventReason.NO_RECORD + async def test_checkpoints_progress_within_a_version( + self, + mock_worker_ctx, + session, + with_refresh_clinvar_controls_job, + sample_refresh_clinvar_controls_job_run, + setup_rt_derived_allele_with_caid, + ): + """A version with many uncached ClinGen lookups keeps the stall reaper's heartbeat fresh.""" + manager = JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id) + + with ( + patch("mavedb.worker.jobs.external_services.clinvar.CLINVAR_PROGRESS_EVERY_ALLELES", 1), + patch( + "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", + return_value=None, + ), + patch( + "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", + return_value=MOCK_CLINVAR_DATA, + ), + patch.object(manager, "update_progress", wraps=manager.update_progress) as update_progress, + ): + result = await refresh_clinvar_controls( + mock_worker_ctx, sample_refresh_clinvar_controls_job_run.id, manager + ) + + assert result.status == JobStatus.SUCCEEDED + within_version = [call.args[2] for call in update_progress.call_args_list if "alleles." in call.args[2]] + assert within_version == [ + "Processing ClinVar version 01_2026 (1/1): 1/2 alleles.", + "Processing ClinVar version 01_2026 (1/1): 2/2 alleles.", + ] + async def test_idempotent_rerun_skips_and_does_not_duplicate( self, mock_worker_ctx, From 30cb4cfc828d83d0991c6c677cdd2c41824dc060 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Fri, 9 Oct 2026 10:28:02 -0700 Subject: [PATCH 235/252] chore(deps): pin variant-annotation to v0.1.0-dev.6 --- poetry.lock | 8 ++++---- pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index 3a451d1a7..ae3d6a64e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -4937,7 +4937,7 @@ test = ["aiohttp (>=3.10.5)", "flake8 (>=6.1,<7.0)", "mypy (>=0.800)", "psutil", [[package]] name = "variant-annotation" -version = "0.1.0-dev.5" +version = "0.1.0-dev.6" description = "Genetic variant mapping and annotation pipeline" optional = true python-versions = ">=3.11" @@ -4967,8 +4967,8 @@ tests = ["pytest"] [package.source] type = "git" url = "https://github.com/bbi-lab/variant-annotation.git" -reference = "v0.1.0-dev.5" -resolved_reference = "b0952affeb3677fdc5b927cf8fc46dcfe7d54cdd" +reference = "v0.1.0-dev.6" +resolved_reference = "dee39b5d03a89a2464b02fe42774e173594422cb" [[package]] name = "variant-translation" @@ -5301,4 +5301,4 @@ server = ["aiocache", "alembic", "alembic-utils", "arq", "authlib", "biocommons" [metadata] lock-version = "2.1" python-versions = "^3.11" -content-hash = "ab765d9795413ebcf330c5040e2e8dddd9eb267bcf918c906aa8c9472f6a2fd7" +content-hash = "9b7c249bd890a9e36b278d6f0c1c0441d57acded1d0f4ca30cdfc6df9d0b2a4d" diff --git a/pyproject.toml b/pyproject.toml index f49a1ae46..78029d1f6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,7 +65,7 @@ starlette = { version = "^1.7.0", optional = true } starlette-context = { version = "^0.3.6", optional = true } slack-sdk = { version = "~3.21.3", optional = true } uvicorn = { extras = ["standard"], version = "*", optional = true } -variant-annotation = { git = "https://github.com/bbi-lab/variant-annotation.git", tag = "v0.1.0-dev.5", optional = true } +variant-annotation = { git = "https://github.com/bbi-lab/variant-annotation.git", tag = "v0.1.0-dev.6", optional = true } watchtower = { version = "~3.2.0", optional = true } asyncclick = "^8.3.0.7" filelock = "^3.29.0" From 5e763acfe1fef7b571d7a8fe9f0649e118a648fc Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Sat, 10 Oct 2026 07:13:50 -0700 Subject: [PATCH 236/252] fix(worker): give mapping an hour before its size-aware budget starts The mapper's BLAT limit is now 20 minutes and a failed parse runs BLAT twice, so the 30-minute floor cancelled small score sets before the mapper's own timeout could fire. --- src/mavedb/worker/jobs/variant_processing/mapping.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mavedb/worker/jobs/variant_processing/mapping.py b/src/mavedb/worker/jobs/variant_processing/mapping.py index dec29a56a..9376b59e2 100644 --- a/src/mavedb/worker/jobs/variant_processing/mapping.py +++ b/src/mavedb/worker/jobs/variant_processing/mapping.py @@ -56,7 +56,7 @@ # delegating opaque work; the real fix is for the mapper to report progress (or to chunk the call). # See worker/best_practices.md, "Long external delegations". TODO: replace with real mapping progress. MAP_PROGRESS_KEEPALIVE_SECONDS = 300 # heartbeat cadence; must stay well under PROGRESS_STALL_MINUTES (30m) -MAP_BUDGET_BASE_SECONDS = 30 * 60 # floor: overhead + small sets +MAP_BUDGET_BASE_SECONDS = 60 * 60 # floor: two dcd-mapping BLAT runs (BLAT_TIMEOUT_SECONDS, 20 min each) + overhead MAP_BUDGET_PER_VARIANT_SECONDS = 1.0 # ~3x the observed ~0.3 s/variant map throughput, as slack MAP_BUDGET_MAX_SECONDS = 22 * 60 * 60 # ceiling: stay under the 23.5h stall backstop / 24h ARQ ceiling From 7f7bb6d089595e76c9398b40de849f85805ad8a0 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Sun, 11 Oct 2026 18:24:06 -0700 Subject: [PATCH 237/252] feat(audit): scope audits to current score sets and split codon-straddle findings Add --collection and --translated-since alongside --score-set on both audit scripts, so findings left by an older pipeline drop out. Split measured_allele_outside_projection_group: changes reaching past the codon RT targeted and reference-identical alleles are now info-level checks, since RT can never list them. Correct the malformed-group description: groups of one come from library projection failures, not only failed translations. --- .../scripts/audit_allele_identifiers.py | 56 ++++++- .../scripts/audit_reverse_translation.py | 151 +++++++++++++----- src/mavedb/scripts/audit_scope.py | 85 ++++++++++ .../scripts/test_audit_reverse_translation.py | 66 ++++++++ tests/scripts/test_audit_scope.py | 86 ++++++++++ 5 files changed, 394 insertions(+), 50 deletions(-) create mode 100644 src/mavedb/scripts/audit_scope.py create mode 100644 tests/scripts/test_audit_scope.py diff --git a/src/mavedb/scripts/audit_allele_identifiers.py b/src/mavedb/scripts/audit_allele_identifiers.py index cbd0292e7..8c9c84042 100644 --- a/src/mavedb/scripts/audit_allele_identifiers.py +++ b/src/mavedb/scripts/audit_allele_identifiers.py @@ -3,10 +3,14 @@ Usage: ``` -python3 -m mavedb.scripts.audit_allele_identifiers audit -python3 -m mavedb.scripts.audit_allele_identifiers audit-locations +python3 -m mavedb.scripts.audit_allele_identifiers audit [SCOPE] +python3 -m mavedb.scripts.audit_allele_identifiers audit-pre-mapped [SCOPE] +python3 -m mavedb.scripts.audit_allele_identifiers audit-locations [SCOPE] ``` +SCOPE restricts each command to alleles and mapping records of some score sets; the options are the ones in +`mavedb.scripts.audit_scope`. Unscoped, each command covers the whole database. + Every row in `alleles` asserts that `vrs_digest` is the GA4GH computed identifier of `post_mapped`. Nothing enforced that until the mapping ingest path began recomputing identity on arrival, so rows written earlier can carry an identifier minted before their own normalization. `audit` reports them, and `--repair` corrects the ones that can be corrected safely. @@ -38,7 +42,8 @@ import logging from collections import Counter -from typing import Any, Optional +from datetime import datetime +from typing import Any, Optional, Sequence from ga4gh.core import ga4gh_identify from ga4gh.vrs.models import Allele as VrsAllele @@ -52,6 +57,8 @@ from mavedb.lib.vrs_utils import identify_variation from mavedb.models.allele import Allele from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.scripts.audit_scope import resolve_scope, scope_options from mavedb.scripts.environment import script_environment, with_database_session logger = logging.getLogger(__name__) @@ -78,13 +85,30 @@ def recomputed_identifier(post_mapped: dict[str, Any]) -> Optional[str]: return None +def _alleles_in_scope(score_set_ids: Optional[Sequence[int]]): + """Alleles with a ``post_mapped`` body, restricted to ones linked from the given score sets.""" + query = select(Allele).where(Allele.post_mapped.isnot(None)).order_by(Allele.id) + if score_set_ids is not None: + linked = select(MappingRecordAllele.allele_id).where(MappingRecordAllele.score_set_id.in_(score_set_ids)) + query = query.where(Allele.id.in_(linked)) + return query + + @script_environment.command() +@scope_options @click.option("--limit", type=int, default=None, help="Examine at most this many alleles.") @click.option("--repair", is_flag=True, help="Rewrite vrs_digest where the corrected identifier is unused.") @with_database_session -def audit(db: Session, limit: Optional[int], repair: bool) -> None: +def audit( + db: Session, + urns: tuple[str, ...], + collection: Optional[str], + translated_since: Optional[datetime], + limit: Optional[int], + repair: bool, +) -> None: """Report — and with ``--repair``, rename — alleles whose identifier disagrees with their content.""" - query = select(Allele).where(Allele.post_mapped.isnot(None)).order_by(Allele.id) + query = _alleles_in_scope(resolve_scope(db, urns, collection, translated_since)) if limit is not None: query = query.limit(limit) @@ -148,9 +172,16 @@ def _rename(db: Session, renames: list[tuple["Allele", str]]) -> None: @script_environment.command() +@scope_options @click.option("--limit", type=int, default=None, help="Examine at most this many mapping records.") @with_database_session -def audit_pre_mapped(db: Session, limit: Optional[int]) -> None: +def audit_pre_mapped( + db: Session, + urns: tuple[str, ...], + collection: Optional[str], + translated_since: Optional[datetime], + limit: Optional[int], +) -> None: """Report mapping records whose ``vrs_digest`` disagrees with their ``pre_mapped`` content. Read-only, and no repair: ``mapping_records.vrs_digest`` is indexed but not unique, so a wrong value @@ -159,6 +190,8 @@ def audit_pre_mapped(db: Session, limit: Optional[int]) -> None: normalization moved a del/dup span. The pre-mapped side had never been measured before 2026-08-25. """ query = select(MappingRecord).where(MappingRecord.pre_mapped.isnot(None)).order_by(MappingRecord.id) + if (score_set_ids := resolve_scope(db, urns, collection, translated_since)) is not None: + query = query.where(MappingRecord.score_set_id.in_(score_set_ids)) if limit is not None: query = query.limit(limit) @@ -191,9 +224,16 @@ def audit_pre_mapped(db: Session, limit: Optional[int]) -> None: @script_environment.command() +@scope_options @click.option("--limit", type=int, default=None, help="Examine at most this many alleles.") @with_database_session -def audit_locations(db: Session, limit: Optional[int]) -> None: +def audit_locations( + db: Session, + urns: tuple[str, ...], + collection: Optional[str], + translated_since: Optional[datetime], + limit: Optional[int], +) -> None: """Report nested ``SequenceLocation`` identifiers that disagree with their own coordinates. Separate from :func:`audit` because it is a separate defect with a separate cause. ``audit`` checks @@ -205,7 +245,7 @@ def audit_locations(db: Session, limit: Optional[int]) -> None: Read-only. A drifted location is repaired by re-mapping the score set, since both writers now restamp the location — see ``lib/vrs_utils.py::identify_allele``. """ - query = select(Allele).where(Allele.post_mapped.isnot(None)).order_by(Allele.id) + query = _alleles_in_scope(resolve_scope(db, urns, collection, translated_since)) if limit is not None: query = query.limit(limit) diff --git a/src/mavedb/scripts/audit_reverse_translation.py b/src/mavedb/scripts/audit_reverse_translation.py index 7a0132aa7..1b400dcf6 100644 --- a/src/mavedb/scripts/audit_reverse_translation.py +++ b/src/mavedb/scripts/audit_reverse_translation.py @@ -3,11 +3,15 @@ Usage: ``` -python3 -m mavedb.scripts.audit_reverse_translation invariants [--score-set URN ...] [--skip-seqrepo] -python3 -m mavedb.scripts.audit_reverse_translation outcomes [--score-set URN ...] [--max-failed-pct 5] -python3 -m mavedb.scripts.audit_reverse_translation round-trip [--score-set URN ...] [--per-score-set 50] +python3 -m mavedb.scripts.audit_reverse_translation invariants [SCOPE] [--skip-seqrepo] +python3 -m mavedb.scripts.audit_reverse_translation outcomes [SCOPE] [--max-failed-pct 5] +python3 -m mavedb.scripts.audit_reverse_translation round-trip [SCOPE] [--per-score-set 50] ``` +SCOPE is any of `--score-set URN` (repeatable), `--collection URN` and `--translated-since YYYY-MM-DD`, which +intersect. `--translated-since` keeps only score sets every variant of which RT reached on or after the date, so +findings left by an older pipeline drop out. Unscoped, every check covers the whole database. + Each check guards a property the serving layer assumes but no constraint enforces. The class of bug this exists for is the NM_007294.3 refget divergence: the mapper and RT built the same variant on two sequences, both digests were internally consistent, every existing check passed, and convergent alleles @@ -32,16 +36,17 @@ import sys from collections import Counter, defaultdict from dataclasses import dataclass, field +from datetime import datetime from typing import Any, Callable, Optional, Sequence import asyncclick as click -from sqlalchemy import select, text +from sqlalchemy import text from sqlalchemy.orm import Session from mavedb.lib.hgvs import strip_protein_prediction_parens from mavedb.lib.seqrepo import AmbiguousSequenceError, SequenceNotFoundError, resolve_refget from mavedb.lib.vrs_utils import CANONICAL_ACCESSION_PREFIXES, identify_variation, translate_hgvs_to_variation -from mavedb.models.score_set import ScoreSet +from mavedb.scripts.audit_scope import resolve_scope, scope_options from mavedb.scripts.environment import script_environment, with_database_session logger = logging.getLogger(__name__) @@ -95,20 +100,59 @@ class SqlCheck: false )""" +# The live candidates RT linked to the measured allele's record, on its own level and sequence, span this +# interval. RT's candidates for one protein change all sit inside the codon it changes. +_CANDIDATE_SPAN_SQL = """ + LEFT JOIN LATERAL ( + SELECT min(CASE WHEN jsonb_typeof(da.post_mapped #> '{location,start}') = 'number' + THEN (da.post_mapped #>> '{location,start}')::int END) AS candidate_start, + max(CASE WHEN jsonb_typeof(da.post_mapped #> '{location,end}') = 'number' + THEN (da.post_mapped #>> '{location,end}')::int END) AS candidate_end + FROM mapping_record_alleles d + JOIN alleles da ON da.id = d.allele_id + WHERE d.mapping_record_id = auth.mapping_record_id AND NOT d.is_authoritative AND d.valid_to IS NULL + AND da.level = a.level + AND da.post_mapped #>> '{location,sequenceReference,refgetAccession}' + = a.post_mapped #>> '{location,sequenceReference,refgetAccession}' + ) span ON true +""" + +# A reference-identical allele (g.=, c.=). RT lists the alternative codons for a synonymous change, never the +# reference codon itself, so such an allele cannot fold in. +_REFERENCE_IDENTICAL = "coalesce(a.hgvs_g, a.hgvs_c) LIKE '%='" + +# A measured allele reaching past every candidate's span changes a base outside the codon RT targeted. With no +# candidate on its level and sequence, or an unreadable location, it counts as inside. +_BEYOND_CANDIDATES = """coalesce( + CASE WHEN jsonb_typeof(a.post_mapped #> '{location,start}') = 'number' + AND jsonb_typeof(a.post_mapped #> '{location,end}') = 'number' + THEN (a.post_mapped #>> '{location,start}')::int < span.candidate_start + OR (a.post_mapped #>> '{location,end}')::int > span.candidate_end + END, + false +)""" + _MEASURED_OUTSIDE_GROUP_SQL = """ SELECT auth.mapping_record_id, auth.allele_id, a.level, coalesce(a.hgvs_g, a.hgvs_c) AS hgvs FROM mapping_record_alleles auth JOIN alleles a ON a.id = auth.allele_id + {candidate_span} WHERE auth.is_authoritative AND auth.valid_to IS NULL AND auth.projection_group IS NULL AND a.level IN ('cdna', 'genomic') - AND {length_condition} + AND {condition} {auth_filter} AND EXISTS (SELECT 1 FROM mapping_record_alleles d JOIN alleles da ON da.id = d.allele_id WHERE d.mapping_record_id = auth.mapping_record_id AND NOT d.is_authoritative AND d.valid_to IS NULL AND da.level IN ('cdna', 'genomic')) -""" +""".replace("{candidate_span}", _CANDIDATE_SPAN_SQL) + + +def _measured_outside_group(condition: str) -> str: + """The measured-allele-outside-projection-group query, restricted to alleles matching ``condition``.""" + return _MEASURED_OUTSIDE_GROUP_SQL.replace("{condition}", condition) + SQL_CHECKS: tuple[SqlCheck, ...] = ( SqlCheck( @@ -216,9 +260,31 @@ class SqlCheck: description=( "RT attached derived nucleotide alleles to a record, but the measured nucleotide allele is in no " "projection group: the fold-in missed it, so serving labels every sibling convergent, its own c/g " - "twin included. Only length-preserving changes, which RT's codon-level candidates always include." + "twin included. Only length-preserving changes inside the codon RT targeted, which its candidates " + "always include." + ), + sql=_measured_outside_group( + f"NOT {_LENGTH_CHANGING} AND NOT {_REFERENCE_IDENTICAL} AND NOT {_BEYOND_CANDIDATES}" ), - sql=_MEASURED_OUTSIDE_GROUP_SQL.replace("{length_condition}", f"NOT {_LENGTH_CHANGING}"), + ), + SqlCheck( + name="measured_allele_beyond_rt_codon", + severity=INFO, + description=( + "A measured change in no projection group that reaches outside the codon RT targeted, such as a " + "multi-base change straddling two codons. RT lists only single-codon changes, so it cannot fold in. " + "Many in one score set can mean its nucleotide variants sit out of frame with its protein variants." + ), + sql=_measured_outside_group(f"NOT {_LENGTH_CHANGING} AND NOT {_REFERENCE_IDENTICAL} AND {_BEYOND_CANDIDATES}"), + ), + SqlCheck( + name="measured_reference_allele_outside_projection_group", + severity=INFO, + description=( + "A measured reference-identical allele (g.=, c.=) in no projection group. Expected: RT lists the " + "alternative codons for a synonymous change, never the reference codon." + ), + sql=_measured_outside_group(_REFERENCE_IDENTICAL), ), SqlCheck( name="measured_indel_outside_projection_group", @@ -227,15 +293,16 @@ class SqlCheck: "A measured indel in no projection group. Expected when the indel is not codon-aligned: it changes the " "protein the same way as a codon-aligned candidate but is a different DNA change, so RT does not list it." ), - sql=_MEASURED_OUTSIDE_GROUP_SQL.replace("{length_condition}", _LENGTH_CHANGING), + sql=_measured_outside_group(f"{_LENGTH_CHANGING} AND NOT {_REFERENCE_IDENTICAL}"), ), SqlCheck( name="malformed_projection_group", severity=WARNING, description=( - "A live projection group is not exactly one cdna allele and one genomic allele. A member RT could not " - "translate leaves a group of one; the variant's cross_level_translation event lists it under " - "failed_candidates with the error." + "A live projection group is not exactly one cdna allele and one genomic allele. Either the library could " + "not project the coding candidate to the genome (its entry under candidates in the variant's " + "cross_level_translation event has no hgvs_g), or a member failed VRS translation (listed under " + "failed_candidates with the error)." ), sql=""" SELECT l.mapping_record_id, l.projection_group, count(*) AS members, @@ -400,34 +467,21 @@ def report(findings: Sequence[Finding]) -> int: return sum(1 for finding in findings if finding.severity == ERROR and finding.count) -def _score_set_ids(db: Session, urns: Sequence[str]) -> Optional[list[int]]: - """Resolve ``--score-set`` URNs to ids, or ``None`` when the audit is unscoped.""" - if not urns: - return None - - found: dict[str, int] = { - urn: id for urn, id in db.execute(select(ScoreSet.urn, ScoreSet.id).where(ScoreSet.urn.in_(urns))).all() if urn - } - missing = sorted(set(urns) - set(found)) - if missing: - raise click.BadParameter(f"No score set with URN: {', '.join(missing)}", param_hint="--score-set") - - return list(found.values()) - - -score_set_option = click.option( - "--score-set", "urns", multiple=True, help="Restrict the audit to this score set. Repeatable." -) - - @script_environment.command() -@score_set_option +@scope_options @click.option("--samples", type=int, default=5, show_default=True, help="Violating rows to print per check.") @click.option("--skip-seqrepo", is_flag=True, help="Skip the refget comparison, which needs the SeqRepo mount.") @with_database_session -def invariants(db: Session, urns: tuple[str, ...], samples: int, skip_seqrepo: bool) -> None: +def invariants( + db: Session, + urns: tuple[str, ...], + collection: Optional[str], + translated_since: Optional[datetime], + samples: int, + skip_seqrepo: bool, +) -> None: """Check the structural invariants of the mapping and reverse-translation allele graph.""" - score_set_ids = _score_set_ids(db, urns) + score_set_ids = resolve_scope(db, urns, collection, translated_since) findings: list[Finding] = [] for check in SQL_CHECKS: @@ -492,7 +546,7 @@ def collect_outcomes(rows: Sequence[tuple[str, str, str, int]]) -> list[ScoreSet @script_environment.command() -@score_set_option +@scope_options @click.option( "--max-failed-pct", type=float, @@ -501,9 +555,15 @@ def collect_outcomes(rows: Sequence[tuple[str, str, str, int]]) -> list[ScoreSet help="Flag score sets where more than this share of variants failed RT.", ) @with_database_session -def outcomes(db: Session, urns: tuple[str, ...], max_failed_pct: float) -> None: +def outcomes( + db: Session, + urns: tuple[str, ...], + collection: Optional[str], + translated_since: Optional[datetime], + max_failed_pct: float, +) -> None: """Summarize the latest reverse-translation result per variant, by score set.""" - score_set_ids = _score_set_ids(db, urns) + score_set_ids = resolve_scope(db, urns, collection, translated_since) event_filter = "AND e.score_set_id = ANY(:score_set_ids)" if score_set_ids is not None else "" params = {"score_set_ids": score_set_ids} if score_set_ids is not None else {} @@ -658,7 +718,7 @@ def round_trip_pair( @script_environment.command() -@score_set_option +@scope_options @click.option( "--per-score-set", type=int, @@ -668,14 +728,21 @@ def round_trip_pair( ) @click.option("--samples", type=int, default=10, show_default=True, help="Rows to print per disagreeing outcome.") @with_database_session -def round_trip(db: Session, urns: tuple[str, ...], per_score_set: int, samples: int) -> None: +def round_trip( + db: Session, + urns: tuple[str, ...], + collection: Optional[str], + translated_since: Optional[datetime], + per_score_set: int, + samples: int, +) -> None: """Re-translate sampled projection pairs and check they identify as the alleles RT stored.""" from ga4gh.vrs.extras.translator import AlleleTranslator from mavedb.data_providers.services import cdot_rest, seqrepo_data_proxy from mavedb.worker.lib.translation_ports import WorkerCoordinateTranslator - score_set_ids = _score_set_ids(db, urns) + score_set_ids = resolve_scope(db, urns, collection, translated_since) link_filter = "AND l.score_set_id = ANY(:score_set_ids)" if score_set_ids is not None else "" params: dict[str, Any] = {"per_score_set": per_score_set} if score_set_ids is not None: diff --git a/src/mavedb/scripts/audit_scope.py b/src/mavedb/scripts/audit_scope.py new file mode 100644 index 000000000..2f7fd24ac --- /dev/null +++ b/src/mavedb/scripts/audit_scope.py @@ -0,0 +1,85 @@ +""" +Score-set scoping shared by the audit scripts. + +An unscoped audit covers the whole database, so on an environment where only some score sets have been +through the current pipeline, findings from stale score sets mix with real ones. These options restrict an +audit to the score sets that matter. Given together, they intersect. +""" + +import datetime +from typing import Callable, Optional, Sequence + +import asyncclick as click +from sqlalchemy import select, text +from sqlalchemy.orm import Session + +from mavedb.models.collection import Collection +from mavedb.models.collection_score_set_association import CollectionScoreSetAssociation +from mavedb.models.score_set import ScoreSet + +# A score set every one of whose variants has an RT event on or after :since. RT writes an event for every +# variant it processes, not_applicable and failed included, so a variant RT has not reached since then keeps +# the score set out. +TRANSLATED_SINCE_SQL = """ + SELECT score_set_id + FROM ( + SELECT e.score_set_id, e.variant_id, max(e.created_at) AS translated_at + FROM annotation_event e + WHERE e.annotation_type = 'cross_level_translation' + GROUP BY 1, 2 + ) latest + GROUP BY score_set_id + HAVING min(translated_at) >= :since +""" + + +def scope_options(command: Callable) -> Callable: + """Add ``--score-set``, ``--collection`` and ``--translated-since`` to an audit command.""" + command = click.option( + "--translated-since", + type=click.DateTime(formats=["%Y-%m-%d"]), + default=None, + help="Only score sets whose every variant was reverse translated on or after this date (UTC).", + )(command) + command = click.option("--collection", default=None, help="Only score sets in this collection (URN).")(command) + command = click.option("--score-set", "urns", multiple=True, help="Only this score set (URN). Repeatable.")(command) + return command + + +def resolve_scope( + db: Session, + urns: Sequence[str], + collection: Optional[str], + translated_since: Optional[datetime.datetime], +) -> Optional[list[int]]: + """The score-set ids the scope options select, or ``None`` when none were given.""" + if not urns and collection is None and translated_since is None: + return None + + query = select(ScoreSet.id) + + if urns: + found = set(db.scalars(select(ScoreSet.urn).where(ScoreSet.urn.in_(urns)))) + if missing := sorted(set(urns) - found): + raise click.BadParameter(f"No score set with URN: {', '.join(missing)}", param_hint="--score-set") + query = query.where(ScoreSet.urn.in_(urns)) + + if collection is not None: + if db.scalar(select(Collection.id).where(Collection.urn == collection)) is None: + raise click.BadParameter(f"No collection with URN: {collection}", param_hint="--collection") + query = ( + query.join(CollectionScoreSetAssociation, CollectionScoreSetAssociation.score_set_id == ScoreSet.id) + .join(Collection, Collection.id == CollectionScoreSetAssociation.collection_id) + .where(Collection.urn == collection) + ) + + if translated_since is not None: + since = translated_since.replace(tzinfo=datetime.timezone.utc) + translated = [row[0] for row in db.execute(text(TRANSLATED_SINCE_SQL), {"since": since})] + query = query.where(ScoreSet.id.in_(translated)) + + score_set_ids = list(db.scalars(query)) + if not score_set_ids: + raise click.UsageError("No score set matches the given scope.") + + return score_set_ids diff --git a/tests/scripts/test_audit_reverse_translation.py b/tests/scripts/test_audit_reverse_translation.py index 1aff5cc93..056c6e415 100644 --- a/tests/scripts/test_audit_reverse_translation.py +++ b/tests/scripts/test_audit_reverse_translation.py @@ -150,6 +150,72 @@ def test_reports_a_measured_indel_outside_the_groups_as_info_not_error(self, ses assert _count(session, "measured_allele_outside_projection_group", [score_set.id]).count == 0 assert _count(session, "measured_indel_outside_projection_group", [score_set.id]).count == 1 + def test_reports_a_change_straddling_two_codons_as_info_not_error(self, session, make_score_set): + # c.135_137delinsTGC changes the third base of codon 45 and the second of codon 46; RT's candidates for + # p.Val46Ala sit inside codon 46 (g.95025144_95025146) only. + score_set, (variant,) = _score_set_with_variants(session, make_score_set, "urn:mavedb:00000013-a-1", 1) + + def located(start, end, sequence): + return { + "type": "Allele", + "location": { + "type": "SequenceLocation", + "sequenceReference": {"refgetAccession": "SQ.genome"}, + "start": start, + "end": end, + }, + "state": {"type": "LiteralSequenceExpression", "sequence": sequence}, + } + + seed_mapping_record( + session, + variant, + alleles=[ + AlleleSpec( + digest="measured", + level="genomic", + is_authoritative=True, + hgvs_g="NC_000008.11:g.95025143_95025145delinsTGC", + post_mapped=located(95025142, 95025145, "TGC"), + ), + AlleleSpec(digest="rt-c", level="cdna", hgvs_c="NM_152416.4:c.137T>C", projection_group=0), + AlleleSpec( + digest="rt-g", + level="genomic", + hgvs_g="NC_000008.11:g.95025145T>C", + projection_group=0, + post_mapped=located(95025144, 95025145, "C"), + ), + AlleleSpec( + digest="rt-g-2", + level="genomic", + hgvs_g="NC_000008.11:g.95025145_95025146delinsCA", + projection_group=1, + post_mapped=located(95025144, 95025146, "CA"), + ), + ], + ) + + assert _count(session, "measured_allele_outside_projection_group", [score_set.id]).count == 0 + assert _count(session, "measured_allele_beyond_rt_codon", [score_set.id]).count == 1 + + def test_reports_a_reference_identical_allele_as_info_not_error(self, session, make_score_set): + score_set, (variant,) = _score_set_with_variants(session, make_score_set, "urn:mavedb:00000014-a-1", 1) + seed_mapping_record( + session, + variant, + alleles=[ + AlleleSpec( + digest="measured", level="genomic", is_authoritative=True, hgvs_g="NC_000011.10:g.102114338T=" + ), + AlleleSpec(digest="rt-c", level="cdna", hgvs_c="NM_001130145.3:c.519G>A", projection_group=0), + AlleleSpec(digest="rt-g", level="genomic", hgvs_g="NC_000011.10:g.102114338T>A", projection_group=0), + ], + ) + + assert _count(session, "measured_allele_outside_projection_group", [score_set.id]).count == 0 + assert _count(session, "measured_reference_allele_outside_projection_group", [score_set.id]).count == 1 + def test_flags_an_accession_whose_alleles_carry_two_refgets(self, session, make_score_set): score_set, variants = _score_set_with_variants(session, make_score_set, "urn:mavedb:00000003-a-1", 2) for n, (variant, refget) in enumerate(zip(variants, ("SQ.ncbi", "SQ.cdot")), start=1): diff --git a/tests/scripts/test_audit_scope.py b/tests/scripts/test_audit_scope.py new file mode 100644 index 000000000..093eb9672 --- /dev/null +++ b/tests/scripts/test_audit_scope.py @@ -0,0 +1,86 @@ +# ruff: noqa: E402 + +from datetime import datetime, timezone + +import pytest + +pytest.importorskip("asyncclick") + +import asyncclick as click + +from mavedb.models.annotation_event import AnnotationEvent +from mavedb.models.collection import Collection +from mavedb.models.collection_score_set_association import CollectionScoreSetAssociation +from mavedb.models.variant import Variant +from mavedb.scripts.audit_scope import resolve_scope + + +def _score_set(session, make_score_set, urn, translated_at): + """A score set with two variants, each reverse translated at the given instants (None: never).""" + score_set = make_score_set() + score_set.urn = urn + session.commit() + + for i, at in enumerate(translated_at, start=1): + variant = Variant(urn=f"{urn}#{i}", score_set_id=score_set.id, hgvs_nt=f"c.{i}A>G", data={}) + session.add(variant) + session.flush() + if at is not None: + session.add( + AnnotationEvent( + annotation_type="cross_level_translation", + variant_id=variant.id, + score_set_id=score_set.id, + disposition="present", + reason="translated", + created_at=at, + ) + ) + session.commit() + return score_set + + +OLD = datetime(2026, 9, 15, tzinfo=timezone.utc) +NEW = datetime(2026, 10, 7, tzinfo=timezone.utc) +SINCE = datetime(2026, 10, 1) + + +@pytest.mark.integration +class TestResolveScope: + def test_returns_none_when_unscoped(self, session): + assert resolve_scope(session, (), None, None) is None + + def test_translated_since_keeps_only_score_sets_every_variant_of_which_rt_reached(self, session, make_score_set): + current = _score_set(session, make_score_set, "urn:mavedb:00000020-a-1", [NEW, NEW]) + _score_set(session, make_score_set, "urn:mavedb:00000021-a-1", [OLD, OLD]) + _score_set(session, make_score_set, "urn:mavedb:00000022-a-1", [NEW, OLD]) + _score_set(session, make_score_set, "urn:mavedb:00000023-a-1", [None, None]) + + assert resolve_scope(session, (), None, SINCE) == [current.id] + + def test_options_intersect(self, session, make_score_set, sample_user): + in_collection = _score_set(session, make_score_set, "urn:mavedb:00000024-a-1", [NEW]) + stale_in_collection = _score_set(session, make_score_set, "urn:mavedb:00000025-a-1", [OLD]) + _score_set(session, make_score_set, "urn:mavedb:00000026-a-1", [NEW]) + + collection = Collection(name="audit", urn="urn:mavedb:collection-audit", created_by_id=sample_user.id) + session.add(collection) + session.flush() + for position, score_set in enumerate((in_collection, stale_in_collection)): + session.add( + CollectionScoreSetAssociation(collection_id=collection.id, score_set_id=score_set.id, position=position) + ) + session.commit() + + assert sorted(resolve_scope(session, (), collection.urn, None)) == sorted( + [in_collection.id, stale_in_collection.id] + ) + assert resolve_scope(session, (), collection.urn, SINCE) == [in_collection.id] + + def test_rejects_an_unknown_collection_and_an_empty_scope(self, session, make_score_set): + _score_set(session, make_score_set, "urn:mavedb:00000027-a-1", [OLD]) + + with pytest.raises(click.BadParameter): + resolve_scope(session, (), "urn:mavedb:collection-missing", None) + with pytest.raises(click.UsageError): + resolve_scope(session, (), None, SINCE) From aff9c826e93ba280351c22e3547e3b65e0d47e4c Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Mon, 12 Oct 2026 11:25:24 -0700 Subject: [PATCH 238/252] feat(audit): flag SeqRepo sequences carrying a foreign ga4gh alias A sequence's only correct ga4gh alias is its own digest. Any other makes refget lookups for it nondeterministic, as the mapper's stop-codon digest bug did on staging. --- .../scripts/audit_reverse_translation.py | 35 ++++++++++++++++++- .../scripts/test_audit_reverse_translation.py | 24 +++++++++++++ 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/src/mavedb/scripts/audit_reverse_translation.py b/src/mavedb/scripts/audit_reverse_translation.py index 1b400dcf6..85c6b36fb 100644 --- a/src/mavedb/scripts/audit_reverse_translation.py +++ b/src/mavedb/scripts/audit_reverse_translation.py @@ -18,7 +18,7 @@ silently read as projections. `invariants` runs the structural checks in SQL, plus a comparison of every stored refget against -SeqRepo. `outcomes` rolls the latest RT result per variant up by score set, so an outlier score set +SeqRepo and a scan of SeqRepo for sequences carrying a ga4gh alias other than their own digest. `outcomes` rolls the latest RT result per variant up by score set, so an outlier score set stands out. `round-trip` re-translates a sample of projection pairs with the same coordinate mapper RT uses and checks that each pair and its protein apex identify as the same VRS alleles RT stored. @@ -33,6 +33,7 @@ import json import logging import re +import sqlite3 import sys from collections import Counter, defaultdict from dataclasses import dataclass, field @@ -454,6 +455,36 @@ def check_refgets_against_seqrepo( ] +# SeqRepo stores each ga4gh alias in its VMC namespace as ``GS_``. A sequence's only correct ga4gh +# alias is its own digest; any other is current and makes refget lookups return either digest. +FOREIGN_REFGET_ALIAS_SQL = """ + SELECT seqalias_id, seq_id, alias, added + FROM seqalias + WHERE namespace = 'VMC' AND is_current = 1 AND alias != 'GS_' || seq_id + ORDER BY added +""" + + +def check_foreign_refget_aliases(aliases_db: sqlite3.Connection, samples: int) -> Finding: + """Find SeqRepo sequences carrying a ga4gh alias other than their own digest. + + Every lookup that turns an accession into a refget takes the first ga4gh alias of the sequence, so a + second one makes the answer vary between processes. Not scoped: SeqRepo is shared by every score set. + """ + rows = [ + {"seqalias_id": seqalias_id, "seq_id": seq_id, "alias": alias, "added": added} + for seqalias_id, seq_id, alias, added in aliases_db.execute(FOREIGN_REFGET_ALIAS_SQL) + ] + return Finding( + "seqrepo_foreign_refget_alias", + ERROR, + "A SeqRepo sequence carries a ga4gh alias that is not its own digest, so refget lookups for it are " + "nondeterministic. Retire the alias (is_current = 0) and re-map the score sets whose records use it.", + len(rows), + rows[:samples], + ) + + def report(findings: Sequence[Finding]) -> int: """Log each finding with its samples and return how many error-level findings are non-zero.""" for finding in findings: @@ -496,6 +527,8 @@ def invariants( sr = seqrepo() logger.info("running refget_disagrees_with_seqrepo") findings.extend(check_refgets_against_seqrepo(db, lambda acc: resolve_refget(sr, acc), score_set_ids, samples)) + logger.info("running seqrepo_foreign_refget_alias") + findings.append(check_foreign_refget_aliases(sr.aliases._db, samples)) if errors := report(findings): logger.error("%d error-level check(s) failed.", errors) diff --git a/tests/scripts/test_audit_reverse_translation.py b/tests/scripts/test_audit_reverse_translation.py index 056c6e415..e4c76b520 100644 --- a/tests/scripts/test_audit_reverse_translation.py +++ b/tests/scripts/test_audit_reverse_translation.py @@ -1,5 +1,6 @@ # ruff: noqa: E402 +import sqlite3 from datetime import datetime, timezone import pytest @@ -21,6 +22,7 @@ ERROR, SQL_CHECKS, ProjectionPair, + check_foreign_refget_aliases, check_refgets_against_seqrepo, collect_outcomes, round_trip_pair, @@ -520,3 +522,25 @@ def test_orders_score_sets_by_failure_share_and_tallies_failure_reasons(self): assert outcomes[0].failed_pct == 50.0 assert outcomes[0].failure_reasons.most_common(1) == [("translation_failed", 30)] assert outcomes[1].total == 100 + + +class TestForeignRefgetAliases: + def test_flags_a_ga4gh_alias_that_is_not_the_sequences_own_digest(self): + db = sqlite3.connect(":memory:") + db.execute( + "CREATE TABLE seqalias (seqalias_id INTEGER, seq_id TEXT, alias TEXT, namespace TEXT, added TEXT, is_current INTEGER)" + ) + db.executemany( + "INSERT INTO seqalias VALUES (?, ?, ?, ?, ?, ?)", + [ + (1, "j1QR", "GS_j1QR", "VMC", "2017-07-19", 1), + (2, "j1QR", "GS__KAVUS", "VMC", "2026-10-02", 1), + (3, "j1QR", "NP_001346945.1", "NCBI", "2018-08-21", 1), + (4, "JtEW", "GS_jqmY", "VMC", "2026-10-08", 0), + ], + ) + + finding = check_foreign_refget_aliases(db, samples=5) + + assert finding.count == 1 + assert finding.samples[0]["alias"] == "GS__KAVUS" From a938dc5b5de4309e38075739b91ec40e9ed38e60 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 14 Oct 2026 13:22:47 -0700 Subject: [PATCH 239/252] chore(dev): run local Postgres on 15 to match production Production RDS runs Postgres 15. Both compose files move to postgres:15 on new volumes, since a 14 data directory won't start under 15; the first `up` initializes an empty database. --- docker-compose-dev.yml | 6 +++--- docker-compose-local.yml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 50e092e3d..013f549bd 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -92,13 +92,13 @@ services: - mavedb-seqrepo-dev:/usr/local/share/seqrepo db: - image: postgres:14 + image: postgres:15 env_file: - settings/.env.dev ports: - "5434:5432" volumes: - - mavedb-data-dev:/var/lib/postgresql/data + - mavedb-data-dev-15:/var/lib/postgresql/data redis: image: redis:7.2.3 @@ -129,6 +129,6 @@ services: - mavedb-seqrepo-dev:/usr/local/share/seqrepo volumes: - mavedb-data-dev: + mavedb-data-dev-15: mavedb-redis-dev: mavedb-seqrepo-dev: diff --git a/docker-compose-local.yml b/docker-compose-local.yml index 69a9fd1a1..a3bf81831 100644 --- a/docker-compose-local.yml +++ b/docker-compose-local.yml @@ -44,14 +44,14 @@ services: restart: unless-stopped db: - image: postgres:14 + image: postgres:15 env_file: - settings/.env.dev restart: unless-stopped ports: - "5434:5432" volumes: - - mavedb-data:/var/lib/postgresql/data + - mavedb-data-15:/var/lib/postgresql/data redis: image: redis:7.2.3 @@ -68,5 +68,5 @@ services: # - "5673:5672" volumes: - mavedb-data: + mavedb-data-15: mavedb-redis: From 6580fe90df03145896d0808b351801803cdccdb0 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 14 Oct 2026 13:24:10 -0700 Subject: [PATCH 240/252] fix(audit): stream identifier audit rows instead of buffering the result Each audit loaded its whole result set into memory before iterating, which exhausted a 14 GB container on a production-sized database (1.6M alleles, 7.3M mapping records). Rows now stream in batches of 5000, keeping memory flat; results are unchanged. --- src/mavedb/scripts/audit_allele_identifiers.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/mavedb/scripts/audit_allele_identifiers.py b/src/mavedb/scripts/audit_allele_identifiers.py index 8c9c84042..fd0d1128d 100644 --- a/src/mavedb/scripts/audit_allele_identifiers.py +++ b/src/mavedb/scripts/audit_allele_identifiers.py @@ -63,6 +63,9 @@ logger = logging.getLogger(__name__) +# Rows are streamed in batches of this size; buffering a production-sized result set exhausts memory. +STREAM_BATCH_SIZE = 5000 + def recomputed_identifier(post_mapped: dict[str, Any]) -> Optional[str]: """The identifier this document's own content implies, or ``None`` if it cannot be hydrated. @@ -117,7 +120,7 @@ def audit( renames: list[tuple[Allele, str]] = [] duplicates: list[tuple[Allele, str]] = [] - for allele in db.scalars(query): + for allele in db.scalars(query.execution_options(yield_per=STREAM_BATCH_SIZE)): counts["examined"] += 1 # post_mapped is nullable on the model, but the query above only selects rows where it isn't. assert allele.post_mapped is not None @@ -198,7 +201,7 @@ def audit_pre_mapped( counts: Counter[str] = Counter() drifted: list[tuple[int, Optional[str], str]] = [] - for record in db.scalars(query): + for record in db.scalars(query.execution_options(yield_per=STREAM_BATCH_SIZE)): counts["examined"] += 1 # pre_mapped is nullable on the model, but the query above only selects rows where it isn't. assert record.pre_mapped is not None @@ -252,7 +255,7 @@ def audit_locations( counts: Counter[str] = Counter() drifted: set[int] = set() - for allele in db.scalars(query): + for allele in db.scalars(query.execution_options(yield_per=STREAM_BATCH_SIZE)): counts["alleles examined"] += 1 for path, location in _stored_locations(allele.post_mapped): stored = location.get("id") From 4820e1cdb5b12942f40e0fe0def54f915ae78b5d Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 14 Oct 2026 14:45:48 -0700 Subject: [PATCH 241/252] fix(tests): make the core-dependency suite collect and pass without server extras The measurements view model imported its relationship enum from lib.allele_measurements, which pulls in fastapi through permissions; the enum now lives in models/enums. The audit-scope and URN-lock tests use the Postgres session fixture, so they skip when psycopg2 is absent. --- src/mavedb/lib/allele_measurements.py | 10 +--------- src/mavedb/models/enums/measurement_relationship.py | 9 +++++++++ src/mavedb/view_models/allele_measurement.py | 2 +- tests/lib/test_allele_measurements.py | 2 +- tests/lib/test_urns.py | 3 +++ tests/scripts/test_audit_scope.py | 1 + 6 files changed, 16 insertions(+), 11 deletions(-) create mode 100644 src/mavedb/models/enums/measurement_relationship.py diff --git a/src/mavedb/lib/allele_measurements.py b/src/mavedb/lib/allele_measurements.py index df1b37b92..587188a25 100644 --- a/src/mavedb/lib/allele_measurements.py +++ b/src/mavedb/lib/allele_measurements.py @@ -33,7 +33,6 @@ import logging from dataclasses import dataclass from datetime import date, datetime -from enum import Enum from typing import Optional from sqlalchemy import and_, select @@ -50,6 +49,7 @@ from mavedb.lib.types.authentication import UserData from mavedb.lib.variants import variant_score from mavedb.models.allele import Allele +from mavedb.models.enums.measurement_relationship import MeasurementRelationship from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.models.experiment import Experiment from mavedb.models.experiment_controlled_keyword import ExperimentControlledKeywordAssociation @@ -63,14 +63,6 @@ logger = logging.getLogger(__name__) -class MeasurementRelationship(str, Enum): - """How a measurement relates to the queried ClinGen id, by the measurement's *measured* level.""" - - direct = "direct" # assayed at the queried allele - protein_consequence = "protein_consequence" # protein measurement of a CA's consequence - nucleotide_encoding = "nucleotide_encoding" # an nt measurement of the protein apex's encoding - - @dataclass(frozen=True) class AlleleMeasurement: """One measurement in the query's equivalence class. diff --git a/src/mavedb/models/enums/measurement_relationship.py b/src/mavedb/models/enums/measurement_relationship.py new file mode 100644 index 000000000..cccf8bc17 --- /dev/null +++ b/src/mavedb/models/enums/measurement_relationship.py @@ -0,0 +1,9 @@ +from enum import Enum + + +class MeasurementRelationship(str, Enum): + """How a measurement relates to the queried ClinGen id, by the measurement's *measured* level.""" + + direct = "direct" # assayed at the queried allele + protein_consequence = "protein_consequence" # protein measurement of a CA's consequence + nucleotide_encoding = "nucleotide_encoding" # an nt measurement of the protein apex's encoding diff --git a/src/mavedb/view_models/allele_measurement.py b/src/mavedb/view_models/allele_measurement.py index ebf8ffcbe..b8f3e969d 100644 --- a/src/mavedb/view_models/allele_measurement.py +++ b/src/mavedb/view_models/allele_measurement.py @@ -7,7 +7,7 @@ from typing import Optional -from mavedb.lib.allele_measurements import MeasurementRelationship +from mavedb.models.enums.measurement_relationship import MeasurementRelationship from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.view_models.base.base import BaseModel from mavedb.view_models.score_calibration import SavedFunctionalClassification diff --git a/tests/lib/test_allele_measurements.py b/tests/lib/test_allele_measurements.py index 4f969a670..8822c928d 100644 --- a/tests/lib/test_allele_measurements.py +++ b/tests/lib/test_allele_measurements.py @@ -23,13 +23,13 @@ from mavedb.lib.allele_measurements import ( AlleleMeasurement, - MeasurementRelationship, _ordering_key, _resolve_protein_apex, get_allele_measurements, ) from mavedb.lib.mondo import get_generic_disease_term from mavedb.lib.types.authentication import UserData +from mavedb.models.enums.measurement_relationship import MeasurementRelationship from mavedb.models.allele import Allele from mavedb.models.mapping_record import MappingRecord from mavedb.models.mapping_record_allele import MappingRecordAllele diff --git a/tests/lib/test_urns.py b/tests/lib/test_urns.py index 600c723b5..8bb56b8d0 100644 --- a/tests/lib/test_urns.py +++ b/tests/lib/test_urns.py @@ -1,3 +1,5 @@ +import importlib.util + import pytest from sqlalchemy import text from sqlalchemy.orm import sessionmaker @@ -101,6 +103,7 @@ def test_urns_without_a_variant_suffix_sort_last_without_raising(self, urn): @pytest.mark.integration +@pytest.mark.skipif(importlib.util.find_spec("psycopg2") is None, reason="needs the Postgres session fixture") class TestLockUrnAssignment: @staticmethod def _other_session_can_take_lock(session) -> bool: diff --git a/tests/scripts/test_audit_scope.py b/tests/scripts/test_audit_scope.py index 093eb9672..b52f4bd09 100644 --- a/tests/scripts/test_audit_scope.py +++ b/tests/scripts/test_audit_scope.py @@ -5,6 +5,7 @@ import pytest pytest.importorskip("asyncclick") +pytest.importorskip("psycopg2") import asyncclick as click From 07e01f9339123d62a32cf62ccd9b051ca3ef6351 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 14 Oct 2026 22:27:46 -0700 Subject: [PATCH 242/252] docs: update instructions and worker docs for projection groups and the 2026.3.0 substrate Describe measured and candidate projection groups, the ValidTime base class, the 30-minute stall window and the public dump's mapped HGVS columns. --- .github/instructions/api.instructions.md | 43 ++++++++----------- .github/instructions/copilot-instructions.md | 13 +++--- .github/instructions/database.instructions.md | 19 +++++--- .github/instructions/worker.instructions.md | 6 +-- README.md | 4 +- src/mavedb/scripts/resources/CHANGELOG.md | 13 ++++++ src/mavedb/scripts/resources/README.md | 28 ++++++++---- src/mavedb/worker/README.md | 11 +++-- src/mavedb/worker/best_practices.md | 11 +++-- src/mavedb/worker/job_registry.md | 9 ++-- src/mavedb/worker/jobs_overview.md | 11 +++-- src/mavedb/worker/pipeline_management.md | 15 ++++--- 12 files changed, 110 insertions(+), 73 deletions(-) diff --git a/.github/instructions/api.instructions.md b/.github/instructions/api.instructions.md index e29553d9b..a19e68521 100644 --- a/.github/instructions/api.instructions.md +++ b/.github/instructions/api.instructions.md @@ -94,10 +94,12 @@ async def publish_score_set( worker: ArqRedis = Depends(get_worker), ): # ... validation and DB updates ... - await worker.enqueue_job( - "create_variants_for_score_set", - score_set.id, - correlation_id, + await enqueue_pipeline_for_score_set( + db=db, + redis=worker, + pipeline_name="publish_score_set", + score_set=item, + user=user_data.user, ) ``` @@ -114,24 +116,20 @@ responses=shared_responses # Defines 4xx/5xx response schemas Most write operations trigger a multi-step pipeline via the worker: ```python -from mavedb.lib.workflow.pipeline_factory import PipelineFactory +from mavedb.lib.workflow.kickoff import enqueue_pipeline_for_score_set # In a router endpoint: -pipeline, entrypoint_job_run = PipelineFactory.create_pipeline( +pipeline, job = await enqueue_pipeline_for_score_set( db=db, - name="validate_map_annotate_score_set", - pipeline_params={ - "score_set_id": score_set.id, - "updater_id": user_data.user.id, - "correlation_id": logging_context().get("correlation_id"), - }, + redis=worker, + pipeline_name="validate_map_annotate_score_set", + score_set=item, + user=user_data.user, + extra_params={"scores_file_key": scores_file_key, ...}, ) -db.commit() - -await worker.enqueue_job("start_pipeline", entrypoint_job_run.id) ``` -This creates a `Pipeline` with multiple `JobRun` records and `JobDependency` records, then enqueues the pipeline's `start_pipeline` entrypoint in ARQ. The worker coordinates the rest — each job runs after its dependencies complete. +`enqueue_pipeline_for_score_set` is the single kickoff seam shared by routers and scripts. It fills `score_set_id`, `updater_id` and `correlation_id`, uses `PipelineFactory` to create a `Pipeline` with its `JobRun` and `JobDependency` records, then enqueues the pipeline's `start_pipeline` entrypoint in ARQ. A failed enqueue discards the just-created pipeline records. The worker coordinates the rest — each job runs after its dependencies complete. ### Job Function Signature @@ -155,18 +153,11 @@ Callers pass only `ctx` and `job_id` when enqueueing. The decorator creates the Correlation IDs flow from the API request through the pipeline to each job: ```python -# In the router — capture correlation ID from starlette-context -from mavedb.lib.logging.context import save_to_logging_context, logging_context +# In the router — save request context; the kickoff seam reads the correlation ID from it +from mavedb.lib.logging.context import save_to_logging_context save_to_logging_context({"score_set_urn": urn}) -correlation_id = logging_context().get("correlation_id") - -# Pass to pipeline via pipeline_params -pipeline, entrypoint = PipelineFactory.create_pipeline( - db=db, - name="validate_map_annotate_score_set", - pipeline_params={"correlation_id": correlation_id, ...}, -) +await enqueue_pipeline_for_score_set(db=db, redis=worker, pipeline_name="validate_map_annotate_score_set", ...) ``` Each job retrieves the correlation ID from its `job_params` and uses `job_manager.save_to_context()` for structured logging. diff --git a/.github/instructions/copilot-instructions.md b/.github/instructions/copilot-instructions.md index 089220c66..884e83245 100644 --- a/.github/instructions/copilot-instructions.md +++ b/.github/instructions/copilot-instructions.md @@ -12,8 +12,8 @@ MaveDB is a bioinformatics database API for Multiplex Assays of Variant Effect ( ### Service Architecture - **FastAPI application** — `src/mavedb/server_main.py`, router-based endpoint organization - **Background worker** — ARQ/Redis for async processing (variant creation, mapping, publication, annotation) -- **Multi-container setup** — API server, worker, PostgreSQL, Redis, plus external services (cdot-rest, dcd-mapping, seqrepo) -- **Docker config** — `docker-compose-dev.yml` (6 services), multi-stage `Dockerfile` +- **Multi-container setup** — API server, worker, PostgreSQL 15, Redis, moto (local S3), plus external services (cdot-rest, dcd-mapping, seqrepo) +- **Docker config** — `docker-compose-dev.yml` (9 services), multi-stage `Dockerfile` ### Core Domain Model - **Hierarchical URN system**: ExperimentSet (`urn:mavedb:00000001`) → Experiment (`00000001-a`) → ScoreSet (`00000001-a-1`) → Variant (ScoreSet URN + `#` + number) @@ -41,8 +41,8 @@ src/mavedb/ ├── worker/ # ARQ background worker system │ ├── jobs/ # Job function implementations (by category) │ │ ├── registry.py # Central registry of all jobs, cron definitions -│ │ ├── variant_processing/ # Variant creation and mapping -│ │ ├── external_services/ # ClinGen, ClinVar, gnomAD, UniProt +│ │ ├── variant_processing/ # Variant creation, mapping, reverse translation +│ │ ├── external_services/ # ClinGen, ClinVar, gnomAD, UniProt, VEP │ │ ├── pipeline_management/ # Pipeline entrypoint (start_pipeline) │ │ └── system/ # Cron maintenance (cleanup stalled jobs) │ ├── lib/ # Infrastructure layer @@ -143,7 +143,7 @@ poetry run python -m mavedb.scripts. - [score_set.py](src/mavedb/models/score_set.py) — Primary data model patterns - [score_sets.py](src/mavedb/routers/score_sets.py) — Complex router with worker integration -- [jobs.py](src/mavedb/worker/jobs.py) — Background processing patterns +- [registry.py](src/mavedb/worker/jobs/registry.py) — Background job registration - [score_set.py](src/mavedb/view_models/score_set.py) — Pydantic model hierarchy - [server_main.py](src/mavedb/server_main.py) — App setup and dependency injection - [authentication.py](src/mavedb/lib/authentication.py) — Auth patterns @@ -222,9 +222,10 @@ poetry run python -m mavedb.scripts. - **Key job types**: - `create_variants_for_score_set` - Parse uploaded CSV, create variant records - `map_variants_for_score_set` - Map variants via DCD Mapping / VRS + - `reverse_translate_variants_for_score_set` - Derive equivalent nucleotide/protein alleles and projection groups via variant-annotation - `submit_score_set_mappings_to_car/ldh` - Submit to ClinGen services - `cleanup_stalled_jobs` - Cron job for recovering stuck jobs -- **Enqueueing pipelines**: Routers call `PipelineFactory.create_pipeline()` then `ArqRedis.enqueue_job("start_pipeline", ...)` with the pipeline's entrypoint JobRun ID +- **Enqueueing pipelines**: Routers and scripts call `enqueue_pipeline_for_score_set()` (`src/mavedb/lib/workflow/kickoff.py`), which creates the pipeline via `PipelineFactory.create_pipeline()` and enqueues its `start_pipeline` entrypoint JobRun - **Detailed documentation**: See `src/mavedb/worker/README.md` and `.github/instructions/worker.instructions.md` ### View Models (Pydantic) diff --git a/.github/instructions/database.instructions.md b/.github/instructions/database.instructions.md index f2b3de6d1..947fc36cc 100644 --- a/.github/instructions/database.instructions.md +++ b/.github/instructions/database.instructions.md @@ -8,13 +8,12 @@ applyTo: 'src/mavedb/models/**/*.py,alembic/**/*.py' ## Model Structure ### Base Class -All models use `@as_declarative()` with a shared `Base` class providing: -- `id`: Integer primary key (auto-increment) -- `creation_date` / `modification_date`: Auto-managed timestamps -- `created_by_id` / `modified_by_id`: Foreign keys to users table +All models subclass the `@as_declarative()` `Base` in `src/mavedb/db/base.py`. It declares no columns; it only defaults `__tablename__` to the lowercased class name. Table models declare their own `id` primary key and set `__tablename__` explicitly; view models map `__table__` to a view instead. ### Common Columns Most resource models include: +- `creation_date` / `modification_date`: `Date` columns defaulting to today (`modification_date` updates on write) +- `created_by_id` / `modified_by_id`: Foreign keys to the users table (user-owned resources: experiment sets, experiments, score sets, calibrations, calibration controls, collections) - `urn`: Unique string identifier (temporary `tmp:` or permanent `urn:mavedb:...`) - `private`: Boolean flag for visibility control - `approved`: Boolean for moderation status @@ -50,6 +49,16 @@ class ScoreSetPublicationIdentifierAssociation(Base): primary = Column(Boolean, default=False) ``` +### Valid-Time Versioning (`ValidTime`) +Rows that change over time mix in `ValidTime` (`src/mavedb/db/mixins.py`) instead of being updated or deleted in place: +- `valid_from` / `valid_to`: a row is live while `valid_to` is NULL; history is the half-open window `[valid_from, valid_to)`. +- Reads: filter explicitly with `Model.current` or `Model.live_at(as_of)` (`None` means current; also works on `aliased()` classes). Never assume a query returns only live rows. +- Replacing a row: `supersede_with` (one row) or `supersede_live_where` (bulk) retire the old rows and stamp the new rows' `valid_from` with the same timestamp, so the handoff has no gap. Never retire and then insert separately. +- Withdrawing with no successor: `retire` (one row) or `retire_live_where` (bulk UPDATE). +- `__retire_cascade__`: relationship names whose live `ValidTime` children are retired when the parent is (one level deep). `supersede_live_where` raises on a class that declares it; use `supersede_with` per row. +- Each table adds a partial unique index over its natural key `WHERE valid_to IS NULL`, so a second live row raises `IntegrityError`. +- Content-addressed, immutable rows (e.g. alleles) do not use the mixin. + ### Enum Columns Use non-native enums stored as strings: ```python @@ -121,4 +130,4 @@ if item.private and not user_has_access(user, item): ``` ## Materialized Views -Used for pre-computed aggregations. Refresh via worker cron jobs defined in `src/mavedb/worker/settings.py`. +Used for pre-computed aggregations. Refresh via worker cron jobs defined in `src/mavedb/worker/jobs/registry.py`. diff --git a/.github/instructions/worker.instructions.md b/.github/instructions/worker.instructions.md index 16b6e19f0..55a31abeb 100644 --- a/.github/instructions/worker.instructions.md +++ b/.github/instructions/worker.instructions.md @@ -86,7 +86,7 @@ Always return using factory methods: ```python return JobExecutionOutcome.succeeded(data={"variants_created": count}) -return JobExecutionOutcome.failed(reason="No mapped variants found", data={...}) +return JobExecutionOutcome.failed(reason="No mapping records found", data={...}) return JobExecutionOutcome.failed(reason="HGVS parse error", failure_category=FailureCategory.DATA_ERROR) return JobExecutionOutcome.skipped(data={"reason": "Feature disabled"}) # For unhandled exceptions: let them propagate — the decorator catches and creates .errored() @@ -127,8 +127,8 @@ Parameters with `None` values in pipeline definitions are filled at runtime from ## Pipeline Lifecycle (Brief) -1. Router calls `PipelineFactory.create_pipeline()` → creates `Pipeline`, `JobRun`, and `JobDependency` records -2. Router enqueues the `start_pipeline` entrypoint job in ARQ +1. Router or script calls `enqueue_pipeline_for_score_set()` (`lib/workflow/kickoff.py`) → `PipelineFactory.create_pipeline()` creates `Pipeline`, `JobRun`, and `JobDependency` records +2. The same call enqueues the `start_pipeline` entrypoint job in ARQ 3. `start_pipeline` runs → its `@with_pipeline_management` decorator starts the pipeline and calls `coordinate_pipeline()` 4. `coordinate_pipeline()` finds PENDING jobs whose dependencies are met → marks them QUEUED → enqueues in ARQ 5. Each job runs → after completion, its decorator calls `coordinate_pipeline()` again diff --git a/README.md b/README.md index 20e7f10ec..c03bc35d7 100644 --- a/README.md +++ b/README.md @@ -67,9 +67,9 @@ To stop the application when it is running as a daemon, run docker-compose -f docker-compose-local.yml down ``` -`docker-compose-local.yml` configures four containers: one for the API server, one for the PostgreSQL database, one for the +`docker-compose-local.yml` configures four containers: one for the API server, one for the PostgreSQL 15 database, one for the worker node and one for the Redis cache which acts as the job queue for the worker node. The worker node stores data in a Docker -volume named `mavedb-redis` and the database stores data in a Docker volume named `mavedb-data`. Both these volumes will persist +volume named `mavedb-redis` and the database stores data in a Docker volume named `mavedb-data-15`. Both these volumes will persist after running `docker-compose down`. **Notes** diff --git a/src/mavedb/scripts/resources/CHANGELOG.md b/src/mavedb/scripts/resources/CHANGELOG.md index 7e9739524..7f3ab1858 100644 --- a/src/mavedb/scripts/resources/CHANGELOG.md +++ b/src/mavedb/scripts/resources/CHANGELOG.md @@ -36,6 +36,19 @@ given archive was generated. ### Changed +- **`va/{urn}.va.ndjson` follows VA-Spec 1.1.0 and Cat-VRS 1.1.1.** Field renames + consumers must follow: a study result's `focusVariant` is `focus`; a proposition's + `subjectVariant` is `subject`, and `objectCondition` / `objectSequenceFeature` are + `object`. The functional classification system is + `ga4gh-gkm-term:experimental-var-func-impact-classification`. +- The VA-Spec proposition subject is a Cat-VRS `CategoricalVariant` over the measured + change's other representations (its genomic or transcript twin and its protein + consequence) where it has any, with `mappings` to their ClinGen, gnomAD and ClinVar + records. +- `mavedb.post_mapped_hgvs_g` / `_c` / `_p` describe the measured variant and its + projections to the other levels, with the genomic string normalized. Score sets + assayed at the protein level fill only `mavedb.post_mapped_hgvs_p`, and their + `gnomad.*` and `clinvar.*` columns are `NA`: both are nucleotide-level resources. - gnomAD frequencies are now reported from whichever release each allele's current record came from, rather than only from the single release the deployment serves. An allele not covered by a newer release previously reported `NA` for every gnomAD diff --git a/src/mavedb/scripts/resources/README.md b/src/mavedb/scripts/resources/README.md index 7b8582b50..4a42b2958 100644 --- a/src/mavedb/scripts/resources/README.md +++ b/src/mavedb/scripts/resources/README.md @@ -143,6 +143,13 @@ Columns are grouped by a namespace prefix. The groups below always appear: | `gnomad.gnomad_version` | gnomAD release *this row's* frequencies were drawn from (e.g. `v4.1`). Varies by row — see Caveats | | `clingen.clingen_allele_id` | ClinGen Allele Registry CA identifier (e.g. `CA12345`) | +The `mavedb.post_mapped_hgvs_*` columns describe the measured variant at each level: the measured HGVS +plus its projections to the other levels. The genomic string is normalized, so for a genomic assay it can +differ from `mavedb.post_mapped_hgvs_at_assay_level`, and a projected inversion is written as a `delins`. +Score sets assayed at the protein level fill only `mavedb.post_mapped_hgvs_p`: a protein change has +no single coding or genomic encoding. gnomAD and ClinVar are nucleotide-level resources, so the `gnomad.*` +and `clinvar.*` columns are `NA` for protein-level variants. + Two further groups vary by score set, because they exist only where MaveDB holds the underlying data. Read the header rather than assuming a fixed column set. @@ -182,9 +189,11 @@ strength that the GA4GH VA-Spec has no equivalent for. The same variant's record therefore reports `moderate` where this file reports `MODERATE_PLUS`. Variants that could not be mapped, or for which a specific annotation is unavailable, will have -`NA` in the corresponding column. For multi-allelic variants (haplotypes), `mavedb.*` HGVS columns -will be `NA` because a single combined HGVS string cannot currently be derived. This may be updated in -a future release. +`NA` in the corresponding column. For a cis-phased multi-variant (haplotype), the measured-level HGVS is +a single combined expression (e.g. `NC_…:g.[123A>G;125T>C]`) when every member has an HGVS expression +on the same sequence, and `NA` otherwise. Haplotypes mapped with VRS 1.x have no VRS allele: they are absent from +`vrs/{urn}.vrs.ndjson` and `va/{urn}.va.ndjson`, `mavedb.post_mapped_vrs_id` and the projected levels +are `NA`, and the measured level carries only the HGVS their original mapping recorded, if any. ### `vrs/{urn}.vrs.ndjson` @@ -309,9 +318,10 @@ mapping pipeline.** All files for a given score set share the same variant identifier: - In CSV files: the `accession` column (e.g. `urn:mavedb:00000001-a-1#42`) -- In `mapped-variants.json`: the `variantUrn` field +- In `vrs.ndjson` and `va.ndjson`: the `variant_urn` field +- In the legacy `mapped-variants.json`: the `variantUrn` field -To combine scores with annotations or with VRS data, join on `accession` = `variantUrn`. +To combine scores with annotations or with VRS data, join on `accession` = `variant_urn`. ### Linking files back to metadata @@ -346,8 +356,8 @@ score_set = next( - Only **published**, **CC0-licensed** data is included. Datasets with other licenses are not present in this dump even if they are publicly visible on MaveDB. -- Annotation files (`.annotations.csv`), the legacy `.mapped-variants.json` files, and - VA-Spec files (`.va.ndjson`) are **only present for score sets that have been processed by the +- Annotation files (`.annotations.csv`), VRS files (`.vrs.ndjson`), the legacy + `.mapped-variants.json` files, and VA-Spec files (`.va.ndjson`) are **only present for score sets that have been processed by the MaveDB variant mapping pipeline**. Score sets that have not yet been mapped, or for which mapping failed entirely, will not have these files. - `annotations.csv` includes **research-use-only** calibrations, flagged by `research_use_only`; the @@ -358,8 +368,8 @@ score_set = next( functional evidence only, not a full clinical ACMG determination. - Mapping is applied per variant within a score set. A score set that has completed the mapping pipeline may still contain individual variants with failed mappings. Those variants have `NA` in - all `mavedb.*`, `vep.*`, `gnomad.*`, and `clingen.*` columns in the annotations CSV, and - `preMapped: null` / `postMapped: null` in the JSON. + all `mavedb.*`, `vep.*`, `gnomad.*`, and `clingen.*` columns in the annotations CSV, are omitted + from the `vrs/` and `va/` files, and have `preMapped: null` / `postMapped: null` in the legacy JSON. - This dump is a snapshot of MaveDB's **current** state, not a historical archive. The `mapped/` JSON files include only each variant's current mapping — superseded records from earlier mapping runs are not retained here. (Callers that need mapping history directly can ask the API for a past diff --git a/src/mavedb/worker/README.md b/src/mavedb/worker/README.md index 34c66ed0f..55ed05eab 100644 --- a/src/mavedb/worker/README.md +++ b/src/mavedb/worker/README.md @@ -60,16 +60,17 @@ worker/ │ ├── jobs/ # ── Business Layer ── │ ├── registry.py # Central registry of all job functions -│ ├── variant_processing/ # Variant creation and mapping jobs +│ ├── variant_processing/ # Variant creation, mapping and reverse translation jobs │ │ ├── creation.py # create_variants_for_score_set -│ │ └── mapping.py # map_variants_for_score_set +│ │ ├── mapping.py # map_variants_for_score_set +│ │ └── reverse_translation.py # reverse_translate_variants_for_score_set │ ├── external_services/ # Integration with external APIs │ │ ├── clingen.py # CAR and LDH submission +│ │ ├── clingen_cache.py # ClinGen allele cache warming │ │ ├── clinvar.py # ClinVar control refresh │ │ ├── gnomad.py # gnomAD variant linking -│ │ ├── hgvs.py # HGVS annotation │ │ ├── uniprot.py # UniProt mapping submission/polling -│ │ └── variant_translation.py # Variant translation population +│ │ └── vep.py # VEP consequence population │ ├── data_management/ # Database maintenance jobs │ │ └── views.py # Materialized view refresh │ ├── pipeline_management/ # Pipeline orchestration jobs @@ -81,6 +82,8 @@ worker/ │ └── constants.py # Job-level constants │ ├── lib/ # ── Infrastructure Layer ── +│ ├── projection.py # Measured projection groups (c./g./p. HGVS per mapping record) +│ ├── translation_ports.py # variant-annotation translation ports (coordinate translator, UTA) │ ├── decorators/ # Job/pipeline lifecycle decorators │ │ ├── job_management.py # @with_job_management │ │ ├── pipeline_management.py # @with_pipeline_management diff --git a/src/mavedb/worker/best_practices.md b/src/mavedb/worker/best_practices.md index 5e3906f9d..92b672617 100644 --- a/src/mavedb/worker/best_practices.md +++ b/src/mavedb/worker/best_practices.md @@ -66,9 +66,9 @@ return JobExecutionOutcome.succeeded(data={"variants_created": count}) ### Failed — a business-logic failure (not a bug) ```python # Missing data, validation failure, precondition not met -if not mapped_variants: +if not mapping_records: return JobExecutionOutcome.failed( - reason="No mapped variants found for score set", + reason="No mapping records found for score set", data={"score_set_id": score_set_id} ) ``` @@ -170,8 +170,11 @@ async def submit_to_external_service(ctx: dict, job_id: int, job_manager: JobMan ).one() # 3. Check preconditions - if not score_set.mapped_variants: - return JobExecutionOutcome.failed(reason="No mapped variants to submit") + has_mappings = job_manager.db.scalar( + select(exists().where(MappingRecord.score_set_id == score_set.id, MappingRecord.current)) + ) + if not has_mappings: + return JobExecutionOutcome.failed(reason="No mapping records to submit") # 4. Submit to the service (let exceptions propagate for service errors) result = await external_client.submit(score_set) diff --git a/src/mavedb/worker/job_registry.md b/src/mavedb/worker/job_registry.md index 952b4359d..44021e333 100644 --- a/src/mavedb/worker/job_registry.md +++ b/src/mavedb/worker/job_registry.md @@ -13,6 +13,7 @@ BACKGROUND_FUNCTIONS: List[Callable] = [ # Variant processing jobs create_variants_for_score_set, map_variants_for_score_set, + reverse_translate_variants_for_score_set, # External service jobs submit_score_set_mappings_to_car, submit_score_set_mappings_to_ldh, @@ -42,8 +43,8 @@ BACKGROUND_CRONJOBS: List[CronJob] = [ cron( cleanup_stalled_jobs, name="cleanup_stalled_jobs_cron", - minute={15, 45}, # Every 30 minutes - keep_result=timedelta(minutes=25).total_seconds(), + minute={5, 15, 25, 35, 45, 55}, # Every 10 minutes + keep_result=timedelta(minutes=9).total_seconds(), ), ] ``` @@ -239,7 +240,9 @@ class ArqWorkerSettings: redis_settings = RedisWorkerSettings functions = BACKGROUND_FUNCTIONS cron_jobs = BACKGROUND_CRONJOBS - job_timeout = 5 * 60 * 60 # 5 hours + max_jobs = MAX_JOBS # 2; limits event-loop starvation from sync DB calls + job_timeout = JOB_TIMEOUT_SECONDS # 24 hours + allow_abort_jobs = True # Lets cleanup_stalled_jobs abort a wedged RUNNING job ``` The lifecycle hooks (in `settings/lifecycle.py`) manage the ARQ context dict (`ctx`): diff --git a/src/mavedb/worker/jobs_overview.md b/src/mavedb/worker/jobs_overview.md index bfca70c6a..2f83e6154 100644 --- a/src/mavedb/worker/jobs_overview.md +++ b/src/mavedb/worker/jobs_overview.md @@ -28,8 +28,10 @@ Router (score_sets.py) │ • start_pipeline JobRun (entrypoint) │ • create_variants_for_score_set JobRun │ • map_variants_for_score_set JobRun (depends on create_variants) - │ • submit_to_car JobRun (depends on map_variants) - │ • link_gnomad JobRun (depends on submit_to_car) + │ • reverse_translate_variants_for_score_set JobRun (depends on map_variants) + │ • submit_to_car JobRun (depends on reverse_translate) + │ • warm_clingen_cache JobRun (depends on submit_to_car) + │ • link_gnomad JobRun (depends on warm_clingen_cache) │ • ... more annotation jobs with dependencies │ • JobDependency records linking them │ @@ -58,7 +60,7 @@ ARQ Worker picks up start_pipeline │ ├─ 6. ARQ picks up map_variants_for_score_set │ ├─ @with_pipeline_management runs job, marks SUCCEEDED - │ └─ coordinate_pipeline() finds submit_to_car, submit_uniprot, etc. → enqueue + │ └─ coordinate_pipeline() finds reverse_translate, submit_uniprot → enqueue │ ├─ 7... Continues until all jobs complete │ @@ -84,7 +86,7 @@ ARQ Cron Scheduler (or manual enqueue) └─ 3. No pipeline coordination (job has no pipeline_id) ``` -Example: `cleanup_stalled_jobs` runs every 30 minutes via ARQ cron to find and handle stuck jobs. +Example: `cleanup_stalled_jobs` runs every 10 minutes via ARQ cron to find and handle stuck jobs. ## Key Models @@ -104,6 +106,7 @@ The central record for every job execution: | `max_retries` | Maximum retry attempts (default: 3) | | `retry_count` | Current retry attempt count | | `progress_current/total/message` | Progress tracking fields | +| `progress_updated_at` | Heartbeat bumped on every committing progress update; `cleanup_stalled_jobs` reaps RUNNING jobs whose heartbeat goes stale | | `error_message/error_traceback` | Error details on failure | | `failure_category` | `FailureCategory` enum for retry classification | | `metadata_` | JSONB for retry history, result snapshots, etc. | diff --git a/src/mavedb/worker/pipeline_management.md b/src/mavedb/worker/pipeline_management.md index 12354746d..9064fdb74 100644 --- a/src/mavedb/worker/pipeline_management.md +++ b/src/mavedb/worker/pipeline_management.md @@ -242,13 +242,14 @@ When a job fails: ### Stalled Job Recovery -The `cleanup_stalled_jobs` cron job (runs every 30 minutes) catches jobs stuck in intermediate states: - -| State | Timeout | Action | -|-------|---------|--------| -| `QUEUED` | 10 minutes | Fail → retry if eligible | -| `RUNNING` | 60 minutes | Fail → retry if eligible | -| `PENDING` (in pipeline) | 30 minutes | Fail → retry if eligible | +The `cleanup_stalled_jobs` cron job (runs every 10 minutes) catches jobs stuck in intermediate states: + +| State | Stall signal | Action | +|-------|--------------|--------| +| `QUEUED` | Never started and absent from ARQ's Redis queue | Fail → retry if eligible | +| `RUNNING` | `progress_updated_at` idle > 30 minutes (`PROGRESS_STALL_MINUTES`), or running > 23.5 hours (`RUNNING_TIMEOUT_MINUTES`) | Abort the ARQ attempt; once it is confirmed dead, fail → retry if eligible. An abort that cannot be confirmed leaves the job in place. | +| `PENDING` (in pipeline) | Created > 5 minutes ago and runnable | Fail → retry if eligible | +| Pipeline (non-terminal, no active jobs) | Created > 5 minutes ago | Re-coordinate to settle the pipeline status | ## See Also From 2c9b19f16cd47e292dc628d6e3b5f77dc0868f8b Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 14 Oct 2026 22:28:02 -0700 Subject: [PATCH 243/252] fix(mapping): restore mapped coding and protein HGVS with projection groups Each mapping record's measured variant now has a projection group holding its genomic, coding and protein HGVS, written from the mapping alone so intronic, UTR, multi-codon and start/stop changes keep their coding and protein descriptions. - Genomic assays without a run transcript fall back to an NM_/ENST target accession, then the MANE Select transcript at the locus. - Genomic HGVS is normalized before projection; inversions are written as the equivalent delins, since VRS cannot represent them; identities keep their span. - A failed projection never replaces a group an earlier run wrote. - A changed standalone re-projection reports the reverse translation links it detaches as detached_rt_links. - Reverse translation joins its candidates to the measured group or its own candidate groups; serving reads mapped HGVS from the measured group. - VRS identifier search also matches pre-mapped identifiers and returns a deterministic order. - project_measured_variants script fills groups for backfilled records. --- ...ate_mapped_variants_to_allele_substrate.py | 3 +- .../c5e1a9d4b7f2_add_projection_groups.py | 75 ++ src/mavedb/lib/allele_detail.py | 25 +- src/mavedb/lib/alleles.py | 62 +- src/mavedb/lib/csv/fetch.py | 59 +- src/mavedb/lib/csv/specs.py | 5 +- src/mavedb/lib/score_set_variants.py | 212 +----- src/mavedb/lib/variant_detail.py | 29 +- src/mavedb/models/__init__.py | 1 + src/mavedb/models/mapping_record.py | 13 +- src/mavedb/models/mapping_record_allele.py | 24 +- src/mavedb/models/projection_group.py | 54 ++ src/mavedb/routers/variants.py | 15 +- .../scripts/audit_reverse_translation.py | 24 +- .../scripts/project_measured_variants.py | 108 +++ src/mavedb/view_models/lean_variant.py | 7 +- src/mavedb/view_models/variant.py | 8 +- src/mavedb/view_models/variant_detail.py | 4 + src/mavedb/worker/jobs/registry.py | 4 +- .../variant_processing/reverse_translation.py | 382 ++++++---- src/mavedb/worker/lib/projection.py | 282 +++++++ src/mavedb/worker/lib/translation_ports.py | 61 ++ tests/helpers/util/annotation.py | 40 +- tests/helpers/util/score_set.py | 12 +- tests/lib/annotation/test_context.py | 16 +- tests/lib/csv/conftest.py | 11 +- tests/lib/csv/test_variant.py | 52 +- tests/lib/test_allele_detail.py | 13 +- tests/lib/test_score_set_variants.py | 135 +++- tests/lib/test_variant_detail.py | 152 +++- tests/routers/test_allele.py | 11 +- tests/routers/test_query_scaling.py | 9 +- tests/routers/test_score_set.py | 9 +- tests/routers/test_variant.py | 31 +- tests/routers/test_variant_annotation.py | 113 +++ .../scripts/test_audit_reverse_translation.py | 78 +- tests/scripts/test_backfill_campaign.py | 4 +- .../test_reverse_translation.py | 704 +++++++++++++++--- tests/worker/lib/test_projection.py | 209 ++++++ 39 files changed, 2384 insertions(+), 672 deletions(-) create mode 100644 alembic/versions/c5e1a9d4b7f2_add_projection_groups.py create mode 100644 src/mavedb/models/projection_group.py create mode 100644 src/mavedb/scripts/project_measured_variants.py create mode 100644 src/mavedb/worker/lib/projection.py create mode 100644 tests/worker/lib/test_projection.py diff --git a/alembic/manual_migrations/migrate_mapped_variants_to_allele_substrate.py b/alembic/manual_migrations/migrate_mapped_variants_to_allele_substrate.py index 71f4aeda2..3d2de474d 100644 --- a/alembic/manual_migrations/migrate_mapped_variants_to_allele_substrate.py +++ b/alembic/manual_migrations/migrate_mapped_variants_to_allele_substrate.py @@ -54,7 +54,8 @@ ``transcript``) NULL even when a real string was recoverable — silently starving any downstream pass that resolves a transcript from it. * **MappingRecordAllele** — one ``is_authoritative=True`` link per record on the - record's validity window. ``projection_group`` stays NULL (no fan-out). + record's validity window. ``projection_group`` stays NULL; ``mavedb.scripts.project_measured_variants`` + writes the measured projection group afterwards. * **Annotation timelines** — reconstructed across *every* mapping version, not just the live one: ``gnomad_variants`` → ``gnomad_allele_links``, ``clinical_controls`` → ``clinvar_allele_links``, VEP columns → ``vep_allele_consequences``. Each mapping version's bundled annotations are captured diff --git a/alembic/versions/c5e1a9d4b7f2_add_projection_groups.py b/alembic/versions/c5e1a9d4b7f2_add_projection_groups.py new file mode 100644 index 000000000..b3762678e --- /dev/null +++ b/alembic/versions/c5e1a9d4b7f2_add_projection_groups.py @@ -0,0 +1,75 @@ +"""add projection groups + +Revision ID: c5e1a9d4b7f2 +Revises: b4d8f2a6c1e3 +Create Date: 2026-10-14 + +Makes ``mapping_record_alleles.projection_group`` a foreign key to a ``projection_groups`` row that holds +the group's coding, genomic and protein HGVS. A group's levels need not all have an allele: an intronic +coding projection has no VRS representation but is still the variant's coding coordinate. + +The composite foreign key on ``(projection_group, mapping_record_id)`` keeps a link inside its own record's +groups. Existing group numbers name no row, so they are cleared; re-run the projection step and reverse +translation to rebuild them. Production has none after the substrate backfill, which writes no groups. +""" + +import sqlalchemy as sa + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "c5e1a9d4b7f2" +down_revision = "b4d8f2a6c1e3" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "projection_groups", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("mapping_record_id", sa.Integer(), nullable=False), + sa.Column("score_set_id", sa.Integer(), nullable=False), + sa.Column("hgvs_c", sa.String(), nullable=True), + sa.Column("hgvs_g", sa.String(), nullable=True), + sa.Column("hgvs_p", sa.String(), nullable=True), + sa.Column("valid_from", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + sa.Column("valid_to", sa.DateTime(timezone=True), nullable=True), + sa.ForeignKeyConstraint( + ["mapping_record_id", "score_set_id"], + ["mapping_records.id", "mapping_records.score_set_id"], + name="fk_projection_groups_mapping_record_score_set", + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index("ix_projection_groups_mapping_record_id", "projection_groups", ["mapping_record_id"]) + # Target of mapping_record_alleles' composite foreign key, which keeps a link inside its own record's groups. + op.create_index( + "uq_projection_groups_id_mapping_record", "projection_groups", ["id", "mapping_record_id"], unique=True + ) + + # Existing group numbers name no row. + mapping_record_alleles = sa.table("mapping_record_alleles", sa.column("projection_group", sa.Integer)) + op.execute( + mapping_record_alleles.update() + .where(mapping_record_alleles.c.projection_group.isnot(None)) + .values(projection_group=None) + ) + op.create_foreign_key( + "fk_mapping_record_alleles_projection_group", + "mapping_record_alleles", + "projection_groups", + ["projection_group", "mapping_record_id"], + ["id", "mapping_record_id"], + ondelete="CASCADE", + ) + op.create_index("ix_mapping_record_alleles_projection_group", "mapping_record_alleles", ["projection_group"]) + + +def downgrade() -> None: + op.drop_index("ix_mapping_record_alleles_projection_group", table_name="mapping_record_alleles") + op.drop_constraint("fk_mapping_record_alleles_projection_group", "mapping_record_alleles", type_="foreignkey") + op.drop_index("uq_projection_groups_id_mapping_record", table_name="projection_groups") + op.drop_index("ix_projection_groups_mapping_record_id", table_name="projection_groups") + op.drop_table("projection_groups") diff --git a/src/mavedb/lib/allele_detail.py b/src/mavedb/lib/allele_detail.py index 5aa70f193..33a6d2429 100644 --- a/src/mavedb/lib/allele_detail.py +++ b/src/mavedb/lib/allele_detail.py @@ -38,7 +38,7 @@ from mavedb.lib.cat_vrs import member_label from mavedb.lib.vrs import vrs_object_from_mapped_variant from mavedb.models.allele import Allele -from mavedb.models.enums.sequence_level import SequenceLevel +from mavedb.models.enums.sequence_level import NUCLEOTIDE_LEVELS, SequenceLevel from mavedb.models.mapping_record_allele import MappingRecordAllele @@ -73,10 +73,12 @@ def _focus_projection_pairs( as_of: Optional[datetime], score_set_ids: Optional[Collection[int]], ) -> dict[str, str]: - """A symmetric ``digest -> digest`` map pairing each focus allele with its c↔g projection. + """A ``digest -> digest`` map pairing each focus allele with its projection. - A ``projection_group`` is one c↔g pair, local to a single mapping record. The projection is the ≤1 - *other* allele sharing the same ``projection_group`` as the focus allele. Reading any one live link + A ``projection_group`` is one variant at each level, local to a single mapping record. A nucleotide + focus pairs with the ≤1 *other* nucleotide allele in its group (c↔g, symmetric). A protein focus in a + measured group pairs with that record's measured allele, one way only, so the measured allele keeps its + nucleotide pairing. Reading any one live link is safe: the focus holds one live link per record it appears in, and each record re-encodes the same change onto the same content-addressed digests, so the pairing is record-independent (assuming those records agree on genome assembly). @@ -88,6 +90,7 @@ def _focus_projection_pairs( for focus in focus_alleles: if focus.vrs_digest is None: continue + nucleotide = focus.level in NUCLEOTIDE_LEVELS # limit(1) rather than one_or_none(), which would raise for any allele measured more than once. link_query = ( @@ -103,7 +106,7 @@ def _focus_projection_pairs( if link is None: continue - projection_digests = db.scalars( + partners = ( select(Allele.vrs_digest) .select_from(MappingRecordAllele) .join(Allele, Allele.id == MappingRecordAllele.allele_id) @@ -111,11 +114,17 @@ def _focus_projection_pairs( .where(MappingRecordAllele.projection_group == link.projection_group) .where(MappingRecordAllele.allele_id != focus.id) .where(MappingRecordAllele.live_at(as_of)) - ).all() - for projection in projection_digests: + ) + partners = ( + partners.where(Allele.level.in_(NUCLEOTIDE_LEVELS)) + if nucleotide + else partners.where(MappingRecordAllele.is_authoritative) + ) + for projection in db.scalars(partners).all(): if projection is not None: pairs[focus.vrs_digest] = projection - pairs.setdefault(projection, focus.vrs_digest) + if nucleotide: + pairs.setdefault(projection, focus.vrs_digest) return pairs diff --git a/src/mavedb/lib/alleles.py b/src/mavedb/lib/alleles.py index 29ce9a758..01a208043 100644 --- a/src/mavedb/lib/alleles.py +++ b/src/mavedb/lib/alleles.py @@ -13,7 +13,7 @@ from datetime import datetime from typing import Optional, TypeVar -from sqlalchemy import select +from sqlalchemy import and_, select from sqlalchemy.orm import Session, joinedload from mavedb.lib.permissions import Action, has_permission @@ -37,8 +37,8 @@ @dataclass(frozen=True) class LiveRecordLinks: - """A variant's live (or as-of) ``MappingRecord`` and that record's live ``MappingRecordAllele`` links, - each with its ``allele`` eagerly loaded.""" + """A variant's live (or as-of) ``MappingRecord``, that record's live ``MappingRecordAllele`` links (each with + its ``allele`` and projection ``group`` eagerly loaded).""" record: MappingRecord links: list[MappingRecordAllele] @@ -114,7 +114,7 @@ def get_live_record_allele_links( select(MappingRecordAllele) .where(MappingRecordAllele.mapping_record_id.in_(links_by_record)) .where(MappingRecordAllele.live_at(as_of)) - .options(joinedload(MappingRecordAllele.allele)) + .options(joinedload(MappingRecordAllele.allele), joinedload(MappingRecordAllele.group)) ): links_by_record[link.mapping_record_id].append(link) @@ -123,19 +123,22 @@ def get_live_record_allele_links( def find_variants_by_vrs_identifier( db: Session, identifier: str, *, as_of: Optional[datetime] = None -) -> list[tuple[Variant, Allele]]: - """Resolve a GA4GH VRS identifier to the variants whose mapping links an allele bearing it. +) -> list[tuple[Variant, Optional[Allele]]]: + """Resolve a GA4GH VRS identifier to the variants it identifies, before or after mapping. - Matches ``identifier`` against a deduplicated allele's ``vrs_digest``, then walks ``Allele → - MappingRecordAllele → MappingRecord → Variant``. Because alleles are deduplicated by ``vrs_digest`` - and shared across variants/score sets, one identifier can resolve to several variants; the - paired allele carries the ClinGen id + level the caller surfaces alongside the URN. + Matches ``identifier`` against both sides of a mapping: + + * a deduplicated allele's ``vrs_digest`` (the reference-level, post-mapped variation), walking + ``Allele → MappingRecordAllele → MappingRecord → Variant``. Alleles are shared across variants and + score sets, so one identifier can resolve to several variants; each is paired with the matched allele. + * a mapping record's ``vrs_digest`` (the pre-mapped variation on the submitted target sequence). Each + such variant is paired with its measured (authoritative) allele, or ``None`` when it did not map. The live link + record set (``valid_to IS NULL`` / ``as_of``) is used to filter results. Returns distinct - ``(variant, matched_allele)`` pairs. Include an ``as_of`` timestamp to reconstruct the set as it stood - at a past instant. + ``(variant, allele)`` pairs ordered by score set, then variant number, then allele. Include an ``as_of`` + timestamp to reconstruct the set as it stood at a past instant. """ - stmt = ( + post_mapped = ( select(Variant, Allele) .select_from(Allele) .join(MappingRecordAllele, MappingRecordAllele.allele_id == Allele.id) @@ -147,8 +150,39 @@ def find_variants_by_vrs_identifier( # Callers check read permission on each match's score set; load them here rather than one per match. .options(joinedload(Variant.score_set)) ) + pre_mapped = ( + select(Variant, Allele) + .select_from(MappingRecord) + .join(Variant, Variant.id == MappingRecord.variant_id) + .outerjoin( + MappingRecordAllele, + and_( + MappingRecordAllele.mapping_record_id == MappingRecord.id, + MappingRecordAllele.is_authoritative, + MappingRecordAllele.live_at(as_of), + ), + ) + .outerjoin(Allele, Allele.id == MappingRecordAllele.allele_id) + .where(MappingRecord.vrs_digest == identifier) + .where(MappingRecord.live_at(as_of)) + .options(joinedload(Variant.score_set)) + ) - return [(row[0], row[1]) for row in db.execute(stmt).unique().all()] + # An accession-based target's pre- and post-mapped variations can be the same, so both queries may + # return a pair. + matches: dict[tuple[int, Optional[int]], tuple[Variant, Optional[Allele]]] = {} + for stmt in (post_mapped, pre_mapped): + for variant, allele in db.execute(stmt).unique().tuples(): + matches.setdefault((variant.id, allele.id if allele is not None else None), (variant, allele)) + return sorted( + matches.values(), + key=lambda match: ( + match[0].score_set_id, + match[0].variant_number or 0, + match[0].id, + match[1].id if match[1] is not None else 0, + ), + ) def get_allele_translations( diff --git a/src/mavedb/lib/csv/fetch.py b/src/mavedb/lib/csv/fetch.py index 6f0a41de4..60d5f2f7f 100644 --- a/src/mavedb/lib/csv/fetch.py +++ b/src/mavedb/lib/csv/fetch.py @@ -8,21 +8,20 @@ from datetime import datetime from typing import Any, Optional, Sequence -from sqlalchemy import Select, and_, func, select -from sqlalchemy.orm import Session, aliased, selectinload +from sqlalchemy import Select, and_, select +from sqlalchemy.orm import Session, selectinload from mavedb.lib.csv.namespaces import CLINVAR_DB_NAME from mavedb.lib.csv.specs import CsvMappedRow, namespace_spec from mavedb.lib.gnomad import GNOMAD_DB_NAME -from mavedb.lib.score_set_variants import get_protein_hgvs_by_record, mapped_hgvs_by_level from mavedb.models.allele import Allele from mavedb.models.clinical_control import ClinvarControl from mavedb.models.clinvar_allele_link import ClinvarAlleleLink -from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.models.gnomad_allele_link import GnomadAlleleLink from mavedb.models.gnomad_variant import GnomADVariant from mavedb.models.mapping_record import MappingRecord from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.projection_group import ProjectionGroup from mavedb.models.score_set import ScoreSet from mavedb.models.variant import Variant from mavedb.models.vep_allele_consequence import VepAlleleConsequence @@ -68,14 +67,11 @@ def _substrate_query( *, score_set_id: Optional[int], variant_ids: Optional[Sequence[int]], as_of: Optional[datetime] ) -> Select[Any]: """The base per-variant query: each variant LEFT-joined to its live mapping record, its authoritative - (measured) allele (digest / ClinGen id / VEP consequence), and the projection-group nucleotide - partner — mirroring the lean whole-set view's join - (:func:`mavedb.lib.score_set_variants.get_lean_score_set_variants`). LEFT joins with ``live_at`` in the - ON clause keep unmapped variants (with null mapped fields); a WHERE would collapse them out. The - protein slot and the gnomAD/ClinVar dimensions are resolved by separate fetches and stitched on by id. + (measured) allele (digest / ClinGen id / VEP consequence), and that allele's measured projection group + (the mapped HGVS at every level). LEFT joins with ``live_at`` in the ON clause keep unmapped variants + (with null mapped fields); a WHERE would collapse them out. The gnomAD/ClinVar dimensions are resolved by + separate fetches and stitched on by id. """ - projection_link = aliased(MappingRecordAllele) - projection_allele = aliased(Allele) query = ( select( Variant, @@ -86,10 +82,9 @@ def _substrate_query( Allele.vrs_digest, Allele.clingen_allele_id, VepAlleleConsequence.functional_consequence, - projection_allele.level.label("projection_level"), - # Exactly one of the projection's hgvs_g/hgvs_c is populated (it is a nucleotide allele), so - # coalesce yields its canonical string regardless of which nucleotide level it sits at. - func.coalesce(projection_allele.hgvs_g, projection_allele.hgvs_c).label("projection_hgvs"), + ProjectionGroup.hgvs_c, + ProjectionGroup.hgvs_g, + ProjectionGroup.hgvs_p, ) .outerjoin(MappingRecord, and_(MappingRecord.variant_id == Variant.id, MappingRecord.live_at(as_of))) .outerjoin( @@ -106,34 +101,21 @@ def _substrate_query( and_(VepAlleleConsequence.allele_id == Allele.id, VepAlleleConsequence.live_at(as_of)), ) .outerjoin( - projection_link, - and_( - projection_link.mapping_record_id == MappingRecord.id, - projection_link.projection_group == MappingRecordAllele.projection_group, - projection_link.is_authoritative.is_(False), - projection_link.live_at(as_of), - ), + ProjectionGroup, + and_(ProjectionGroup.id == MappingRecordAllele.projection_group, ProjectionGroup.live_at(as_of)), ) - .outerjoin(projection_allele, projection_allele.id == projection_link.allele_id) ) return _scope_query(query, score_set_id=score_set_id, variant_ids=variant_ids) -def _mapped_row(row: Any, protein_hgvs_by_record: dict[int, str]) -> CsvMappedRow: - """Assemble one variant's :class:`CsvMappedRow` from a substrate-query row: the canonical post-mapped - HGVS at each level (reconstructed via the shared projection rule) plus the assay-level string, VRS id, - VEP consequence, and ClinGen id off the authoritative allele.""" - slots = mapped_hgvs_by_level( - assay_level=SequenceLevel(row.assay_level) if row.assay_level else None, - assay_level_hgvs=row.hgvs_assay_level, - projection_level=row.projection_level, - projection_hgvs=row.projection_hgvs, - protein_hgvs=protein_hgvs_by_record.get(row.mapping_record_id), - ) +def _mapped_row(row: Any) -> CsvMappedRow: + """Assemble one variant's :class:`CsvMappedRow` from a substrate-query row: the measured projection + group's HGVS at each level plus the assay-level string, VRS id, VEP consequence, and ClinGen id off the + authoritative allele.""" return CsvMappedRow( - hgvs_g=slots.get(SequenceLevel.genomic.value), - hgvs_c=slots.get(SequenceLevel.cdna.value), - hgvs_p=slots.get(SequenceLevel.protein.value), + hgvs_g=row.hgvs_g, + hgvs_c=row.hgvs_c, + hgvs_p=row.hgvs_p, hgvs_assay_level=row.hgvs_assay_level, vrs_digest=row.vrs_digest, vep_functional_consequence=row.functional_consequence, @@ -264,7 +246,6 @@ def fetch_variant_csv_data( rows = _restore_variant_id_order(rows, variant_ids, key=lambda row: row.Variant.id) # Protein slot: the standalone protein subquery over the whole set, stitched back by record id. - protein_hgvs_by_record = get_protein_hgvs_by_record(db, score_set_id, variant_ids=variant_ids, as_of=as_of) # The authoritative allele ids on this page — the join key for the batch gnomAD/ClinVar fetches. allele_ids = [row.allele_id for row in rows if row.allele_id is not None] gnomad_by_allele = _gnomad_by_allele(db, allele_ids, as_of=as_of) if need_gnomad else {} @@ -279,7 +260,7 @@ def fetch_variant_csv_data( for row in rows: variants.append(row.Variant) allele_id = row.allele_id - mappings.append(_mapped_row(row, protein_hgvs_by_record) if row.mapping_record_id is not None else None) + mappings.append(_mapped_row(row) if row.mapping_record_id is not None else None) if gnomad_data is not None: gnomad_data.append(gnomad_by_allele.get(allele_id) if allele_id is not None else None) if clinvar_per_variant is not None: diff --git a/src/mavedb/lib/csv/specs.py b/src/mavedb/lib/csv/specs.py index bd37aa240..db89b5234 100644 --- a/src/mavedb/lib/csv/specs.py +++ b/src/mavedb/lib/csv/specs.py @@ -15,9 +15,8 @@ class CsvMappedRow: """The mapping-derived CSV fields for one variant, resolved from its live mapping record's authoritative allele (post-mapped HGVS per level, assay-level HGVS, VRS digest, VEP consequence, - ClinGen id). HGVS strings are already resolved to their canonical per-level form by the fetch layer - (see ``mavedb.lib.score_set_variants.mapped_hgvs_by_level``) — no raw VRS payload to fall back on - parsing here, unlike the pre-allele-graph ``MappedVariant`` substrate.""" + ClinGen id). The per-level HGVS strings are the measured projection group's — no raw VRS payload to + fall back on parsing here, unlike the pre-allele-graph ``MappedVariant`` substrate.""" hgvs_g: Optional[str] hgvs_c: Optional[str] diff --git a/src/mavedb/lib/score_set_variants.py b/src/mavedb/lib/score_set_variants.py index 319e3928a..bb1952608 100644 --- a/src/mavedb/lib/score_set_variants.py +++ b/src/mavedb/lib/score_set_variants.py @@ -9,10 +9,10 @@ assay fills all three slots; a protein assay fills only ``protein`` (the c/g fan-out is ambiguous, so no pick is fabricated). This is the canonical projection only, never the reverse-translation fan-out. -The nucleotide pair comes from the authoritative (measured) allele plus its ``projection_group`` -partner at the other nucleotide level (a group has ≤2 members, so this stays one row per variant). -Where ``projection_group`` is unpopulated (pre-reverse-translation data), that slot is ``None`` until -reprocessed. Does not assemble Cat-VRS. +The mapped slots come from the record's measured projection group (the group its authoritative link points +at), which describes the measured variant at every level it has, including levels with no VRS allele +(an intronic coding projection). Where the group is not yet written, only the measured slot is filled. +Does not assemble Cat-VRS. ``as_of`` reconstructs the annotation layer at a past instant; submitted HGVS and scores are immutable and unaffected. Defaults to currently-live rows. @@ -20,9 +20,9 @@ from dataclasses import dataclass from datetime import datetime -from typing import Optional, Sequence +from typing import Optional -from sqlalchemy import and_, func, select, true +from sqlalchemy import and_, select, true from sqlalchemy.orm import Session, aliased from mavedb.lib.hgvs import parse_simple_substitution @@ -31,6 +31,7 @@ from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.models.mapping_record import MappingRecord from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.projection_group import ProjectionGroup from mavedb.models.score_set import ScoreSet from mavedb.models.variant import Variant from mavedb.models.vep_allele_consequence import VepAlleleConsequence @@ -54,9 +55,9 @@ class HgvsField: class MappedTriple: """The canonical mapped (reference-frame) HGVS at each level, or ``None`` where absent. - A nucleotide assay populates all three slots (measured + the authoritative allele's - ``projection_group`` partner + protein apex); a protein assay populates only ``protein`` — the - c/g fan-out is ambiguous, so no pick is fabricated for ``cdna``/``genomic``. + A nucleotide assay populates the measured slot and whichever projections its measured projection + group holds (no protein for an intronic or UTR change); a protein assay populates only ``protein`` — + the c/g fan-out is ambiguous, so no pick is fabricated for ``cdna``/``genomic``. ``cdna`` is the level-invariant field: populated whether the assay was measured at ``cdna`` or ``genomic``, so it's the field to key cross-assay searches on. @@ -107,117 +108,19 @@ def _hgvs_field_for_str(hgvs: Optional[str]) -> Optional[HgvsField]: return HgvsField(hgvs=hgvs, position=block.position, ref=block.ref, alt=block.alt) -def mapped_hgvs_by_level( - *, - assay_level: Optional[SequenceLevel], - assay_level_hgvs: Optional[str], - projection_level: Optional[str], - projection_hgvs: Optional[str], - protein_hgvs: Optional[str], -) -> dict[str, Optional[str]]: - """The canonical mapped HGVS string at each populated level, keyed by ``SequenceLevel`` value. +def mapped_triple(*, hgvs_c: Optional[str], hgvs_g: Optional[str], hgvs_p: Optional[str]) -> MappedTriple: + """Wrap a measured projection group's HGVS strings as a :class:`MappedTriple`. - - Measured slot ← ``assay_level_hgvs``. - - Other nucleotide slot ← the authoritative link's ``projection_group`` partner (nucleotide assays - only, and only where the pairing is recorded). - - ``protein`` slot ← the separate protein-apex subquery. - - A protein assay fills only ``protein``. An unmapped variant (``assay_level is None``) yields an - empty mapping. + The group describes the measured variant at every level it has, the measured level included: a + protein assay's group holds only ``hgvs_p``, and a nucleotide assay's holds its normalized measured + string and the projections. A record whose group is not yet written (between mapping and the + projection step) gets an empty triple. """ - slots: dict[str, Optional[str]] = {} - if assay_level == SequenceLevel.protein: - slots[SequenceLevel.protein.value] = assay_level_hgvs - elif assay_level in (SequenceLevel.cdna, SequenceLevel.genomic): - slots[assay_level.value] = assay_level_hgvs - slots[SequenceLevel.protein.value] = protein_hgvs - # The other nucleotide level, from the projection_group partner (null where unpopulated). - if projection_level is not None: - slots[projection_level] = projection_hgvs - - return slots - - -def _mapped_triple( - *, - assay_level: Optional[SequenceLevel], - assay_level_hgvs: Optional[str], - projection_level: Optional[str], - projection_hgvs: Optional[str], - protein_hgvs: Optional[str], -) -> MappedTriple: - """Assemble the canonical :class:`MappedTriple` for one variant, wrapping each slot from - :func:`mapped_hgvs_by_level` as an :class:`HgvsField`.""" - slots = mapped_hgvs_by_level( - assay_level=assay_level, - assay_level_hgvs=assay_level_hgvs, - projection_level=projection_level, - projection_hgvs=projection_hgvs, - protein_hgvs=protein_hgvs, - ) return MappedTriple( - genomic=_hgvs_field_for_str(slots.get(SequenceLevel.genomic.value)), - cdna=_hgvs_field_for_str(slots.get(SequenceLevel.cdna.value)), - protein=_hgvs_field_for_str(slots.get(SequenceLevel.protein.value)), - ) - - -def get_protein_hgvs_by_record( - db: Session, - score_set_id: Optional[int] = None, - *, - variant_ids: Optional[Sequence[int]] = None, - as_of: Optional[datetime] = None, -) -> dict[int, str]: - """The canonical protein-level mapped HGVS for a set of live mapping records, keyed by - ``mapping_record.id``. Records with no protein projection (UTR/intronic) are absent from the map. - Used by the CSV export. - - Exactly one of *score_set_id* (every variant in one score set) or *variant_ids* (an explicit set, - which may span several score sets — the variant-level CSV's cross-set equivalent measurements) is - required. - - Runs as its own query rather than a join: the ``DISTINCT ON`` cardinality is opaque to the planner, - so a join lands it on the inner side of a nested loop and rescans it once per variant — O(N^2), and - historically ~97% of the lean endpoint's runtime. Stitching by ``mapping_record_id`` in Python keeps - both queries O(N). - """ - if (score_set_id is None) == (variant_ids is None): - raise ValueError("exactly one of score_set_id or variant_ids must be provided") - - prot_link = aliased(MappingRecordAllele) - prot_allele = aliased(Allele) - prot_record = aliased(MappingRecord) - prot_variant = aliased(Variant) - protein_statement = ( - select( - prot_link.mapping_record_id.label("mapping_record_id"), - prot_allele.hgvs_p.label("protein_hgvs"), - ) - # DISTINCT ON -> at most one protein level row per record. - .distinct(prot_link.mapping_record_id) - # link -> allele: level and hgvs_p live on the *allele*. - .join(prot_allele, prot_allele.id == prot_link.allele_id) - # link -> record -> variant: the bridge to scoreset_id/variant_id, to scope the query. - .join(prot_record, prot_record.id == prot_link.mapping_record_id) - .join(prot_variant, prot_variant.id == prot_record.variant_id) - # Scope to this score set (or this explicit variant set) so the cost scales with the response. - .where( - prot_variant.score_set_id == score_set_id - if score_set_id is not None - else prot_variant.id.in_(variant_ids or []) - ) - # Live links only; a live link implies a live parent record (ValidTime invariant), so no - # separate prot_record.live_at check is needed. - .where(prot_link.live_at(as_of)) - # Only the protein-level allele. - .where(prot_allele.level == SequenceLevel.protein.value) - # DISTINCT ON needs ORDER BY to lead with its column; allele.id breaks ties (inert today — - # only one protein allele exists per record). - .order_by(prot_link.mapping_record_id, prot_allele.id) + genomic=_hgvs_field_for_str(hgvs_g), + cdna=_hgvs_field_for_str(hgvs_c), + protein=_hgvs_field_for_str(hgvs_p), ) - # Keyed by record id (a globally-unique PK -> the right protein row, no cross-set risk). - return {row.mapping_record_id: row.protein_hgvs for row in db.execute(protein_statement)} def get_lean_score_set_variants( @@ -232,33 +135,40 @@ def get_lean_score_set_variants( crosses a threshold, which is orders of magnitude slower when that table is not cached. A lateral subquery with a ``LIMIT`` cannot be flattened into the join tree, so each lookup stays a per-variant index probe whatever the planner's statistics say. Each lookup yields at most one row by invariant (one live - record per variant, one live authoritative link per record, at most one projection partner, one - protein allele); the ``ORDER BY`` only makes a violated invariant deterministic. + record per variant, one live authoritative link per record, which points at one group); the ``ORDER BY`` + only makes a violated invariant deterministic. """ # The variant's live mapping record. record = ( - select(MappingRecord.id, MappingRecord.assay_level, MappingRecord.hgvs_assay_level) + select(MappingRecord.id, MappingRecord.assay_level) .where(MappingRecord.variant_id == Variant.id, MappingRecord.live_at(as_of)) .order_by(MappingRecord.id) .limit(1) .lateral("record") ) - # The authoritative (measured) allele, with its VEP consequence. ``is_authoritative`` is tested as a - # bare boolean: ``IS TRUE`` would not match the partial unique index's ``WHERE is_authoritative``. + # The authoritative (measured) allele, with its VEP consequence and its measured projection group's + # HGVS. ``is_authoritative`` is tested as a bare boolean: ``IS TRUE`` would not match the partial unique + # index's ``WHERE is_authoritative``. authoritative_link = aliased(MappingRecordAllele) authoritative = ( select( - authoritative_link.projection_group, Allele.vrs_digest, Allele.clingen_allele_id, VepAlleleConsequence.functional_consequence, + ProjectionGroup.hgvs_c, + ProjectionGroup.hgvs_g, + ProjectionGroup.hgvs_p, ) .select_from(authoritative_link) .join(Allele, Allele.id == authoritative_link.allele_id) .outerjoin( VepAlleleConsequence, and_(VepAlleleConsequence.allele_id == Allele.id, VepAlleleConsequence.live_at(as_of)) ) + .outerjoin( + ProjectionGroup, + and_(ProjectionGroup.id == authoritative_link.projection_group, ProjectionGroup.live_at(as_of)), + ) .where( authoritative_link.mapping_record_id == record.c.id, authoritative_link.is_authoritative, @@ -269,47 +179,6 @@ def get_lean_score_set_variants( .lateral("authoritative") ) - # The authoritative link's projection_group partner: the canonical HGVS at the unmeasured nucleotide - # level. A NULL group (protein assay, or pre-reverse-translation data) matches nothing, so the slot - # degrades to None. - projection_link = aliased(MappingRecordAllele) - projection_allele = aliased(Allele) - projection = ( - select( - projection_allele.level.label("projection_level"), - # Exactly one of hgvs_g/hgvs_c is populated (it's a nucleotide allele); coalesce picks it. - func.coalesce(projection_allele.hgvs_g, projection_allele.hgvs_c).label("projection_hgvs"), - ) - .select_from(projection_link) - .join(projection_allele, projection_allele.id == projection_link.allele_id) - .where( - projection_link.mapping_record_id == record.c.id, - projection_link.projection_group == authoritative.c.projection_group, - ~projection_link.is_authoritative, - projection_link.live_at(as_of), - ) - .order_by(projection_link.id) - .limit(1) - .lateral("projection") - ) - - # The record's protein-level allele (the protein apex for a nucleotide assay). Absent for UTR/intronic. - protein_link = aliased(MappingRecordAllele) - protein_allele = aliased(Allele) - protein = ( - select(protein_allele.hgvs_p.label("protein_hgvs")) - .select_from(protein_link) - .join(protein_allele, protein_allele.id == protein_link.allele_id) - .where( - protein_link.mapping_record_id == record.c.id, - protein_link.live_at(as_of), - protein_allele.level == SequenceLevel.protein.value, - ) - .order_by(protein_allele.id) - .limit(1) - .lateral("protein") - ) - statement = ( select( Variant.urn, @@ -318,20 +187,17 @@ def get_lean_score_set_variants( Variant.hgvs_pro, Variant.hgvs_splice, record.c.assay_level, - record.c.hgvs_assay_level, authoritative.c.vrs_digest, authoritative.c.clingen_allele_id, authoritative.c.functional_consequence, - projection.c.projection_level, - projection.c.projection_hgvs, - protein.c.protein_hgvs, + authoritative.c.hgvs_c, + authoritative.c.hgvs_g, + authoritative.c.hgvs_p, ) .select_from(Variant) # LEFT joins so unmapped variants stay (null mapped fields) for the table + score histogram. .outerjoin(record, true()) .outerjoin(authoritative, true()) - .outerjoin(projection, true()) - .outerjoin(protein, true()) .where(Variant.score_set_id == score_set.id) # Natural table order: variant number (the integer after '#'), id breaks ties stably. .order_by(Variant.variant_number, Variant.id) @@ -348,13 +214,7 @@ def get_lean_score_set_variants( hgvs_pro=_hgvs_field_for_str(row.hgvs_pro), hgvs_splice=_hgvs_field_for_str(row.hgvs_splice), assay_level=_level(row.assay_level), - mapped=_mapped_triple( - assay_level=_level(row.assay_level), - assay_level_hgvs=row.hgvs_assay_level, - projection_level=row.projection_level, - projection_hgvs=row.projection_hgvs, - protein_hgvs=row.protein_hgvs, - ), + mapped=mapped_triple(hgvs_c=row.hgvs_c, hgvs_g=row.hgvs_g, hgvs_p=row.hgvs_p), ) for row in db.execute(statement) ] diff --git a/src/mavedb/lib/variant_detail.py b/src/mavedb/lib/variant_detail.py index c334ec155..e9b33e65a 100644 --- a/src/mavedb/lib/variant_detail.py +++ b/src/mavedb/lib/variant_detail.py @@ -23,7 +23,8 @@ from mavedb.lib.alleles import LiveRecordLinks, get_live_record_allele_links from mavedb.lib.cat_vrs import build_categorical_variant, is_projection_partner, member_label from mavedb.lib.score_calibrations import calibration_preference_key, classifications_by_variant -from mavedb.models.enums.sequence_level import SequenceLevel +from mavedb.lib.score_set_variants import MappedTriple, mapped_triple +from mavedb.models.enums.sequence_level import NUCLEOTIDE_LEVELS, SequenceLevel from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification from mavedb.models.score_set import ScoreSet from mavedb.models.variant import Variant @@ -97,6 +98,9 @@ class VariantDetail: # `annotations`), carrying level, HGVS, ClinGen id, and relation. molecular_representation: Optional[dict[str, Any]] mode: Optional[str] # CatVrsMode value — projection | reverse_translation + # The measured variant's mapped HGVS at every level, from its measured projection group; includes + # levels with no allele in `alleles` (an intronic coding projection). + mapped: MappedTriple alleles: dict[str, AlleleIdentity] # External annotations, keyed by VRS digest, joined to the Cat-VRS members / the alleles sidecar. @@ -205,6 +209,7 @@ def _assemble_variant_detail( # linked alleles seed the digest-keyed annotation map. links = record_links.links if record_links is not None else [] authoritative = next((link for link in links if link.is_authoritative), None) + measured_group = authoritative.group if authoritative is not None else None assay_level_digest = authoritative.allele.vrs_digest if authoritative is not None else None clingen_allele_id = authoritative.allele.clingen_allele_id if authoritative is not None else None @@ -231,12 +236,16 @@ def _assemble_variant_detail( # set that seeds `annotations`, so the two maps share keys). Carries level, reference-frame HGVS # (exactly one of hgvs_g/c/p is populated per allele), ClinGen id, and the member→defining relation. - # projection_group -> the digests sharing it, for within-record projection grouping. A link's - # projection is the ≤1 *other* digest in its group; the protein apex and any pre-RT link carry a - # NULL group, so they appear in no bucket and projection_of stays None for them. + # projection_group -> the nucleotide digests sharing it, for within-record c↔g pairing. A nucleotide link's + # projection is the ≤1 *other* nucleotide digest in its group. The protein consequence in the measured group + # pairs with the measured allele: it is the part of the measurement the score describes. group_digests: dict[int, list[str]] = {} for link in links: - if link.projection_group is not None and link.allele.vrs_digest is not None: + if ( + link.projection_group is not None + and link.allele.vrs_digest is not None + and link.allele.level in NUCLEOTIDE_LEVELS + ): group_digests.setdefault(link.projection_group, []).append(link.allele.vrs_digest) # Members are labelled relative to the measured (authoritative) allele. @@ -247,13 +256,14 @@ def _assemble_variant_detail( for link in links: allele = link.allele - # The other digest in this link's projection_group, if any (groups have ≤2 members). projection_of: Optional[str] = None - if link.projection_group is not None: + if link.projection_group is not None and allele.level in NUCLEOTIDE_LEVELS: projection_of = next( (digest for digest in group_digests.get(link.projection_group, []) if digest != allele.vrs_digest), None, ) + elif link.projection_group is not None and link.projection_group == focus_group and authoritative is not None: + projection_of = authoritative.allele.vrs_digest relation, derivation = ( (None, None) @@ -289,6 +299,11 @@ def _assemble_variant_detail( post_mapped=authoritative.allele.post_mapped if authoritative is not None else None, molecular_representation=molecular_representation, mode=mode, + mapped=mapped_triple( + hgvs_c=measured_group.hgvs_c if measured_group is not None else None, + hgvs_g=measured_group.hgvs_g if measured_group is not None else None, + hgvs_p=measured_group.hgvs_p if measured_group is not None else None, + ), alleles=alleles, annotations=annotations, is_current=superseding_score_set is None, diff --git a/src/mavedb/models/__init__.py b/src/mavedb/models/__init__.py index 38d111b74..534ff0ec5 100644 --- a/src/mavedb/models/__init__.py +++ b/src/mavedb/models/__init__.py @@ -24,6 +24,7 @@ "mapping_record_allele", "mondo_term", "pipeline", + "projection_group", "publication_identifier", "published_variant", "variant_annotation_view", diff --git a/src/mavedb/models/mapping_record.py b/src/mavedb/models/mapping_record.py index 78ad5f8d6..d0838ab27 100644 --- a/src/mavedb/models/mapping_record.py +++ b/src/mavedb/models/mapping_record.py @@ -13,6 +13,7 @@ if TYPE_CHECKING: from .mapping_record_allele import MappingRecordAllele + from .projection_group import ProjectionGroup from .target_gene_mapping import TargetGeneMapping from .variant import Variant @@ -97,9 +98,15 @@ def _transcript_expression(cls): cascade="all, delete-orphan", ) - # Retiring a record retires its live allele links too (both the authoritative link and any - # derived links a reverse-translation run attached). See ValidTime.__retire_cascade__. - __retire_cascade__ = ("allele_links",) + projection_groups: Mapped[list["ProjectionGroup"]] = relationship( + "ProjectionGroup", + back_populates="mapping_record", + cascade="all, delete-orphan", + ) + + # Retiring a record retires its live allele links and projection groups (the measured group and any + # candidate groups a reverse-translation run attached). See ValidTime.__retire_cascade__. + __retire_cascade__ = ("allele_links", "projection_groups") __table_args__ = ( ForeignKeyConstraint( diff --git a/src/mavedb/models/mapping_record_allele.py b/src/mavedb/models/mapping_record_allele.py index 79a4f7573..b19823067 100644 --- a/src/mavedb/models/mapping_record_allele.py +++ b/src/mavedb/models/mapping_record_allele.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Optional from sqlalchemy import Boolean, Column, ForeignKey, ForeignKeyConstraint, Index, Integer, text from sqlalchemy.orm import Mapped, relationship @@ -9,6 +9,7 @@ if TYPE_CHECKING: from .allele import Allele from .mapping_record import MappingRecord + from .projection_group import ProjectionGroup class MappingRecordAllele(ValidTime, Base): @@ -27,13 +28,20 @@ class MappingRecordAllele(ValidTime, Base): # alleles. Lives on the link, not Allele, since the same allele can be authoritative for one # mapping record and derived for another. is_authoritative: Mapped[bool] = Column(Boolean, nullable=False, default=False) - # Pairs a link with its coding/genomic counterpart within one mapping record (set by the - # reverse-translation job). NULL for unpaired links. Local to this record, not a cross-record - # identity — see the RT job for group-assignment logic. + # The projection group this allele describes one level of. The authoritative link's group is the record's + # measured group. NULL for an allele outside any group (the protein apex of reverse translation). projection_group = Column(Integer, nullable=True) mapping_record: Mapped["MappingRecord"] = relationship("MappingRecord", back_populates="allele_links") allele: Mapped["Allele"] = relationship("Allele", back_populates="mapping_record_links") + # Read-only: writers set ``projection_group`` directly. A group row is never edited in place (a new + # description is a new row, and its links are re-pointed), so this resolves to the group the link was + # written against at any instant. + group: Mapped[Optional["ProjectionGroup"]] = relationship( + "ProjectionGroup", + primaryjoin="foreign(MappingRecordAllele.projection_group) == ProjectionGroup.id", + viewonly=True, + ) __table_args__ = ( ForeignKeyConstraint( @@ -42,6 +50,13 @@ class MappingRecordAllele(ValidTime, Base): name="fk_mapping_record_alleles_mapping_record_score_set", ondelete="CASCADE", ), + # MATCH SIMPLE: a NULL projection_group is not checked. + ForeignKeyConstraint( + [projection_group, mapping_record_id], + ["projection_groups.id", "projection_groups.mapping_record_id"], + name="fk_mapping_record_alleles_projection_group", + ondelete="CASCADE", + ), Index( "ix_mapping_record_alleles_mapping_record_id", "mapping_record_id", @@ -50,6 +65,7 @@ class MappingRecordAllele(ValidTime, Base): "ix_mapping_record_alleles_allele_id", "allele_id", ), + Index("ix_mapping_record_alleles_projection_group", "projection_group"), # At most one live link per (mapping_record, allele); retired links (valid_to set) are # exempt so history is preserved. Index( diff --git a/src/mavedb/models/projection_group.py b/src/mavedb/models/projection_group.py new file mode 100644 index 000000000..89e2a93cf --- /dev/null +++ b/src/mavedb/models/projection_group.py @@ -0,0 +1,54 @@ +from typing import TYPE_CHECKING + +from sqlalchemy import Column, ForeignKeyConstraint, Index, Integer, String +from sqlalchemy.orm import Mapped, relationship + +from mavedb.db.base import Base +from mavedb.db.mixins import ValidTime + +if TYPE_CHECKING: + from .mapping_record import MappingRecord + + +class ProjectionGroup(ValidTime, Base): + """One variant described at the coding, genomic and protein levels, within one mapping record. + + A record has one *measured* group, the one its authoritative allele link points at, holding the + projections of the variant it measured; and one *candidate* group per pair reverse translation + constructs from the protein consequence. Allele links join a group through + ``MappingRecordAllele.projection_group``. + + The HGVS strings are the group's description, independent of whether each level has an allele: an + intronic or UTR-spanning coding projection has no VRS allele (the transcript has no sequence there) + but is still the variant's coding coordinate. Where a level does have an allele, its HGVS repeats the + string here. + """ + + __tablename__ = "projection_groups" + + id: Mapped[int] = Column(Integer, primary_key=True) + mapping_record_id: Mapped[int] = Column(Integer, nullable=False) + + # Denormalized from the mapping record for the RLS policy, like MappingRecord.score_set_id (#833). + score_set_id: Mapped[int] = Column(Integer, nullable=False) + + hgvs_c = Column(String, nullable=True) + hgvs_g = Column(String, nullable=True) + # The protein consequence, shared by every group of a record. Null where the variant has none (intronic, + # UTR) or it could not be determined. + hgvs_p = Column(String, nullable=True) + + mapping_record: Mapped["MappingRecord"] = relationship("MappingRecord", back_populates="projection_groups") + + __table_args__ = ( + ForeignKeyConstraint( + [mapping_record_id, score_set_id], + ["mapping_records.id", "mapping_records.score_set_id"], + name="fk_projection_groups_mapping_record_score_set", + ondelete="CASCADE", + ), + Index("ix_projection_groups_mapping_record_id", "mapping_record_id"), + # Target of mapping_record_alleles' composite foreign key, which keeps a link inside its own + # record's groups. + Index("uq_projection_groups_id_mapping_record", "id", "mapping_record_id", unique=True), + ) diff --git a/src/mavedb/routers/variants.py b/src/mavedb/routers/variants.py index 890e4ea43..79f4b92be 100644 --- a/src/mavedb/routers/variants.py +++ b/src/mavedb/routers/variants.py @@ -107,7 +107,7 @@ def lookup_variants_by_vrs_identifier( identifier: Annotated[ str, Path( - description="A valid GA4GH digest-based identifier for the mapped allele.", + description="A GA4GH digest-based identifier for a variant's pre-mapped or post-mapped variation.", json_schema_extra={"example": "ga4gh:VA.0123abcd"}, regex=GA4GH_IR_REGEXP, ), @@ -124,10 +124,11 @@ def lookup_variants_by_vrs_identifier( db: Session = Depends(deps.get_db), user_data: Optional[UserData] = Depends(get_current_user), ) -> list[VariantVrsMatch]: - """Resolve a GA4GH VRS identifier to the readable variants whose mapping links that allele. + """Resolve a GA4GH VRS identifier to the readable variants it identifies. - A deduplicated allele may be shared across score sets, so one identifier can resolve to several - variants. This is a lookup returning a collection: results are filtered to the score sets the caller + The identifier may be a reference-level (post-mapped) allele or a variant's pre-mapped variation on + its submitted target sequence. A deduplicated allele may be shared across score sets, so one + identifier can resolve to several variants. This is a lookup returning a collection: results are filtered to the score sets the caller may read, and an empty list is returned when nothing readable matches. An absent identifier and a match visible only in a private score set are deliberately indistinguishable (both yield ``[]``), so the response never reveals a private allele's existence. @@ -145,9 +146,9 @@ def lookup_variants_by_vrs_identifier( return [ VariantVrsMatch( variant_urn=variant.urn or "", - clingen_allele_id=allele.clingen_allele_id, - vrs_id=(allele.post_mapped or {}).get("id"), - level=allele.level, + clingen_allele_id=allele.clingen_allele_id if allele is not None else None, + vrs_id=(allele.post_mapped or {}).get("id") if allele is not None else None, + level=allele.level if allele is not None else None, ) for variant, allele in permitted ] diff --git a/src/mavedb/scripts/audit_reverse_translation.py b/src/mavedb/scripts/audit_reverse_translation.py index 85c6b36fb..b40006273 100644 --- a/src/mavedb/scripts/audit_reverse_translation.py +++ b/src/mavedb/scripts/audit_reverse_translation.py @@ -138,7 +138,11 @@ class SqlCheck: FROM mapping_record_alleles auth JOIN alleles a ON a.id = auth.allele_id {candidate_span} - WHERE auth.is_authoritative AND auth.valid_to IS NULL AND auth.projection_group IS NULL + WHERE auth.is_authoritative AND auth.valid_to IS NULL + AND NOT EXISTS (SELECT 1 FROM mapping_record_alleles pl + JOIN alleles pa ON pa.id = pl.allele_id + WHERE pl.projection_group = auth.projection_group AND pl.allele_id <> auth.allele_id + AND pl.valid_to IS NULL AND pa.level IN ('cdna', 'genomic')) AND a.level IN ('cdna', 'genomic') AND {condition} {auth_filter} @@ -259,9 +263,9 @@ def _measured_outside_group(condition: str) -> str: name="measured_allele_outside_projection_group", severity=ERROR, description=( - "RT attached derived nucleotide alleles to a record, but the measured nucleotide allele is in no " - "projection group: the fold-in missed it, so serving labels every sibling convergent, its own c/g " - "twin included. Only length-preserving changes inside the codon RT targeted, which its candidates " + "RT attached derived nucleotide alleles to a record, but the measured nucleotide allele has no partner " + "allele in its measured projection group: RT matched none of its candidates to it, so serving labels " + "every sibling convergent, its own c/g twin included. Only length-preserving changes inside the codon RT targeted, which its candidates " "always include." ), sql=_measured_outside_group( @@ -272,7 +276,7 @@ def _measured_outside_group(condition: str) -> str: name="measured_allele_beyond_rt_codon", severity=INFO, description=( - "A measured change in no projection group that reaches outside the codon RT targeted, such as a " + "A measured change with no partner allele that reaches outside the codon RT targeted, such as a " "multi-base change straddling two codons. RT lists only single-codon changes, so it cannot fold in. " "Many in one score set can mean its nucleotide variants sit out of frame with its protein variants." ), @@ -282,7 +286,7 @@ def _measured_outside_group(condition: str) -> str: name="measured_reference_allele_outside_projection_group", severity=INFO, description=( - "A measured reference-identical allele (g.=, c.=) in no projection group. Expected: RT lists the " + "A measured reference-identical allele (g.=, c.=) with no partner allele. Expected: RT lists the " "alternative codons for a synonymous change, never the reference codon." ), sql=_measured_outside_group(_REFERENCE_IDENTICAL), @@ -291,7 +295,7 @@ def _measured_outside_group(condition: str) -> str: name="measured_indel_outside_projection_group", severity=INFO, description=( - "A measured indel in no projection group. Expected when the indel is not codon-aligned: it changes the " + "A measured indel with no partner allele. Expected when the indel is not codon-aligned: it changes the " "protein the same way as a codon-aligned candidate but is a different DNA change, so RT does not list it." ), sql=_measured_outside_group(f"{_LENGTH_CHANGING} AND NOT {_REFERENCE_IDENTICAL}"), @@ -300,7 +304,7 @@ def _measured_outside_group(condition: str) -> str: name="malformed_projection_group", severity=WARNING, description=( - "A live projection group is not exactly one cdna allele and one genomic allele. Either the library could " + "A live candidate projection group is not exactly one cdna allele and one genomic allele. Either the library could " "not project the coding candidate to the genome (its entry under candidates in the variant's " "cross_level_translation event has no hgvs_g), or a member failed VRS translation (listed under " "failed_candidates with the error)." @@ -311,6 +315,10 @@ def _measured_outside_group(condition: str) -> str: FROM mapping_record_alleles l JOIN alleles a ON a.id = l.allele_id WHERE l.valid_to IS NULL AND l.projection_group IS NOT NULL {link_filter} + -- Measured groups legitimately lack an allele at a level VRS can't represent (intronic). + AND NOT EXISTS (SELECT 1 FROM mapping_record_alleles auth + WHERE auth.projection_group = l.projection_group + AND auth.is_authoritative AND auth.valid_to IS NULL) GROUP BY 1, 2 HAVING string_agg(a.level, ',' ORDER BY a.level) <> 'cdna,genomic' """, diff --git a/src/mavedb/scripts/project_measured_variants.py b/src/mavedb/scripts/project_measured_variants.py new file mode 100644 index 000000000..5e0b8ae19 --- /dev/null +++ b/src/mavedb/scripts/project_measured_variants.py @@ -0,0 +1,108 @@ +"""Write the measured projection group of every live mapping record, score set by score set. + +New mappings get their groups from the reverse translation job. This covers records that have not been +through it since projection groups existed, notably those the substrate backfill reshaped from legacy +mappings, which carry no groups. Only coordinates are needed (cdot and UTA), not the mapper, so it is fast +enough to run straight after a release. Re-running is safe: unchanged groups are left alone. + +Rewriting a changed group detaches the alleles reverse translation added to it, the measured variant's +other-level partner and its protein consequence, along with their ClinVar, gnomAD and VEP annotations. +Each score set's log line counts them as ``detached_rt_links``; re-run reverse translation on every score +set with a non-zero count. ``--missing-only`` only writes groups that do not exist yet, so it never +detaches anything. + +With ``--commit``, each score set commits as it finishes, so an interrupted run keeps its progress and a run +against a serving database holds row locks on one score set at a time. ``--shard`` splits the score sets +between concurrent processes. + +Usage: + python -m mavedb.scripts.project_measured_variants --dry-run --score-set urn:mavedb:00000663-a-1 + python -m mavedb.scripts.project_measured_variants --commit --shard 0/4 + python -m mavedb.scripts.project_measured_variants --commit --missing-only +""" + +import logging +import time +from collections import Counter +from typing import Optional + +import asyncclick as click +from sqlalchemy import select +from sqlalchemy.orm import Session + +from mavedb.data_providers.services import cdot_rest +from mavedb.models.score_set import ScoreSet +from mavedb.scripts.environment import DatabaseSessionAction, script_environment, with_database_session +from mavedb.worker.jobs.variant_processing.reverse_translation import cdna_transcript_resolver +from mavedb.worker.lib.projection import project_measured_variants +from mavedb.worker.lib.translation_ports import WorkerCoordinateTranslator + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + + +def _parse_shard(shard: Optional[str]) -> Optional[tuple[int, int]]: + if shard is None: + return None + index, _, count = shard.partition("/") + if not (index.isdigit() and count.isdigit()) or not 0 <= int(index) < int(count): + raise click.BadParameter("expected INDEX/COUNT, e.g. 0/4", param_hint="--shard") + return int(index), int(count) + + +@script_environment.command() +@with_database_session(pass_action=True) +@click.option("--score-set", "urns", multiple=True, help="Only this score set (URN). Repeatable.") +@click.option( + "--shard", default=None, help="Process only every COUNT-th score set by size, starting at INDEX, as INDEX/COUNT." +) +@click.option( + "--missing-only", + is_flag=True, + help="Only records with no measured projection group yet. Never detaches reverse translation alleles.", +) +def project_measured_variants_command( + db: Session, action: DatabaseSessionAction, urns: tuple[str, ...], shard: Optional[str], missing_only: bool +) -> None: + """Write measured projection groups for every score set with variants.""" + shard_spec = _parse_shard(shard) + # Largest first, dealt round-robin, so shards finish together rather than one holding the big score sets. + query = ( + select(ScoreSet.id, ScoreSet.urn) + .where(ScoreSet.num_variants > 0) + .order_by(ScoreSet.num_variants.desc(), ScoreSet.id) + ) + if urns: + query = query.where(ScoreSet.urn.in_(urns)) + score_sets = [ + (id_, urn) + for position, (id_, urn) in enumerate(db.execute(query).tuples()) + if shard_spec is None or position % shard_spec[1] == shard_spec[0] + ] + logger.info(f"Projecting measured variants for {len(score_sets)} score sets.") + + coordinates = WorkerCoordinateTranslator(cdot_rest()) + totals: Counter[str] = Counter() + started = time.monotonic() + for index, (score_set_id, urn) in enumerate(score_sets, start=1): + counts = project_measured_variants( + db, + score_set_id, + coordinates=coordinates, + resolve_transcript=cdna_transcript_resolver(db, score_set_id, coordinates), + missing_only=missing_only, + ) + if action is DatabaseSessionAction.COMMIT: + db.commit() + else: + db.flush() + db.expunge_all() + totals.update(counts) + logger.info(f"[{index}/{len(score_sets)}] {urn}: {dict(counts)}") + + elapsed = time.monotonic() - started + logger.info(f"Done in {elapsed:.0f}s: {dict(totals)}") + + +if __name__ == "__main__": + project_measured_variants_command() diff --git a/src/mavedb/view_models/lean_variant.py b/src/mavedb/view_models/lean_variant.py index 474484e40..1d3a2bf1e 100644 --- a/src/mavedb/view_models/lean_variant.py +++ b/src/mavedb/view_models/lean_variant.py @@ -33,9 +33,10 @@ class Config: class MappedTriple(BaseModel): """The mapped (reference-frame) HGVS keyed by level — the canonical projection of the measured change. - One slot per level. A nucleotide assay populates all three (``mapped[assayLevel]`` is the measured - slot; ``cdna`` is the level-invariant search key, present even when ``assayLevel`` is ``genomic``); a - protein assay populates only ``protein`` (the ambiguous c/g fan-out is not fabricated). Null slots are + One slot per level. A nucleotide assay populates the measured slot (``mapped[assayLevel]``) and the + projections it has (``cdna`` is the level-invariant search key, present even when ``assayLevel`` is + ``genomic``; an intronic or UTR change has no ``protein``); a protein assay populates only ``protein`` + (the ambiguous c/g fan-out is not fabricated). Null slots are omitted under ``response_model_exclude_none``. """ diff --git a/src/mavedb/view_models/variant.py b/src/mavedb/view_models/variant.py index 48d4edb85..d9257da37 100644 --- a/src/mavedb/view_models/variant.py +++ b/src/mavedb/view_models/variant.py @@ -10,11 +10,11 @@ class VariantVrsMatch(BaseModel): - """One variant whose mapping links an allele bearing the queried VRS identifier. + """One variant the queried VRS identifier identifies, before or after mapping. - The ``GET /variants/vrs/{identifier}`` lookup result. The ClinGen id + level ride from the matched - (deduplicated) allele; the URN from the variant it is linked to. Replaces the legacy MappedVariant VRS - lookup on the new Allele substrate (#743, item 3.5). + The ``GET /variants/vrs/{identifier}`` lookup result. The ClinGen id, VRS id and level ride from the + matched (deduplicated) allele for a post-mapped match, or from the variant's measured allele for a + pre-mapped match; they are null when that variant did not map. """ variant_urn: str diff --git a/src/mavedb/view_models/variant_detail.py b/src/mavedb/view_models/variant_detail.py index 9603bc7b3..2c9f6004d 100644 --- a/src/mavedb/view_models/variant_detail.py +++ b/src/mavedb/view_models/variant_detail.py @@ -12,6 +12,7 @@ from mavedb.view_models.allele_annotation import AlleleAnnotations from mavedb.view_models.allele_identity import AlleleIdentity from mavedb.view_models.base.base import BaseModel +from mavedb.view_models.lean_variant import MappedTriple from mavedb.view_models.score_calibration import SavedFunctionalClassification @@ -69,6 +70,9 @@ class VariantDetail(BaseModel): molecular_representation: Optional[dict[str, Any]] = None mode: Optional[str] = None + # The measured variant's mapped HGVS at every level it has, including levels with no allele in + # ``alleles`` (an intronic coding projection has no VRS representation). + mapped: Optional[MappedTriple] = None alleles: dict[str, AlleleIdentity] = {} annotations: dict[str, AlleleAnnotations] = {} diff --git a/src/mavedb/worker/jobs/registry.py b/src/mavedb/worker/jobs/registry.py index 8b2b712f8..33878de9e 100644 --- a/src/mavedb/worker/jobs/registry.py +++ b/src/mavedb/worker/jobs/registry.py @@ -69,8 +69,8 @@ cron( cleanup_stalled_jobs, name="cleanup_stalled_jobs_cron", - # Every 10 minutes. Paired with the 20-min progress-stall window this gives an effective - # stall-detection latency of ~20-30 min (vs. up to 50 min on the old 30-min cadence). + # Every 10 minutes. Paired with the 30-min progress-stall window this gives an effective + # stall-detection latency of ~30-40 min. minute={5, 15, 25, 35, 45, 55}, keep_result=timedelta(minutes=9).total_seconds(), ), diff --git a/src/mavedb/worker/jobs/variant_processing/reverse_translation.py b/src/mavedb/worker/jobs/variant_processing/reverse_translation.py index 46d18d28e..91b30b6a2 100644 --- a/src/mavedb/worker/jobs/variant_processing/reverse_translation.py +++ b/src/mavedb/worker/jobs/variant_processing/reverse_translation.py @@ -1,21 +1,15 @@ """Reverse translation worker job — builds cross-level HGVS equivalence classes. -For each mapped variant in a score set, calls construct_equivalent_variants from -the variant-annotation library to produce the coding/genomic HGVS candidates -encoding the same protein consequence, plus that protein consequence itself. Each -candidate is written as a non-authoritative Allele row linked to the existing -MappingRecord via MappingRecordAllele. - -The library returns the equivalence class as a list of ProjectionPair objects, -each one projection pair (a coding candidate and its deterministic genomic projection -— the same change at two levels). The job preserves that pairing by stamping both -links of a projection pair with a shared per-record ``projection_group`` id; the protein -apex is shared across all projection pairs and carries no group. Where a projection pair member -equals the record's authoritative (measured) allele, its group is folded onto the -authoritative link rather than duplicated, so the canonical c/g projection -is resolvable from the authoritative allele's group downstream. An authoritative link -whose group changes is superseded rather than updated, so ``as_of`` reads keep the -group that was live at the time. +First refreshes every record's measured projection group (see :mod:`mavedb.worker.lib.projection`), the +measured variant described at each level. Then, for each mapped variant in the score set, calls +construct_equivalent_variants from the variant-annotation library to produce the coding/genomic HGVS +candidates encoding the same protein consequence, plus that protein consequence itself. Each candidate is +written as a non-authoritative Allele row linked to the existing MappingRecord via MappingRecordAllele. + +The library returns the equivalence class as a list of ProjectionPair objects, each a coding candidate and +its deterministic genomic projection — the same change at two levels. Each pair becomes a candidate +ProjectionGroup row, and both of its links point at it. A pair holding the measured allele is the measured +variant itself, so its other member joins the measured group instead; so does the protein consequence. """ import asyncio @@ -27,8 +21,8 @@ from typing import Any, Callable, NamedTuple, Sequence from ga4gh.vrs.extras.translator import AlleleTranslator -from sqlalchemy import or_, select -from sqlalchemy.orm import Session, joinedload +from sqlalchemy import func, select +from sqlalchemy.orm import Session from variant_annotation import __version__ as variant_annotation_version from variant_annotation.lib.accessions import looks_like_refseq_protein_accession from variant_annotation.lib.translation import construct_equivalent_variants @@ -53,17 +47,22 @@ from mavedb.models.enums.target_category import TargetCategory from mavedb.models.mapping_record import MappingRecord from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.projection_group import ProjectionGroup from mavedb.models.score_set import ScoreSet +from mavedb.models.target_accession import TargetAccession from mavedb.models.target_gene import TargetGene from mavedb.models.target_gene_mapping import TargetGeneMapping from mavedb.models.variant import Variant from mavedb.worker.jobs.utils.setup import validate_job_params from mavedb.worker.lib.decorators.pipeline_management import with_pipeline_management from mavedb.worker.lib.managers.job_manager import JobManager +from mavedb.worker.lib.projection import project_measured_variants from mavedb.worker.lib.translation_ports import WorkerCoordinateTranslator, uta_transcript_source logger = logging.getLogger(__name__) +TranscriptResolver = Callable[[int | None, int, date | None, str | None], str | None] + # Job defaults: full coding equivalence class — every synonymous codon, indels included. # wt_codon_mode "all" requires include_indels=True. _DEFAULT_TRANSLATION_CONFIG: dict[str, Any] = { @@ -75,6 +74,10 @@ "upstream_retry_backoff_seconds": 5.0, } +# A protein-coding target with no MANE Select transcript at its first loci (an intergenic tile, say) is +# given up on after this many lookups rather than queried once per record. +_MANE_ATTEMPTS_PER_TARGET = 5 + class _TranscriptResolution(NamedTuple): """Pairs a mapping record with its pre-UTA transcript info: gene_transcript if the mapper @@ -123,13 +126,29 @@ def _coding_transcripts_for_proteins(protein_accessions: set[str]) -> dict[str, } -def _cdna_transcript_resolver(db: Session, score_set_id: int) -> Callable[[int | None, int, date | None], str | None]: - """Return a resolver from a record's ``(job_run_id, target_gene_id, mapped_date)`` to its cdna transcript. - - A record binds to *its own run's* cdna row via ``job_run_id``. Records with no ``job_run_id`` (pre-column, - or reshaped from legacy) fall back to the day-granular ``(target_gene_id, mapped_date)`` key, which cannot - separate two runs on one calendar day. Within either key, highest ``TargetGeneMapping.id`` wins. The two - lookup tables are built once per score set here and closed over, so per-record resolution is a dict hit. +def cdna_transcript_resolver( + db: Session, score_set_id: int, coordinates: WorkerCoordinateTranslator +) -> TranscriptResolver: + """Return a resolver from a record's ``(job_run_id, target_gene_id, mapped_date, genomic_hgvs)`` to its + cdna transcript. + + In order: + + 1. The run's own cdna mapping row. A record binds to it via ``job_run_id``; records with no + ``job_run_id`` (pre-column, or reshaped from legacy) use the day-granular + ``(target_gene_id, mapped_date)`` key, which cannot separate two runs on one calendar day. Within + either key, highest ``TargetGeneMapping.id`` wins. + 2. For records with no ``job_run_id``, the coding accession the legacy mapper recorded in the target's + ``post_mapped_metadata``: legacy mappings wrote no cdna row, and their genomic HGVS came from that + same run. + 3. For genomic records, the target's own accession when it is a transcript (NM_, ENST). + 4. For genomic records of a protein-coding target, the MANE Select transcript at the record's locus, + found once per target gene. Genomic-only targets (most saturation genome editing) have no other + transcript; this is the one the legacy ClinGen HGVS lookup reported for them. + + Protein records pass no ``genomic_hgvs``, so they stop after step 2 and keep their NP_→NM_ lookup. The + lookup tables are built once per score set here and closed over. A failed MANE Select lookup raises + rather than resolving to no transcript, which would erase a coding projection an earlier run wrote. """ by_job_run: dict[tuple[int, int], str | None] = {} by_run_date: dict[tuple[int, date | None], str | None] = {} @@ -154,10 +173,51 @@ def _cdna_transcript_resolver(db: Session, score_set_id: int) -> Callable[[int | by_job_run[(job_run_id, target_gene_id)] = reference_accession by_run_date[(target_gene_id, mapped_date)] = reference_accession - def resolve(job_run_id: int | None, target_gene_id: int, mapped_date: date | None) -> str | None: + legacy_by_target_gene: dict[int, str] = {} + accession_by_target_gene: dict[int, str] = {} + protein_coding: set[int] = set() + for target_gene_id, metadata, category, target_accession in ( + db.execute( + select(TargetGene.id, TargetGene.post_mapped_metadata, TargetGene.category, TargetAccession.accession) + .outerjoin(TargetAccession, TargetAccession.id == TargetGene.accession_id) + .where(TargetGene.score_set_id == score_set_id) + ) + .tuples() + .all() + ): + cdna = metadata.get("cdna") if isinstance(metadata, dict) else None + accessions = cdna.get("sequence_accessions") if isinstance(cdna, dict) else None + if accessions: + legacy_by_target_gene[target_gene_id] = accessions[0] + if target_accession and target_accession.startswith(("NM_", "ENST")): + accession_by_target_gene[target_gene_id] = target_accession + if category == TargetCategory.protein_coding: + protein_coding.add(target_gene_id) + + mane_by_target_gene: dict[int, str] = {} + mane_attempts: Counter[int] = Counter() + + def mane_select(target_gene_id: int, genomic_hgvs: str) -> str | None: + if target_gene_id in mane_by_target_gene: + return mane_by_target_gene[target_gene_id] + if target_gene_id not in protein_coding or mane_attempts[target_gene_id] >= _MANE_ATTEMPTS_PER_TARGET: + return None + transcript = coordinates.mane_select_transcript(genomic_hgvs) + mane_attempts[target_gene_id] += 1 + if transcript is not None: + mane_by_target_gene[target_gene_id] = transcript + return transcript + + def resolve( + job_run_id: int | None, target_gene_id: int, mapped_date: date | None, genomic_hgvs: str | None + ) -> str | None: if job_run_id is not None: - return by_job_run.get((job_run_id, target_gene_id)) - return by_run_date.get((target_gene_id, mapped_date)) + transcript = by_job_run.get((job_run_id, target_gene_id)) + else: + transcript = by_run_date.get((target_gene_id, mapped_date)) or legacy_by_target_gene.get(target_gene_id) + if transcript is not None or genomic_hgvs is None: + return transcript + return accession_by_target_gene.get(target_gene_id) or mane_select(target_gene_id, genomic_hgvs) return resolve @@ -254,7 +314,17 @@ async def reverse_translate_variants_for_score_set( job_manager.update_progress(0, 100, "Starting reverse translation job.") logger.info(msg="Started reverse translation job.", extra=job_manager.logging_context()) - resolve_cdna_transcript = _cdna_transcript_resolver(job_manager.db, score_set_id) + coordinates = WorkerCoordinateTranslator(ctx["hdp"]) + resolve_cdna_transcript = cdna_transcript_resolver(job_manager.db, score_set_id, coordinates) + + # The measured groups come first: candidates that match the measured variant join its group below. + projection_counts = project_measured_variants( + job_manager.db, score_set_id, coordinates=coordinates, resolve_transcript=resolve_cdna_transcript + ) + logger.info( + msg=f"Measured projection groups: {dict(projection_counts)}.", + extra=job_manager.logging_context(), + ) # Load current authoritative MappingRecords with their Variant and TargetGeneMapping. The record's own # TargetGeneMapping (its measured alignment) carries the run's job_run_id and mapped_date, which anchor @@ -303,7 +373,8 @@ async def reverse_translate_variants_for_score_set( transcript_resolutions: list[_TranscriptResolution] = [] protein_accessions: set[str] = set() for rec, variant, target_gene_id, mapped_date, record_job_run_id in rows: - coding_accession = resolve_cdna_transcript(record_job_run_id, target_gene_id, mapped_date) + genomic_hgvs = rec.hgvs_assay_level if rec.assay_level == SequenceLevel.genomic else None + coding_accession = resolve_cdna_transcript(record_job_run_id, target_gene_id, mapped_date, genomic_hgvs) protein_accession = None if not coding_accession and rec.hgvs_assay_level is not None: raw_accession = extract_accession(rec.hgvs_assay_level) @@ -378,7 +449,6 @@ async def reverse_translate_variants_for_score_set( # construct_equivalent_variants is I/O-bound (blocks on a subprocess); run in the # default thread pool rather than the process pool to avoid pickling the port objects. - coordinates = WorkerCoordinateTranslator(ctx["hdp"]) loop = asyncio.get_running_loop() job_manager.update_progress(20, 100, "Running reverse translation subprocess.") # WtCodonMode.ALL reads the reference codon via TranscriptSource.codon_at, so pass a @@ -416,65 +486,43 @@ async def reverse_translate_variants_for_score_set( .where(MappingRecord.current.is_(True)) ) - # Defer linkage until all candidates are processed so the prior derived links can be - # superseded atomically — retire and insert share a timestamp with no gap. + # Derived links and candidate groups are deferred until every record is processed, so the prior derived + # set is superseded atomically: retire and insert share one timestamp. new_links: list[MappingRecordAllele] = [] + new_groups: list[tuple[ProjectionGroup, list[MappingRecordAllele]]] = [] - # The live authoritative links for the current records, keyed by (record_id, allele_id). When an - # RT candidate's allele equals a record's authoritative (measured) allele, that link already exists - # and uq_mapping_record_alleles_live forbids a derived duplicate, so the candidate's group is - # recorded in folded_groups instead. After the loop, each link whose group changed is superseded by - # a copy carrying the new one. - authoritative_links: dict[tuple[int, int], MappingRecordAllele] = { - (link.mapping_record_id, link.allele_id): link + # The live authoritative link of each current record. It points at the record's measured projection + # group (written by project_measured_variants above), which this job adds alleles to but never rewrites. + authoritative_by_record: dict[int, MappingRecordAllele] = { + link.mapping_record_id: link for link in job_manager.db.scalars( select(MappingRecordAllele) .where(MappingRecordAllele.is_authoritative.is_(True)) .where(MappingRecordAllele.current) .where(MappingRecordAllele.mapping_record_id.in_(current_record_ids)) - .options(joinedload(MappingRecordAllele.allele)) ).all() } - # record_id -> the projection group this run folded onto its authoritative allele, and the group its - # authoritative link already holds. - folded_groups: dict[int, int | None] = {} - prior_groups = {record_id: link.projection_group for (record_id, _), link in authoritative_links.items()} - measured_alleles = {record_id: link.allele for (record_id, _), link in authoritative_links.items()} + seqrepo = getattr(ctx["seqrepo"], "sr", None) for result in results: rec, variant = variant_input_map[id(result.input)] + authoritative = authoritative_by_record.get(rec.id) + measured_allele_id = authoritative.allele_id if authoritative is not None else None + measured_group = authoritative.projection_group if authoritative is not None else None + # Prediction parens (p.(Ala222Val)) are stripped before translation and storage. None for protein-assay + # inputs, where the protein is already the authoritative allele. + hgvs_p = strip_protein_prediction_parens(result.hgvs_p) if result.hgvs_p else None + candidate_count = 0 - record_links_start = len(new_links) linked_nucleotide = False - - # Equivalence generation may surface the same VRS object more than once. - # Dedup by vrs_digest per mapping record to avoid duplicate links. + folded_in = False + # Equivalence generation may surface the same VRS object more than once; link each allele once. seen_digests: set[str] = set() failed_candidates: list[dict[str, str]] = [] + candidates_metadata: list[dict[str, Any]] = [] - # Flatten the projection pairs into a work list of (hgvs, level, field, projection_group) members, - # preserving the coding↔genomic pairing the library emits. Each ProjectionPair's coding and - # genomic members share the pair's per-record group id — its index in the equivalence class - # (0..N-1), until the fold-in renumbering below. hgvs_c is always present (the pair key); hgvs_g - # is None when the c→g projection failed, yielding a well-formed one-member (coding only) group - # rather than a desync. The group id is assigned once per pair here, so the two members are - # guaranteed to carry the same id even though they translate and link independently below. - members: list[tuple[str, SequenceLevel, str, int | None]] = [] - for group_id, pair in enumerate(result.projection_pairs): - members.append((pair.hgvs_c, SequenceLevel.cdna, "hgvs_c", group_id)) - if pair.hgvs_g is not None: - members.append((pair.hgvs_g, SequenceLevel.genomic, "hgvs_g", group_id)) - - # The protein consequence is the apex of the equivalence set — shared across every - # pair, a member of none — so it carries no group (None). Prediction parens - # (p.(Ala222Val)) are stripped before translation and storage. None for protein-assay - # inputs, where the protein is already the authoritative allele. - if result.hgvs_p: - members.append((strip_protein_prediction_parens(result.hgvs_p), SequenceLevel.protein, "hgvs_p", None)) - - for hgvs, level, hgvs_field, projection_group in members: - # A candidate may not be translatable (intronic projection, malformed expression). - # Track per-candidate failures; a variant fails only if *all* candidates fail. + def translated(hgvs: str, level: SequenceLevel, hgvs_field: str) -> AlleleDbModel | None: + """The allele for one candidate HGVS, or None if it cannot be translated (intronic, malformed).""" try: variation = translate_hgvs_to_variation(hgvs, allele_translator) except Exception as e: @@ -483,84 +531,101 @@ async def reverse_translate_variants_for_score_set( extra=job_manager.logging_context(), ) failed_candidates.append({"hgvs": hgvs, "level": level.value, "error": str(e)}) - continue + return None - # Verify that the post-mapped allele's sequence matches what SeqRepo holds for its accession. - if (seqrepo := getattr(ctx["seqrepo"], "sr", None)) is not None: + # Verify that the allele's sequence matches what SeqRepo holds for its accession. + if seqrepo is not None: verify_allele_refget( variation.model_dump(exclude_none=True), hgvs, seqrepo, subject=f"variant record {rec.id}" ) # translate_hgvs_to_variation identifies every variation it returns. assert variation.id is not None - if variation.id in seen_digests: - continue - - seen_digests.add(variation.id) - draft_allele = AlleleDbModel( - vrs_digest=variation.id, - # exclude_none mirrors the mapper's serialization. - post_mapped=variation.model_dump(exclude_none=True), - level=level, - **{hgvs_field: hgvs}, # type: ignore[arg-type] + allele = get_or_create_allele( + job_manager.db, + AlleleDbModel( + vrs_digest=variation.id, + # exclude_none mirrors the mapper's serialization. + post_mapped=variation.model_dump(exclude_none=True), + level=level, + **{hgvs_field: hgvs}, # type: ignore[arg-type] + ), ) - allele = get_or_create_allele(job_manager.db, draft_allele) job_manager.db.flush() + return allele - # Authoritative fold-in: this member is the record's measured allele, already linked - # authoritatively. Keep its group to preserve the c↔g pairing; no derived link. - if (rec.id, allele.id) in authoritative_links: - folded_groups[rec.id] = projection_group + def link(allele: AlleleDbModel) -> MappingRecordAllele: + seen_digests.add(allele.vrs_digest) + return MappingRecordAllele( + mapping_record_id=rec.id, score_set_id=rec.score_set_id, allele_id=allele.id, is_authoritative=False + ) + + # Each ProjectionPair is a coding candidate and its genomic projection: the same change at two levels. + # hgvs_g is None when the c→g projection failed, leaving a one-member group. + for pair in result.projection_pairs: + members = [(pair.hgvs_c, SequenceLevel.cdna, "hgvs_c")] + if pair.hgvs_g is not None: + members.append((pair.hgvs_g, SequenceLevel.genomic, "hgvs_g")) + alleles = [ + allele for hgvs, level, field in members if (allele := translated(hgvs, level, field)) is not None + ] + # Unseen alleles, once each: a pair's two members can identify to one VRS object. + fresh = list({a.vrs_digest: a for a in alleles if a.vrs_digest not in seen_digests}.values()) + + # A pair holding the measured allele is the measured variant itself: its other member joins the + # measured group rather than forming a candidate group. + is_measured = measured_allele_id is not None and any(a.id == measured_allele_id for a in alleles) + candidates_metadata.append( + { + "hgvs_c": pair.hgvs_c, + "hgvs_g": pair.hgvs_g, + "variant_type": pair.variant_type, + "measured": is_measured, + } + ) + if is_measured: + folded_in = True + for allele in fresh: + if allele.id != measured_allele_id: + partner = link(allele) + partner.projection_group = measured_group + new_links.append(partner) + candidate_count += 1 continue - new_links.append( - MappingRecordAllele( + if fresh: + group = ProjectionGroup( mapping_record_id=rec.id, score_set_id=rec.score_set_id, - allele_id=allele.id, - is_authoritative=False, - projection_group=projection_group, + hgvs_c=pair.hgvs_c, + hgvs_g=pair.hgvs_g, + hgvs_p=hgvs_p, ) - ) - candidate_count += 1 - linked_nucleotide |= level in NUCLEOTIDE_LEVELS - - # Group ids only pair links within this record, so when the measured allele folds into a different - # position than the group its link already holds, swap the two ids rather than supersede the link. - folded, prior = folded_groups.get(rec.id), prior_groups.get(rec.id) - renumber: dict[int | None, int | None] = ( - {folded: prior, prior: folded} if folded is not None and prior is not None and folded != prior else {} - ) - for link in new_links[record_links_start:]: - link.projection_group = renumber.get(link.projection_group, link.projection_group) - if renumber: - folded_groups[rec.id] = prior - - # A nucleotide measured allele that RT linked nucleotide siblings to but never folded in has no projection - # partner, so serving labels every sibling convergent. Expected for an indel RT's codon-level candidates - # can't express; otherwise the mapper and RT produced different VRS objects for one change. - measured_allele = measured_alleles.get(rec.id) + new_groups.append((group, [link(allele) for allele in fresh])) + candidate_count += len(fresh) + linked_nucleotide = True + + # The protein consequence is the measured variant's own, so it joins the measured group. + if hgvs_p is not None: + protein = translated(hgvs_p, SequenceLevel.protein, "hgvs_p") + if protein is not None and protein.id != measured_allele_id and protein.vrs_digest not in seen_digests: + apex = link(protein) + apex.projection_group = measured_group + new_links.append(apex) + candidate_count += 1 + + # A nucleotide measured allele that RT linked candidates to but never matched: its candidates are distinct + # encodings of the consequence. Expected for changes RT's codon-level candidates can't express; otherwise + # the mapper and RT produced different VRS objects for one change. fold_in_missed = ( - linked_nucleotide - and rec.id not in folded_groups - and measured_allele is not None - and measured_allele.level in NUCLEOTIDE_LEVELS + linked_nucleotide and not folded_in and authoritative is not None and rec.assay_level in NUCLEOTIDE_LEVELS ) annotation_counts["fold_in_missed"] += fold_in_missed annotation_counts["alleles_created"] += candidate_count annotation_metadata = { "hgvs_input": result.input.hgvs, - # Serializable projection of the projection pairs and the group each one's links carry. - "candidates": [ - { - "hgvs_c": pair.hgvs_c, - "hgvs_g": pair.hgvs_g, - "variant_type": pair.variant_type, - "projection_group": renumber.get(position, position), - } - for position, pair in enumerate(result.projection_pairs) - ], + "candidates": candidates_metadata, "hgvs_p": result.hgvs_p, "alleles_created": candidate_count, "failed_candidates": failed_candidates, @@ -589,48 +654,49 @@ async def reverse_translate_variants_for_score_set( metadata=annotation_metadata, ) - # A record the library couldn't ask about because UTA was unavailable keeps its derived links: - # its inputs are unchanged since the run that produced them, so they are still that run's answer. - # Every other outcome, including a skip or a failed translation, is this run's answer and replaces - # the prior set. + # A record the library couldn't ask about because UTA was unavailable keeps its derived links and + # candidate groups: its inputs are unchanged since the run that produced them, so they are still that + # run's answer. Every other outcome, including a skip or a failed translation, is this run's answer and + # replaces the prior set. unavailable_record_ids = { variant_input_map[id(error.input)][0].id for error in errors if error.reason is TranslationErrorReason.UPSTREAM_UNAVAILABLE } + measured_group_ids = ( + select(MappingRecordAllele.projection_group) + .where(MappingRecordAllele.is_authoritative.is_(True)) + .where(MappingRecordAllele.current) + .where(MappingRecordAllele.projection_group.isnot(None)) + .where(MappingRecordAllele.mapping_record_id.in_(current_record_ids)) + ) - # An authoritative link takes this run's fold-in group, or NULL when nothing folded in. Group ids - # are positions in this run's pair list, so a group left from a prior run would name an unrelated - # pair. Records whose links are kept (unavailable) keep their group too. - regrouped_link_ids: list[int] = [] - for (record_id, _), link in authoritative_links.items(): - group = folded_groups.get(record_id) - if record_id in unavailable_record_ids or link.projection_group == group: - continue - - regrouped_link_ids.append(link.id) - new_links.append( - MappingRecordAllele( - mapping_record_id=link.mapping_record_id, - score_set_id=link.score_set_id, - allele_id=link.allele_id, - is_authoritative=True, - projection_group=group, - ) - ) - - # Supersede prior live derived links, and the regrouped authoritative links, under one timestamp. + # Supersede the prior candidate groups and derived links under one timestamp. Measured groups belong to + # project_measured_variants and are never retired here; their derived links are, and are re-added above. # TODO#765: re-runs retire and recreate the whole derived set because re-mapping re-mints # records; idempotent records would allow unchanged links to stay live. + superseded_at = job_manager.db.scalar(select(func.now())) + ProjectionGroup.supersede_live_where( + job_manager.db, + [group for group, _ in new_groups], + ProjectionGroup.id.not_in(measured_group_ids), + ProjectionGroup.mapping_record_id.in_(current_record_ids), + ProjectionGroup.mapping_record_id.not_in(unavailable_record_ids), + at=superseded_at, + ) + job_manager.db.flush() + for group, links in new_groups: + for group_link in links: + group_link.projection_group = group.id + new_links.extend(links) + MappingRecordAllele.supersede_live_where( job_manager.db, new_links, - or_( - MappingRecordAllele.is_authoritative.is_(False), - MappingRecordAllele.id.in_(regrouped_link_ids), - ), + MappingRecordAllele.is_authoritative.is_(False), MappingRecordAllele.mapping_record_id.in_(current_record_ids), MappingRecordAllele.mapping_record_id.not_in(unavailable_record_ids), + at=superseded_at, ) # The library types each error's reason: NOT_TRANSLATABLE is a benign structural gap (the diff --git a/src/mavedb/worker/lib/projection.py b/src/mavedb/worker/lib/projection.py new file mode 100644 index 000000000..38bc92b02 --- /dev/null +++ b/src/mavedb/worker/lib/projection.py @@ -0,0 +1,282 @@ +"""Measured projection groups: each mapping record's measured variant described at every level it has. + +A record's measured group holds its measured variant's HGVS at the measured level and its projections to +the other levels: the coding HGVS of a genomic measurement (``g_to_c`` on the record's transcript) or the +genomic HGVS of a coding one (``c_to_g``), plus the protein consequence (``c_to_p``). They are computed from +the mapping alone, so every nucleotide record gets them, including variants reverse translation cannot +reach: intronic and UTR changes, multi-codon indels, start and stop changes. + +The group carries strings only. Alleles join it through their links: the authoritative link here, the rest +from reverse translation when it constructs the same change (see the reverse translation job). +""" + +import logging +import re +from collections import Counter +from dataclasses import dataclass +from datetime import date +from typing import Callable, Optional, cast + +from sqlalchemy import CursorResult, select, true +from sqlalchemy.orm import Session + +from mavedb.lib.hgvs import extract_accession, is_cis_phased_hgvs, strip_protein_prediction_parens +from mavedb.models.enums.sequence_level import SequenceLevel +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.projection_group import ProjectionGroup +from mavedb.models.target_gene_mapping import TargetGeneMapping +from mavedb.models.variant import Variant +from mavedb.worker.lib.translation_ports import WorkerCoordinateTranslator + +logger = logging.getLogger(__name__) + +TranscriptResolver = Callable[[Optional[int], int, Optional[date], Optional[str]], Optional[str]] + +_SYNONYMOUS_STOP = re.compile(r"p\.Ter(\d+)Ter$") +_SPANLESS_IDENTITY = re.compile(r"^(?P[^:]+):g\.(?P\d+)(?P[ACGTN]{2,})=$") + + +@dataclass(frozen=True) +class MeasuredHgvs: + """A measured variant's HGVS at each level; ``None`` where a level has none or could not be projected.""" + + hgvs_c: Optional[str] = None + hgvs_g: Optional[str] = None + hgvs_p: Optional[str] = None + + +def _expand_reference_identity(hgvs: str) -> str: + """Give a multi-base reference identity the span its reference implies. + + Legacy mappings wrote a codon-level identity as ``g.100CCT=``: one position with a three-base + reference, which hgvs rejects. ``g.100_102=`` is the same variant in valid HGVS. + """ + match = _SPANLESS_IDENTITY.match(hgvs) + if match is None: + return hgvs + start = int(match["start"]) + return f"{match['accession']}:g.{start}_{start + len(match['ref']) - 1}=" + + +def _is_inversion(hgvs: str) -> bool: + """Whether hgvs wrote a variant as ``inv``, which VRS cannot represent; projections keep the delins.""" + return hgvs.endswith("inv") + + +def _protein_consequence(coordinates: WorkerCoordinateTranslator, hgvs_c: str) -> Optional[str]: + """The protein consequence of a coding variant, or ``None`` when it is unknown (``p.?``: intronic, UTR).""" + hgvs_p = strip_protein_prediction_parens(coordinates.c_to_p(hgvs_c)) + if hgvs_p.endswith(":p.?"): + return None + + # hgvs writes a synonymous stop as ``p.Ter778Ter``; HGVS writes every synonymous change with ``=``. + return _SYNONYMOUS_STOP.sub(r"p.Ter\1=", hgvs_p) + + +def project_measured_hgvs( + assay_level: SequenceLevel, + hgvs: str, + *, + transcript: Optional[str], + coordinates: WorkerCoordinateTranslator, +) -> tuple[MeasuredHgvs, list[str]]: + """Describe a measured variant at every level, returning the description and any projection errors. + + Each projection is attempted independently, so a failed protein consequence still keeps the coding + projection. A genomic measurement is normalized before it is projected, and the group carries the + normalized form; one with no coding transcript has only its genomic HGVS. + """ + errors: list[str] = [] + if assay_level == SequenceLevel.protein: + return MeasuredHgvs(hgvs_p=hgvs), errors + + hgvs_c: Optional[str] = None + hgvs_g: Optional[str] = None + if assay_level == SequenceLevel.genomic: + hgvs_g = _expand_reference_identity(hgvs) + # A reference identity has nothing to trim, and hgvs would collapse its span to one position. + if not hgvs_g.endswith("="): + try: + hgvs_g = coordinates.normalize(hgvs) + except Exception as e: + # A variant hgvs cannot normalize (a reference mismatch, say) still projects as written. + logger.debug("Could not normalize %s: %s", hgvs, e) + + if transcript is not None: + try: + hgvs_c = coordinates.g_to_c(hgvs_g, transcript) + if _is_inversion(hgvs_c): + hgvs_c = coordinates.g_to_c_literal(hgvs_g, transcript) + except Exception as e: + errors.append(f"g_to_c: {e}") + + else: + hgvs_c = hgvs + try: + hgvs_g = coordinates.c_to_g(hgvs) + if _is_inversion(hgvs_g): + hgvs_g = coordinates.c_to_g_literal(hgvs) + except Exception as e: + errors.append(f"c_to_g: {e}") + + hgvs_p: Optional[str] = None + if hgvs_c is not None: + try: + hgvs_p = _protein_consequence(coordinates, hgvs_c) + except Exception as e: + errors.append(f"c_to_p: {e}") + + return MeasuredHgvs(hgvs_c=hgvs_c, hgvs_g=hgvs_g, hgvs_p=hgvs_p), errors + + +def project_measured_variants( + db: Session, + score_set_id: int, + *, + coordinates: WorkerCoordinateTranslator, + resolve_transcript: TranscriptResolver, + missing_only: bool = False, +) -> Counter[str]: + """Write or refresh the measured projection group of every current mapping record in a score set. + + ``missing_only`` restricts the run to records with no measured group yet, skipping the projection of + records that have one. + + A record whose live measured group already holds the same description is left alone, so a re-run + writes nothing. Otherwise its group is superseded and its authoritative link is superseded by a copy + pointing at the new group. A record whose projection raises keeps the group it has, so an unreachable + data provider cannot erase earlier projections. + + Superseding a group retires the other links in it, because they described the old group: the measured + variant's partner at the other nucleotide level and its protein consequence, both added by reverse + translation. Until reverse translation runs again on the score set, the record serves neither allele + nor the ClinVar, gnomAD and VEP annotations reached through them. Candidate groups are not touched. + The reverse translation job runs this first, so it re-attaches them in the same run; a standalone run + reports the retired links as ``detached_rt_links`` and must be followed by reverse translation for + each score set with a non-zero count. ``missing_only`` runs never supersede a group. + + Does not commit. + """ + counts: Counter[str] = Counter() + # Scoped through variants: mapping_records has no score-set index, so filtering on its own score_set_id + # would scan the table once per score set. + rows = ( + db.execute( + select( + MappingRecord, + MappingRecordAllele, + TargetGeneMapping.target_gene_id, + TargetGeneMapping.mapped_date, + TargetGeneMapping.job_run_id, + ) + .join(Variant, Variant.id == MappingRecord.variant_id) + .join(MappingRecordAllele, MappingRecordAllele.mapping_record_id == MappingRecord.id) + .outerjoin(TargetGeneMapping, TargetGeneMapping.id == MappingRecord.target_gene_mapping_id) + .where(Variant.score_set_id == score_set_id) + .where(MappingRecord.current) + .where(MappingRecordAllele.is_authoritative.is_(True)) + .where(MappingRecordAllele.current) + .where(MappingRecordAllele.projection_group.is_(None) if missing_only else true()) + ) + .tuples() + .all() + ) + + group_ids = [link.projection_group for _, link, *_ in rows if link.projection_group is not None] + live_groups = { + group.id: group + for group in db.scalars( + select(ProjectionGroup).where(ProjectionGroup.id.in_(group_ids)).where(ProjectionGroup.current) + ) + } + + pending: list[tuple[MappingRecord, MappingRecordAllele, MeasuredHgvs]] = [] + for record, link, target_gene_id, mapped_date, job_run_id in rows: + hgvs = record.hgvs_assay_level + if not hgvs or is_cis_phased_hgvs(hgvs): + counts["skipped_no_hgvs" if not hgvs else "skipped_cis_phased"] += 1 + continue + + level = SequenceLevel(record.assay_level) + transcript = None + resolve_errors: list[str] = [] + if level == SequenceLevel.genomic and target_gene_id is not None: + try: + transcript = resolve_transcript(job_run_id, target_gene_id, mapped_date, hgvs) + except Exception as e: + resolve_errors.append(f"transcript: {e}") + elif level == SequenceLevel.cdna: + transcript = extract_accession(hgvs) + + measured, errors = project_measured_hgvs(level, hgvs, transcript=transcript, coordinates=coordinates) + errors = resolve_errors + errors + current = live_groups.get(link.projection_group) if link.projection_group is not None else None + if errors: + counts["projection_errors"] += 1 + logger.debug("Projection errors for mapping record %s (%s): %s", record.id, hgvs, errors) + # An error may be transient (the data provider unreachable), so it never replaces a group a + # previous run wrote; a record with no group yet still gets the levels that did project. + if current is not None: + counts["kept_on_error"] += 1 + continue + + if current is not None and (current.hgvs_c, current.hgvs_g, current.hgvs_p) == ( + measured.hgvs_c, + measured.hgvs_g, + measured.hgvs_p, + ): + counts["unchanged"] += 1 + continue + + pending.append((record, link, measured)) + + if not pending: + return counts + + # Retire before inserting: the live-unique indexes on links (one per record and allele, one authoritative + # per record) must see the old rows closed. Both statements and the inserts below take the transaction's + # now(), so the handoff is gap-free. + prior_group_ids = [link.projection_group for _, link, _ in pending if link.projection_group is not None] + if prior_group_ids: + detached = cast( + CursorResult, + db.execute( + MappingRecordAllele.retire_live_where( + MappingRecordAllele.projection_group.in_(prior_group_ids), + MappingRecordAllele.is_authoritative.is_(False), + ) + ), + ) + if detached.rowcount: + counts["detached_rt_links"] += detached.rowcount + db.execute(ProjectionGroup.retire_live_where(ProjectionGroup.id.in_(prior_group_ids))) + db.execute(MappingRecordAllele.retire_live_where(MappingRecordAllele.id.in_([link.id for _, link, _ in pending]))) + + groups = [ + ProjectionGroup( + mapping_record_id=record.id, + score_set_id=record.score_set_id, + hgvs_c=measured.hgvs_c, + hgvs_g=measured.hgvs_g, + hgvs_p=measured.hgvs_p, + ) + for record, _, measured in pending + ] + db.add_all(groups) + db.flush() + + db.add_all( + MappingRecordAllele( + mapping_record_id=link.mapping_record_id, + score_set_id=link.score_set_id, + allele_id=link.allele_id, + is_authoritative=True, + projection_group=group.id, + ) + for (_, link, _), group in zip(pending, groups) + ) + db.flush() + + counts["projected"] += len(pending) + return counts diff --git a/src/mavedb/worker/lib/translation_ports.py b/src/mavedb/worker/lib/translation_ports.py index b3a2ea210..2ea801f03 100644 --- a/src/mavedb/worker/lib/translation_ports.py +++ b/src/mavedb/worker/lib/translation_ports.py @@ -24,6 +24,8 @@ UTA_MAX_ATTEMPTS = 6 UTA_BACKOFF_SECONDS = 2.0 +_COMPLEMENT = str.maketrans("ACGTN", "TGCAN") + @contextlib.contextmanager def uta_transcript_source() -> Generator[UtaClient]: @@ -41,6 +43,10 @@ def uta_transcript_source() -> Generator[UtaClient]: yield client +def _reverse_complement(sequence: str) -> str: + return sequence.translate(_COMPLEMENT)[::-1] + + class WorkerCoordinateTranslator: """CoordinateTranslator backed by the worker's HGVS data provider.""" @@ -49,13 +55,16 @@ def __init__(self, hdp: Any) -> None: self._parser: Any = None self._mapper: Any = None self._literal_mapper: Any = None + self._normalizer: Any = None def _ensure_initialized(self) -> None: if self._mapper is None: import hgvs.assemblymapper + import hgvs.normalizer # type: ignore[import-untyped] import hgvs.parser self._parser = hgvs.parser.Parser() + self._normalizer = hgvs.normalizer.Normalizer(self._hdp) self._mapper = hgvs.assemblymapper.AssemblyMapper( self._hdp, assembly_name="GRCh38", @@ -69,6 +78,25 @@ def _ensure_initialized(self) -> None: normalize=False, ) + def normalize(self, g_hgvs: str) -> str: + """Trim and 3'-shift a genomic variant on its reference sequence, keeping it representable in VRS. + + A genomic delins that shares a base with its insertion, at an exon boundary, otherwise projects to + an intron-spanning coding delins with no protein consequence instead of the coding deletion it is. + hgvs rewrites a reverse-complement delins as ``inv``, which VRS cannot represent, so that is + written back as the equivalent delins. + """ + self._ensure_initialized() + variant = self._normalizer.normalize(self._parser.parse(g_hgvs)) + if variant.posedit.edit.type == "inv": + import hgvs.edit # type: ignore[import-untyped] + + start, end = variant.posedit.pos.start.base, variant.posedit.pos.end.base + ref = self._hdp.get_seq(variant.ac, start - 1, end).upper() + variant.posedit.edit = hgvs.edit.NARefAlt(ref=ref, alt=_reverse_complement(ref)) + + return str(variant) + def c_to_p(self, c_hgvs: str) -> str: self._ensure_initialized() return str(self._mapper.c_to_p(self._parser.parse(c_hgvs))) @@ -77,10 +105,43 @@ def g_to_c(self, g_hgvs: str, transcript: str) -> str: self._ensure_initialized() return str(self._mapper.g_to_t(self._parser.parse(g_hgvs), transcript)) + def g_to_c_literal(self, g_hgvs: str, transcript: str) -> str: + """Project to coding without normalization, so a delins stays a delins rather than becoming ``inv``.""" + self._ensure_initialized() + return str(self._literal_mapper.g_to_t(self._parser.parse(g_hgvs), transcript)) + def c_to_g(self, c_hgvs: str) -> str: self._ensure_initialized() return str(self._mapper.c_to_g(self._parser.parse(c_hgvs))) + def mane_select_transcript(self, g_hgvs: str) -> str | None: + """The MANE Select RefSeq transcript overlapping a genomic variant, or ``None`` if there is none. + + Reads cdot's per-build ``tag``, so it needs a cdot data provider. Where MANE Select transcripts of two + genes overlap the variant, the lowest accession wins, so the choice is stable across runs. HGVS that + does not parse has no locus and gives ``None``; data provider failures raise, so callers can tell an + outage from a locus with no MANE transcript. + """ + self._ensure_initialized() + try: + variant = self._parser.parse(g_hgvs) + except Exception: + return None + + candidates = sorted( + tx["tx_ac"] + for tx in self._hdp.get_tx_for_region( + variant.ac, "splign", variant.posedit.pos.start.base - 1, variant.posedit.pos.end.base + ) + if tx["tx_ac"].startswith("NM_") + ) + for accession in candidates: + build = self._hdp._get_transcript(accession).get("genome_builds", {}).get("GRCh38", {}) + if "MANE Select" in (build.get("tag") or ""): + return accession + + return None + def c_to_g_literal(self, c_hgvs: str) -> str: """Project to genomic without normalization; variant-annotation uses it to rewrite an ``inv`` as delins.""" self._ensure_initialized() diff --git a/tests/helpers/util/annotation.py b/tests/helpers/util/annotation.py index d8ee5ca31..6edf490de 100644 --- a/tests/helpers/util/annotation.py +++ b/tests/helpers/util/annotation.py @@ -25,16 +25,30 @@ from mavedb.models.gnomad_allele_link import GnomadAlleleLink from mavedb.models.mapping_record import MappingRecord from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.projection_group import ProjectionGroup from mavedb.models.variant import Variant from mavedb.models.vep_allele_consequence import VepAlleleConsequence +@dataclass(eq=False) +class GroupSpec: + """A ProjectionGroup row on a mapping record: one variant described at the c./g./p. levels. + + Specs that share a GroupSpec instance join that one group; its HGVS strings are written as given. + """ + + hgvs_c: Optional[str] = None + hgvs_g: Optional[str] = None + hgvs_p: Optional[str] = None + + @dataclass class AlleleSpec: """One allele on a mapping record, plus the annotations that hang off it. Digests are the allele's identity in the serving layer: a digest already seeded reuses that allele, so give a shared allele's annotations on one spec only. + ``group`` places the allele in a ProjectionGroup (see ``GroupSpec``). ``clinvar_control_ids`` / ``gnomad_variant_ids`` reference rows seeded elsewhere (the router fixtures seed ClinVar controls with ids 1 and 2); each id becomes a live link on this allele. """ @@ -48,7 +62,7 @@ class AlleleSpec: hgvs_g: Optional[str] = None post_mapped: Optional[dict] = None vep_consequence: Optional[str] = None - projection_group: Optional[int] = None + group: Optional[GroupSpec] = None clinvar_control_ids: Sequence[int] = field(default_factory=tuple) gnomad_variant_ids: Sequence[int] = field(default_factory=tuple) @@ -70,7 +84,8 @@ def seed_mapping_record( VRS (the flat ``preMapped`` field on the variant detail). When ``valid_from`` is set it stamps the record, its allele links, and every annotation link, so ``as_of`` reconstruction tests can place the whole graph at a chosen instant (absent it, ``valid_from`` defaults to ``now()`` and the row - is live). Returns the created ``MappingRecord``. + is live). Each distinct ``GroupSpec`` instance among the specs becomes one ProjectionGroup row with + its HGVS strings copied verbatim. Returns the created ``MappingRecord``. """ if isinstance(variant, str): resolved = session.scalar(select(Variant).where(Variant.urn == variant)) @@ -90,6 +105,25 @@ def seed_mapping_record( session.add(record) session.commit() + # Keyed by instance identity so equal-looking GroupSpecs stay distinct groups. + groups: dict[int, ProjectionGroup] = {} + for spec in alleles: + if spec.group is None or id(spec.group) in groups: + continue + group = ProjectionGroup( + mapping_record_id=record.id, + score_set_id=record.score_set_id, + hgvs_c=spec.group.hgvs_c, + hgvs_g=spec.group.hgvs_g, + hgvs_p=spec.group.hgvs_p, + ) + if valid_from is not None: + group.valid_from = valid_from + groups[id(spec.group)] = group + + session.add_all(groups.values()) + session.flush() + # Alleles are deduplicated by digest, so one seeded under the same digest before is reused (an allele # several records share, like a protein consequence). for spec in alleles: @@ -113,7 +147,7 @@ def seed_mapping_record( score_set_id=record.score_set_id, allele_id=allele.id, is_authoritative=spec.is_authoritative, - projection_group=spec.projection_group, + projection_group=groups[id(spec.group)].id if spec.group is not None else None, ) ] if spec.vep_consequence is not None: diff --git a/tests/helpers/util/score_set.py b/tests/helpers/util/score_set.py index 3801910d5..7a0f3df46 100644 --- a/tests/helpers/util/score_set.py +++ b/tests/helpers/util/score_set.py @@ -23,7 +23,7 @@ TEST_VALID_PRE_MAPPED_VRS_ALLELE_VRS2_X, TEST_VALID_PRE_MAPPED_VRS_CIS_PHASED_BLOCK, ) -from tests.helpers.util.annotation import AlleleSpec, seed_mapping_record +from tests.helpers.util.annotation import AlleleSpec, GroupSpec, seed_mapping_record from tests.helpers.util.variant import mock_worker_variant_insertion @@ -293,12 +293,18 @@ def seed_csv_substrate( level_hgvs = {"genomic": hgvs_g, "cdna": hgvs_c, "protein": hgvs_p} assay_hgvs = level_hgvs[assay_level] annotated = annotate == "all" or index == 0 + # A protein assay's group carries only the protein string (no c/g fan-out). + measured = ( + GroupSpec(hgvs_g=hgvs_g, hgvs_c=hgvs_c, hgvs_p=hgvs_p) + if assay_level in ("genomic", "cdna") + else GroupSpec(hgvs_p=hgvs_p) + ) specs = [ AlleleSpec( digest=f"csv-auth-{variant.id}", level=assay_level, is_authoritative=True, - projection_group=0, + group=measured, hgvs_g=hgvs_g if assay_level == "genomic" else None, hgvs_c=hgvs_c if assay_level == "cdna" else None, hgvs_p=hgvs_p if assay_level == "protein" else None, @@ -317,7 +323,7 @@ def seed_csv_substrate( AlleleSpec( digest=f"csv-sib-{variant.id}", level=other_level, - projection_group=0, + group=measured, hgvs_g=hgvs_g if other_level == "genomic" else None, hgvs_c=hgvs_c if other_level == "cdna" else None, ) diff --git a/tests/lib/annotation/test_context.py b/tests/lib/annotation/test_context.py index f0a5824bb..479e8f694 100644 --- a/tests/lib/annotation/test_context.py +++ b/tests/lib/annotation/test_context.py @@ -18,7 +18,7 @@ from mavedb.models.gnomad_allele_link import GnomadAlleleLink from mavedb.models.gnomad_variant import GnomADVariant from tests.helpers.constants import TEST_VALID_POST_MAPPED_VRS_ALLELE -from tests.helpers.util.annotation import AlleleSpec, seed_mapping_record +from tests.helpers.util.annotation import AlleleSpec, GroupSpec, seed_mapping_record T0 = datetime(2020, 1, 1, tzinfo=timezone.utc) @@ -81,6 +81,7 @@ def test_nucleotide_assay_subject_is_a_categorical_variant_over_the_measured_cha """Anchored on the measured nt change: keep its projection_group coordinate partner + the protein consequence, exclude the sibling encoder (a distinct variant, different projection group).""" mapped_variant = setup_lib_db_with_mapped_variant + measured, sibling = GroupSpec(), GroupSpec() seed_mapping_record( session, mapped_variant.variant, @@ -90,16 +91,16 @@ def test_nucleotide_assay_subject_is_a_categorical_variant_over_the_measured_cha digest="cdna", level="cdna", is_authoritative=True, - projection_group=0, + group=measured, post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE, ), # The measured change's precise coordinate partner (same projection group). AlleleSpec( - digest="gen", level="genomic", projection_group=0, post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE + digest="gen", level="genomic", group=measured, post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE ), # A sibling encoder of the same protein consequence — distinct variant, other group: excluded. AlleleSpec( - digest="sibling", level="cdna", projection_group=1, post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE + digest="sibling", level="cdna", group=sibling, post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE ), AlleleSpec(digest="prot", level="protein", post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE), ], @@ -145,6 +146,7 @@ def test_subject_mappings_cover_only_the_narrow_members_at_as_of(self, session, old_id = _gnomad_variant(session, "1-100-A-G") new_id = _gnomad_variant(session, "1-100-A-T") sibling_id = _gnomad_variant(session, "1-101-C-T") + measured, sibling = GroupSpec(), GroupSpec() seed_mapping_record( session, mapped_variant.variant, @@ -155,20 +157,20 @@ def test_subject_mappings_cover_only_the_narrow_members_at_as_of(self, session, digest="cdna", level="cdna", is_authoritative=True, - projection_group=0, + group=measured, post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE, ), AlleleSpec( digest="gen", level="genomic", - projection_group=0, + group=measured, post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE, gnomad_variant_ids=[old_id], ), AlleleSpec( digest="sibling", level="cdna", - projection_group=1, + group=sibling, post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE, gnomad_variant_ids=[sibling_id], ), diff --git a/tests/lib/csv/conftest.py b/tests/lib/csv/conftest.py index f79741225..29230b84c 100644 --- a/tests/lib/csv/conftest.py +++ b/tests/lib/csv/conftest.py @@ -7,7 +7,7 @@ pytest.importorskip("psycopg2") from tests.helpers.constants import TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION -from tests.helpers.util.annotation import AlleleSpec, seed_mapping_record +from tests.helpers.util.annotation import AlleleSpec, GroupSpec, seed_mapping_record @pytest.fixture @@ -20,9 +20,10 @@ def setup_lib_db_with_mapped_variant(session, setup_lib_db_with_variant, setup_l their own records, and a second live record per variant violates ``uq_mapping_records_current``. Seeded as the full g/c/p triple the CSV reconstructs from — an authoritative genomic allele, its - projection-group cdna sibling, and the protein apex — so a test can populate any level's HGVS by - setting it on that level's allele. Mirrors ``tests.helpers.util.score_set.seed_csv_substrate``. + projection-group cdna sibling, and the protein apex — so a test can populate a non-measured level's HGVS + by setting it on the measured projection group. Mirrors ``tests.helpers.util.score_set.seed_csv_substrate``. """ + measured = GroupSpec(hgvs_g=TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION["hgvs_g"]) seed_mapping_record( session, setup_lib_db_with_variant, @@ -33,11 +34,11 @@ def setup_lib_db_with_mapped_variant(session, setup_lib_db_with_variant, setup_l digest="lib-csv-genomic", level="genomic", is_authoritative=True, - projection_group=0, + group=measured, hgvs_g=TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION["hgvs_g"], vep_consequence=TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION["vep_functional_consequence"], ), - AlleleSpec(digest="lib-csv-cdna", level="cdna", projection_group=0), + AlleleSpec(digest="lib-csv-cdna", level="cdna", group=measured), AlleleSpec(digest="lib-csv-protein", level="protein"), ], ) diff --git a/tests/lib/csv/test_variant.py b/tests/lib/csv/test_variant.py index 71ecbc31a..0b67aec3d 100644 --- a/tests/lib/csv/test_variant.py +++ b/tests/lib/csv/test_variant.py @@ -4,7 +4,7 @@ import io from unittest.mock import Mock -from tests.helpers.util.annotation import AlleleSpec, seed_mapping_record +from tests.helpers.util.annotation import AlleleSpec, GroupSpec, seed_mapping_record from tests.helpers.util.query_plan import captured_statements from sqlalchemy import select from sqlalchemy.exc import IntegrityError @@ -37,6 +37,7 @@ from mavedb.models.mapped_variant import MappedVariant from mavedb.models.mapping_record import MappingRecord from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.projection_group import ProjectionGroup from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification from mavedb.models.score_set import ScoreSet @@ -246,6 +247,18 @@ def _live_mapping_record(db, variant_id): return record +def _measured_group(db, variant_id): + """The live record's measured projection group — where the CSV reads the non-measured levels from.""" + return db.scalars( + select(ProjectionGroup) + .join(MappingRecordAllele, MappingRecordAllele.projection_group == ProjectionGroup.id) + .join(MappingRecord, MappingRecord.id == MappingRecordAllele.mapping_record_id) + .where(MappingRecord.variant_id == variant_id) + .where(MappingRecord.valid_to.is_(None)) + .where(MappingRecordAllele.is_authoritative.is_(True)) + ).one() + + def _authoritative_allele(db, variant_id): allele = db.scalars( select(Allele) @@ -535,14 +548,14 @@ def test_variant_without_clingen_allele_id_stands_alone(self, session, setup_lib def test_mapped_coordinates_and_external_annotations(self, session, setup_lib_db_with_mapped_variant): mapped_variant = setup_lib_db_with_mapped_variant - # The triple reconstructs from three different places, not from one row's columns: the measured - # slot is the record's assay-level string, the other nucleotide slot is the projection-group - # partner's HGVS, and the protein slot is the apex allele's. + # Every level is the measured projection group's string, including the normalized measured one. by_level = _live_alleles_by_level(session, mapped_variant.variant_id) record = _live_mapping_record(session, mapped_variant.variant_id) - record.hgvs_assay_level = "NC_000010.11:g.87933147C>T" - by_level["cdna"].hgvs_c = "NM_000314.8:c.100A>G" - by_level["protein"].hgvs_p = "NP_000305.3:p.Lys34Glu" + record.hgvs_assay_level = "NC_000010.11:g.87933146_87933147delinsCT" + measured_group = _measured_group(session, mapped_variant.variant_id) + measured_group.hgvs_g = "NC_000010.11:g.87933147C>T" + measured_group.hgvs_c = "NM_000314.8:c.100A>G" + measured_group.hgvs_p = "NP_000305.3:p.Lys34Glu" authoritative = by_level["genomic"] authoritative.vrs_digest = TEST_GA4GH_IDENTIFIER authoritative.clingen_allele_id = "CA123456" @@ -560,6 +573,31 @@ def test_mapped_coordinates_and_external_annotations(self, session, setup_lib_db assert rows[0]["gnomad.gnomad_af"] == str(TEST_GNOMAD_VARIANT["allele_frequency"]) assert rows[0]["clingen.clingen_allele_id"] == "CA123456" + def test_coding_projection_without_an_allele_is_served(self, session, setup_lib_db_with_variant): + """An intronic genomic change has a coding coordinate but no cdna VRS allele: the CSV serves the measured + group's hgvs_c anyway, and no protein consequence.""" + variant = setup_lib_db_with_variant + seed_mapping_record( + session, + variant, + assay_level="genomic", + hgvs_assay_level="NC_000010.11:g.87933147C>T", + alleles=[ + AlleleSpec( + digest="intronic-genomic", + level="genomic", + is_authoritative=True, + group=GroupSpec(hgvs_c="NM_000465.4:c.123+5A>G", hgvs_g="NC_000010.11:g.87933147C>T"), + ) + ], + ) + + rows = _parse_csv(get_variant_csv(session, variant.urn)) + + assert rows[0]["mavedb.post_mapped_hgvs_g"] == "NC_000010.11:g.87933147C>T" + assert rows[0]["mavedb.post_mapped_hgvs_c"] == "NM_000465.4:c.123+5A>G" + assert rows[0]["mavedb.post_mapped_hgvs_p"] == "NA" + def test_post_mapped_vrs_id_is_the_stored_identifier_verbatim(self, session, setup_lib_db_with_mapped_variant): """The column is reported as-is, never rebuilt from the post-mapped payload. diff --git a/tests/lib/test_allele_detail.py b/tests/lib/test_allele_detail.py index 75375b7a4..9402d400a 100644 --- a/tests/lib/test_allele_detail.py +++ b/tests/lib/test_allele_detail.py @@ -21,7 +21,7 @@ from mavedb.models.allele import Allele from mavedb.models.variant import Variant from tests.helpers.constants import TEST_MINIMAL_VARIANT -from tests.helpers.util.annotation import AlleleSpec, seed_mapping_record +from tests.helpers.util.annotation import AlleleSpec, GroupSpec, seed_mapping_record T0 = datetime(2020, 1, 1, tzinfo=timezone.utc) T1 = datetime(2021, 1, 1, tzinfo=timezone.utc) @@ -66,7 +66,10 @@ def _allele(session, digest) -> Allele: def _coding_measured_specs(): """A coding-measured record: authoritative cdna (+VEP+ClinGen), its genomic projection partner - (shared projection_group 0), the protein apex, and a synonymous cousin (own projection_group 1).""" + (shared measured group), the protein apex, and a synonymous cousin (its own group).""" + measured = GroupSpec( + hgvs_c="NM_000546.6:c.1216G>A", hgvs_g="NC_000017.11:g.7676154C>T", hgvs_p="NP_000537.3:p.Ala406Thr" + ) return [ AlleleSpec( digest="cdna-d", @@ -76,7 +79,7 @@ def _coding_measured_specs(): hgvs_c="NM_000546.6:c.1216G>A", post_mapped=_post_mapped(), vep_consequence="missense_variant", - projection_group=0, + group=measured, ), AlleleSpec( digest="gen-d", @@ -84,7 +87,7 @@ def _coding_measured_specs(): clingen_allele_id="CA1", hgvs_g="NC_000017.11:g.7676154C>T", post_mapped=_post_mapped(), - projection_group=0, + group=measured, ), AlleleSpec(digest="prot-d", level="protein", hgvs_p="NP_000537.3:p.Ala406Thr", post_mapped=_post_mapped()), AlleleSpec( @@ -92,7 +95,7 @@ def _coding_measured_specs(): level="cdna", hgvs_c="NM_000546.6:c.1218C>T", post_mapped=_post_mapped(), - projection_group=1, + group=GroupSpec(hgvs_c="NM_000546.6:c.1218C>T"), ), ] diff --git a/tests/lib/test_score_set_variants.py b/tests/lib/test_score_set_variants.py index 130e17747..9a516bc3e 100644 --- a/tests/lib/test_score_set_variants.py +++ b/tests/lib/test_score_set_variants.py @@ -19,6 +19,7 @@ from mavedb.models.allele import Allele from mavedb.models.mapping_record import MappingRecord from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.projection_group import ProjectionGroup from mavedb.models.variant import Variant from mavedb.models.vep_allele_consequence import VepAlleleConsequence from tests.helpers.constants import TEST_MINIMAL_VARIANT @@ -76,13 +77,27 @@ def _record(session, variant, *, assay_level="cdna", hgvs_assay_level=None, vali return record -def _link(session, record, allele, *, is_authoritative=True, projection_group=None, valid_from=None): +def _group(session, record, *, hgvs_c=None, hgvs_g=None, hgvs_p=None, valid_from=None): + group = ProjectionGroup( + mapping_record_id=record.id, + score_set_id=record.score_set_id, + hgvs_c=hgvs_c, + hgvs_g=hgvs_g, + hgvs_p=hgvs_p, + valid_from=valid_from, + ) + session.add(group) + session.commit() + return group + + +def _link(session, record, allele, *, is_authoritative=True, group=None, valid_from=None): link = MappingRecordAllele( mapping_record_id=record.id, score_set_id=record.score_set_id, allele_id=allele.id, is_authoritative=is_authoritative, - projection_group=projection_group, + projection_group=group.id if group is not None else None, valid_from=valid_from, ) session.add(link) @@ -102,19 +117,24 @@ def _consequence(session, allele, value): @pytest.mark.integration def test_nucleotide_assay_measured_at_cdna(session, setup_lib_db_with_score_set): """Coding assay: submitted nt+pro from the variant; the mapped triple carries the measured cdna slot - (from the record), its projection_group genomic sibling, and the protein apex; digest/ClinGen - id/consequence off the authoritative allele. Each parseable string gets its block.""" + (from the record) and the measured group's genomic and protein strings; digest/ClinGen id/consequence off + the authoritative allele. Each parseable string gets its block.""" score_set = setup_lib_db_with_score_set variant = _variant(session, score_set, 1, score=-2.3, hgvs_nt="c.1A>T", hgvs_pro="p.Thr1Ser") record = _record(session, variant, assay_level="cdna", hgvs_assay_level="NM_000546.6:c.1216G>A") measured = _allele(session, "cdna-digest", level="cdna", clingen_allele_id="CA123") genomic = _allele(session, "gen-digest", level="genomic", hgvs_g="NC_000017.11:g.7676154C>T") protein = _allele(session, "prot-digest", level="protein", hgvs_p="NP_000537.3:p.Ala406Thr") - # The measured cdna link and its genomic projection share a projection_group (the RT fold-in); the - # protein apex is a member of no pair (group None). - _link(session, record, measured, is_authoritative=True, projection_group=0) - _link(session, record, genomic, is_authoritative=False, projection_group=0) - _link(session, record, protein, is_authoritative=False) + measured_group = _group( + session, + record, + hgvs_c="NM_000546.6:c.1216G>A", + hgvs_g="NC_000017.11:g.7676154C>T", + hgvs_p="NP_000537.3:p.Ala406Thr", + ) + _link(session, record, measured, is_authoritative=True, group=measured_group) + _link(session, record, genomic, is_authoritative=False, group=measured_group) + _link(session, record, protein, is_authoritative=False, group=measured_group) _consequence(session, measured, "missense_variant") [record_out] = get_lean_score_set_variants(session, score_set) @@ -139,22 +159,29 @@ def test_nucleotide_assay_measured_at_cdna(session, setup_lib_db_with_score_set) @pytest.mark.integration def test_nucleotide_assay_measured_at_genomic(session, setup_lib_db_with_score_set): """Genomic assay: the measured slot is genomic (from the record); the cdna slot — the level-invariant - search key — comes off the projection_group sibling even though the assay was measured at genomic.""" + search key — comes off the measured group even though the assay was measured at genomic.""" score_set = setup_lib_db_with_score_set variant = _variant(session, score_set, 1, score=0.1, hgvs_nt="g.7676154C>T") record = _record(session, variant, assay_level="genomic", hgvs_assay_level="NC_000017.11:g.7676154C>T") measured = _allele(session, "gen-digest", level="genomic", clingen_allele_id="CA9") cdna = _allele(session, "cdna-digest", level="cdna", hgvs_c="NM_000546.6:c.1216G>A") protein = _allele(session, "prot-digest", level="protein", hgvs_p="NP_000537.3:p.Ala406Thr") - _link(session, record, measured, is_authoritative=True, projection_group=0) - _link(session, record, cdna, is_authoritative=False, projection_group=0) - _link(session, record, protein, is_authoritative=False) + measured_group = _group( + session, + record, + hgvs_c="NM_000546.6:c.1216G>A", + hgvs_g="NC_000017.11:g.7676154C>T", + hgvs_p="NP_000537.3:p.Ala406Thr", + ) + _link(session, record, measured, is_authoritative=True, group=measured_group) + _link(session, record, cdna, is_authoritative=False, group=measured_group) + _link(session, record, protein, is_authoritative=False, group=measured_group) [record_out] = get_lean_score_set_variants(session, score_set) assert record_out.assay_level == "genomic" assert record_out.assay_level_digest == "gen-digest" - # Measured slot is genomic (from the record); cdna (search key) comes off the sibling allele. + # Measured slot is genomic (from the record); cdna (search key) comes off the measured group. assert record_out.mapped == MappedTriple( genomic=HgvsField(hgvs="NC_000017.11:g.7676154C>T", position=7676154, ref="C", alt="T"), cdna=HgvsField(hgvs="NM_000546.6:c.1216G>A", position=1216, ref="G", alt="A"), @@ -172,11 +199,17 @@ def test_protein_assay_serves_only_the_protein_slot(session, setup_lib_db_with_s measured = _allele( session, "prot-digest", level="protein", clingen_allele_id="PA9", hgvs_p="NP_000537.3:p.Ala406Thr" ) - _link(session, record, measured, is_authoritative=True) - # A candidate fan-out member (cdna) with its own projection pair group — the protein authoritative link is - # in no group, so this candidate must NOT be pulled into the canonical triple. + _link( + session, + record, + measured, + is_authoritative=True, + group=_group(session, record, hgvs_p="NP_000537.3:p.Ala406Thr"), + ) + # A candidate fan-out member (cdna) in its own candidate group must NOT be pulled into the canonical triple. candidate = _allele(session, "cand-digest", level="cdna", hgvs_c="NM_000546.6:c.1216G>A") - _link(session, record, candidate, is_authoritative=False, projection_group=0) + candidate_group = _group(session, record, hgvs_c="NM_000546.6:c.1216G>A") + _link(session, record, candidate, is_authoritative=False, group=candidate_group) [record_out] = get_lean_score_set_variants(session, score_set) @@ -190,10 +223,9 @@ def test_protein_assay_serves_only_the_protein_slot(session, setup_lib_db_with_s @pytest.mark.integration -def test_unpopulated_projection_group_degrades_to_a_null_sibling(session, setup_lib_db_with_score_set): - """Pre-reverse-translation data (no projection_group on the authoritative link): the sibling join - returns nothing, so the other nucleotide slot stays null — today's behavior — while the measured and - protein slots still populate. Still one row per variant.""" +def test_unwritten_measured_group_serves_an_empty_triple(session, setup_lib_db_with_score_set): + """Between mapping and the projection step the authoritative link has no group: the triple is empty, + even with a protein allele linked to the record. Still one row per variant.""" score_set = setup_lib_db_with_score_set variant = _variant(session, score_set, 1, score=-2.3, hgvs_nt="c.1A>T") record = _record(session, variant, assay_level="cdna", hgvs_assay_level="NM_000546.6:c.1216G>A") @@ -205,10 +237,25 @@ def test_unpopulated_projection_group_degrades_to_a_null_sibling(session, setup_ [record_out] = get_lean_score_set_variants(session, score_set) assert record_out.assay_level == "cdna" - # Measured cdna + protein apex populate; genomic is null (no recorded sibling — graceful degradation). + assert record_out.mapped == MappedTriple() + + +@pytest.mark.integration +def test_genomic_assay_serves_a_coding_projection_with_no_allele(session, setup_lib_db_with_score_set): + """An intronic genomic change has a coding coordinate but no cdna VRS allele: the cdna slot comes off the + measured group's string alone, and the protein slot stays null (no consequence).""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, hgvs_nt="g.7676154C>T") + record = _record(session, variant, assay_level="genomic", hgvs_assay_level="NC_000017.11:g.7676154C>T") + measured = _allele(session, "gen-digest", level="genomic", hgvs_g="NC_000017.11:g.7676154C>T") + measured_group = _group(session, record, hgvs_c="NM_000465.4:c.123+5A>G", hgvs_g="NC_000017.11:g.7676154C>T") + _link(session, record, measured, is_authoritative=True, group=measured_group) + + [record_out] = get_lean_score_set_variants(session, score_set) + assert record_out.mapped == MappedTriple( - cdna=HgvsField(hgvs="NM_000546.6:c.1216G>A", position=1216, ref="G", alt="A"), - protein=HgvsField(hgvs="NP_000537.3:p.Ala406Thr", position=406, ref="Ala", alt="Thr"), + genomic=HgvsField(hgvs="NC_000017.11:g.7676154C>T", position=7676154, ref="C", alt="T"), + cdna=HgvsField(hgvs="NM_000465.4:c.123+5A>G"), ) @@ -219,7 +266,7 @@ def test_hgvs_string_rides_even_when_unparseable(session, setup_lib_db_with_scor variant = _variant(session, score_set, 1, hgvs_nt="NM_000546.6:c.[197A>G;472T>C]", hgvs_splice="c.122-6T>A") record = _record(session, variant, assay_level="cdna", hgvs_assay_level="c.76_78del") measured = _allele(session, "cdna-digest", level="cdna") - _link(session, record, measured, is_authoritative=True) + _link(session, record, measured, is_authoritative=True, group=_group(session, record, hgvs_c="c.76_78del")) [record_out] = get_lean_score_set_variants(session, score_set) @@ -256,7 +303,9 @@ def test_no_vep_consequence_is_none(session, setup_lib_db_with_score_set): variant = _variant(session, score_set, 1, hgvs_nt="c.1A>T") record = _record(session, variant, assay_level="cdna", hgvs_assay_level="NM_000546.6:c.1216G>A") measured = _allele(session, "cdna-digest", level="cdna") - _link(session, record, measured, is_authoritative=True) + _link( + session, record, measured, is_authoritative=True, group=_group(session, record, hgvs_c="NM_000546.6:c.1216G>A") + ) [record_out] = get_lean_score_set_variants(session, score_set) @@ -286,12 +335,26 @@ def test_as_of_reconstructs_the_historical_assay_level(session, setup_lib_db_wit old_record = _record(session, variant, assay_level="cdna", hgvs_assay_level="NM_000546.6:c.1216G>A", valid_from=T0) old_allele = _allele(session, "old-digest", level="cdna") - _link(session, old_record, old_allele, is_authoritative=True, valid_from=T0) + _link( + session, + old_record, + old_allele, + is_authoritative=True, + group=_group(session, old_record, hgvs_c="NM_000546.6:c.1216G>A", valid_from=T0), + valid_from=T0, + ) old_record.retire(session, at=T1) # cascades to the link via __retire_cascade__ new_record = _record(session, variant, assay_level="cdna", hgvs_assay_level="NM_000546.6:c.500A>T", valid_from=T1) new_allele = _allele(session, "new-digest", level="cdna") - _link(session, new_record, new_allele, is_authoritative=True, valid_from=T1) + _link( + session, + new_record, + new_allele, + is_authoritative=True, + group=_group(session, new_record, hgvs_c="NM_000546.6:c.500A>T", valid_from=T1), + valid_from=T1, + ) session.commit() [current] = get_lean_score_set_variants(session, score_set) @@ -323,9 +386,17 @@ def test_per_variant_lookups_cannot_become_whole_table_reads(session, setup_lib_ measured = _allele(session, "cdna-digest", level="cdna") genomic = _allele(session, "gen-digest", level="genomic", hgvs_g="NC_000017.11:g.7676154C>T") protein = _allele(session, "prot-digest", level="protein", hgvs_p="NP_000537.3:p.Ala406Thr") - _link(session, record, measured, is_authoritative=True, projection_group=0, valid_from=T0) - _link(session, record, genomic, is_authoritative=False, projection_group=0, valid_from=T0) - _link(session, record, protein, is_authoritative=False, valid_from=T0) + measured_group = _group( + session, + record, + hgvs_c="NM_000546.6:c.1216G>A", + hgvs_g="NC_000017.11:g.7676154C>T", + hgvs_p="NP_000537.3:p.Ala406Thr", + valid_from=T0, + ) + _link(session, record, measured, is_authoritative=True, group=measured_group, valid_from=T0) + _link(session, record, genomic, is_authoritative=False, group=measured_group, valid_from=T0) + _link(session, record, protein, is_authoritative=False, group=measured_group, valid_from=T0) with captured_statements(session) as captured: get_lean_score_set_variants(session, score_set, as_of=as_of) diff --git a/tests/lib/test_variant_detail.py b/tests/lib/test_variant_detail.py index 60151cb84..edd380c8e 100644 --- a/tests/lib/test_variant_detail.py +++ b/tests/lib/test_variant_detail.py @@ -19,6 +19,7 @@ from mavedb.lib.annotation.context import load_annotation_context_inputs, variant_annotation_context from mavedb.lib.allele_detail import get_allele_detail from mavedb.lib.allele_identity import AlleleDerivation +from mavedb.lib.score_set_variants import HgvsField, MappedTriple from mavedb.lib.variant_detail import get_variant_detail, load_variant_detail_inputs from mavedb.lib.mondo import get_generic_disease_term from mavedb.models.allele import Allele @@ -28,6 +29,7 @@ from mavedb.models.gnomad_variant import GnomADVariant from mavedb.models.mapping_record import MappingRecord from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.projection_group import ProjectionGroup from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification from mavedb.models.user import User @@ -104,13 +106,22 @@ def _record(session, variant, *, assay_level="cdna", hgvs_assay_level=None): return record -def _link(session, record, allele, *, is_authoritative=False, projection_group=None): +def _group(session, record, *, hgvs_c=None, hgvs_g=None, hgvs_p=None): + group = ProjectionGroup( + mapping_record_id=record.id, score_set_id=record.score_set_id, hgvs_c=hgvs_c, hgvs_g=hgvs_g, hgvs_p=hgvs_p + ) + session.add(group) + session.commit() + return group + + +def _link(session, record, allele, *, is_authoritative=False, group=None): link = MappingRecordAllele( mapping_record_id=record.id, score_set_id=record.score_set_id, allele_id=allele.id, is_authoritative=is_authoritative, - projection_group=projection_group, + projection_group=group.id if group is not None else None, ) session.add(link) session.commit() @@ -166,16 +177,29 @@ def test_full_envelope_for_a_coding_assay(session, setup_lib_db_with_score_set): # A synonymous cousin: an nt encoder of the same protein consequence in a *different* projection group # — a distinct, unmeasured variant carried by the reverse-translation fan. cousin = _allele(session, "cousin-digest", level="cdna", hgvs_c="NM_000546.6:c.1218C>T") - # The measured cdna link and its genomic projection share a projection_group (the RT fold-in); the - # protein apex is in no pair (group None); the cousin is in its own group. - _link(session, record, measured, is_authoritative=True, projection_group=0) - _link(session, record, genomic, projection_group=0) - _link(session, record, protein) - _link(session, record, cousin, projection_group=1) + # The measured group holds the measured cdna, its genomic projection, and the protein consequence; the + # cousin is in its own candidate group. + measured_group = _group( + session, + record, + hgvs_c="NM_000546.6:c.1216G>A", + hgvs_g="NC_000017.11:g.7676154C>T", + hgvs_p="NP_000537.3:p.Ala406Thr", + ) + _link(session, record, measured, is_authoritative=True, group=measured_group) + _link(session, record, genomic, group=measured_group) + _link(session, record, protein, group=measured_group) + _link(session, record, cousin, group=_group(session, record, hgvs_c="NM_000546.6:c.1218C>T")) _vep(session, measured, "missense_variant") detail = get_variant_detail(session, variant) + assert detail.mapped == MappedTriple( + cdna=HgvsField(hgvs="NM_000546.6:c.1216G>A", position=1216, ref="G", alt="A"), + genomic=HgvsField(hgvs="NC_000017.11:g.7676154C>T", position=7676154, ref="C", alt="T"), + protein=HgvsField(hgvs="NP_000537.3:p.Ala406Thr", position=406, ref="Ala", alt="Thr"), + ) + assert detail.urn == variant.urn assert detail.scores == {"score": -2.3} assert detail.assay_level == "cdna" @@ -214,10 +238,10 @@ def test_full_envelope_for_a_coding_assay(session, setup_lib_db_with_score_set): assert protein_identity.hgvs == "NP_000537.3:p.Ala406Thr" # coalesced from hgvs_p # The protein member is a translation of the measured coding allele (member -> defining)... assert protein_identity.relation == "translation_of" - # ...and, on a nucleotide assay, is itself a deterministic projection (axes are independent) with no - # projection sibling (the apex is in no c/g pair). + # ...and, on a nucleotide assay, is itself a deterministic projection (axes are independent) that pairs + # with the measured allele it shares the measured group with. assert protein_identity.derivation == "projection" - assert protein_identity.projection_of is None + assert protein_identity.projection_of == "cdna-digest" # The synonymous cousin (different projection group) is a member wearing co_encodes (structural), and # labelled `convergent` (provenance) — a distinct, precise change that converges on the measured # consequence, not an ambiguous candidate and not a projection of the measured change. It pairs with @@ -251,8 +275,9 @@ def test_protein_assay_targets_the_protein_frame(session, setup_lib_db_with_scor genomic = _allele(session, "gen-digest", level="genomic", hgvs_g="NC_000017.11:g.7676154C>T") _link(session, record, measured, is_authoritative=True) # protein apex — group None # One reverse-translation projection pair: a coding candidate paired with its genomic projection. - _link(session, record, coding, projection_group=0) - _link(session, record, genomic, projection_group=0) + candidate_group = _group(session, record, hgvs_c="NM_000546.6:c.1216G>A", hgvs_g="NC_000017.11:g.7676154C>T") + _link(session, record, coding, group=candidate_group) + _link(session, record, genomic, group=candidate_group) detail = get_variant_detail(session, variant) @@ -302,6 +327,64 @@ def test_pre_reverse_translation_data_degrades_projection_of_to_null(session, se assert detail.alleles["prot-digest"].projection_of is None +@pytest.mark.integration +def test_mapped_triple_carries_a_coding_projection_with_no_allele(session, setup_lib_db_with_score_set): + """An intronic genomic change has a coding coordinate but no cdna VRS allele: ``mapped`` serves the + measured group's hgvs_c anyway, and no protein consequence.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, hgvs_nt="g.7676154C>T") + record = _record(session, variant, assay_level="genomic", hgvs_assay_level="NC_000017.11:g.7676154C>T") + measured = _allele(session, "gen-digest", level="genomic", hgvs_g="NC_000017.11:g.7676154C>T") + measured_group = _group( + session, record, hgvs_c="NM_000465.4:c.123+5A>G", hgvs_g="NC_000017.11:g.7676154C>T", hgvs_p=None + ) + _link(session, record, measured, is_authoritative=True, group=measured_group) + + detail = get_variant_detail(session, variant) + + assert detail.mapped == MappedTriple( + genomic=HgvsField(hgvs="NC_000017.11:g.7676154C>T", position=7676154, ref="C", alt="T"), + cdna=HgvsField(hgvs="NM_000465.4:c.123+5A>G"), + ) + assert set(detail.alleles) == {"gen-digest"} + assert detail.alleles["gen-digest"].projection_of is None + + +@pytest.mark.integration +def test_protein_link_in_the_measured_group_pairs_with_the_measured_allele(session, setup_lib_db_with_score_set): + """The protein consequence is the part of the measurement the score describes, so it pairs with the + measured allele; the nucleotide pair still points at each other rather than at it.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, hgvs_nt="c.1216G>A") + record = _record(session, variant, assay_level="cdna", hgvs_assay_level="NM_000546.6:c.1216G>A") + measured = _allele(session, "cdna-digest", level="cdna", hgvs_c="NM_000546.6:c.1216G>A") + genomic = _allele(session, "gen-digest", level="genomic", hgvs_g="NC_000017.11:g.7676154C>T") + protein = _allele(session, "prot-digest", level="protein", hgvs_p="NP_000537.3:p.Ala406Thr") + measured_group = _group( + session, + record, + hgvs_c="NM_000546.6:c.1216G>A", + hgvs_g="NC_000017.11:g.7676154C>T", + hgvs_p="NP_000537.3:p.Ala406Thr", + ) + # Protein linked first, so a pairing that ignored level would pick it as the cdna allele's partner. + _link(session, record, protein, group=measured_group) + _link(session, record, measured, is_authoritative=True, group=measured_group) + _link(session, record, genomic, group=measured_group) + + variant_alleles = get_variant_detail(session, variant).alleles + assert variant_alleles["prot-digest"].projection_of == "cdna-digest" + assert variant_alleles["cdna-digest"].projection_of == "gen-digest" + assert variant_alleles["gen-digest"].projection_of == "cdna-digest" + + nucleotide_focus = get_allele_detail(session, measured, focus_digests={"cdna-digest"}).alleles + assert nucleotide_focus["cdna-digest"].projection_of == "gen-digest" + assert nucleotide_focus["gen-digest"].projection_of == "cdna-digest" + + protein_focus = get_allele_detail(session, protein, focus_digests={"prot-digest"}).alleles + assert protein_focus["prot-digest"].projection_of == "cdna-digest" + + T0 = datetime(2020, 1, 1, tzinfo=timezone.utc) T1 = datetime(2021, 1, 1, tzinfo=timezone.utc) @@ -335,9 +418,10 @@ def test_categorical_variant_cross_references_linked_gnomad_and_clinvar_records( measured = _allele(session, "cdna-digest", level="cdna", clingen_allele_id="CA123") genomic = _allele(session, "gen-digest", level="genomic", clingen_allele_id="CA123") cousin = _allele(session, "cousin-digest", level="cdna") - _link(session, record, measured, is_authoritative=True, projection_group=0) - _link(session, record, genomic, projection_group=0) - _link(session, record, cousin, projection_group=1) + measured_group = _group(session, record) + _link(session, record, measured, is_authoritative=True, group=measured_group) + _link(session, record, genomic, group=measured_group) + _link(session, record, cousin, group=_group(session, record)) _gnomad_link(session, genomic, "17-7676154-C-T") _gnomad_link(session, cousin, "17-7676152-G-A") @@ -376,11 +460,13 @@ def test_categorical_variant_mappings_follow_as_of(session, setup_lib_db_with_sc record = _record(session, variant, assay_level="cdna") measured = _allele(session, "cdna-digest", level="cdna") genomic = _allele(session, "gen-digest", level="genomic") - for link in ( - _link(session, record, measured, is_authoritative=True, projection_group=0), - _link(session, record, genomic, projection_group=0), + measured_group = _group(session, record) + for row in ( + _link(session, record, measured, is_authoritative=True, group=measured_group), + _link(session, record, genomic, group=measured_group), + measured_group, ): - link.valid_from = T0 + row.valid_from = T0 record.valid_from = T0 _gnomad_link(session, genomic, "17-1-A-G", valid_from=T0).retire(at=T1) _gnomad_link(session, genomic, "17-1-A-T", valid_from=T1) @@ -473,9 +559,16 @@ def _seed_mapped_variants(session, score_set, count): record = _record(session, variant, hgvs_assay_level=f"NM_000546.6:c.{1216 + index}G>A") measured = _allele(session, f"cdna-{index}", level="cdna", hgvs_c=f"NM_000546.6:c.{1216 + index}G>A") genomic = _allele(session, f"gen-{index}", level="genomic", hgvs_g=f"NC_000017.11:g.{7676154 + index}C>T") - _link(session, record, measured, is_authoritative=True, projection_group=0) - _link(session, record, genomic, projection_group=0) - _link(session, record, protein) + measured_group = _group( + session, + record, + hgvs_c=measured.hgvs_c, + hgvs_g=genomic.hgvs_g, + hgvs_p=protein.hgvs_p, + ) + _link(session, record, measured, is_authoritative=True, group=measured_group) + _link(session, record, genomic, group=measured_group) + _link(session, record, protein, group=measured_group) _vep(session, measured, "missense_variant") variants.append(variant) _calibration(session, score_set, primary=True, classifications=[("abnormal", variants)]) @@ -527,8 +620,8 @@ def test_batched_loads_run_a_fixed_number_of_queries(session, setup_lib_db_with_ @pytest.mark.integration -@pytest.mark.parametrize("measured_group", [0, None], ids=["folded", "fold_in_missed"]) -def test_variant_and_allele_detail_label_members_alike(session, setup_lib_db_with_score_set, measured_group): +@pytest.mark.parametrize("folded", [True, False], ids=["folded", "fold_in_missed"]) +def test_variant_and_allele_detail_label_members_alike(session, setup_lib_db_with_score_set, folded): """Both views label each member relative to the measured allele through one function, so they agree, including when the measured allele has no projection group and an ungrouped member sits beside it.""" score_set = setup_lib_db_with_score_set @@ -539,10 +632,11 @@ def test_variant_and_allele_detail_label_members_alike(session, setup_lib_db_wit ungrouped = _allele(session, "ungrouped", level="genomic", hgvs_g="NC_000017.11:g.7676160C>T") convergent = _allele(session, "convergent", level="cdna", hgvs_c="NM_000546.6:c.1218C>T") protein = _allele(session, "protein", level="protein", hgvs_p="NP_000537.3:p.Ala406Thr") - _link(session, record, measured, is_authoritative=True, projection_group=measured_group) - _link(session, record, twin, projection_group=0) + twin_group = _group(session, record, hgvs_c=measured.hgvs_c, hgvs_g=twin.hgvs_g) + _link(session, record, measured, is_authoritative=True, group=twin_group if folded else None) + _link(session, record, twin, group=twin_group) _link(session, record, ungrouped) - _link(session, record, convergent, projection_group=1) + _link(session, record, convergent, group=_group(session, record, hgvs_c=convergent.hgvs_c)) _link(session, record, protein) variant_alleles = get_variant_detail(session, variant).alleles @@ -551,4 +645,4 @@ def test_variant_and_allele_detail_label_members_alike(session, setup_lib_db_wit labels = {digest: (identity.relation, identity.derivation) for digest, identity in variant_alleles.items()} assert labels == {digest: (identity.relation, identity.derivation) for digest, identity in allele_alleles.items()} assert labels["ungrouped"] == ("co_encodes", AlleleDerivation.CONVERGENT) - assert labels["twin"][0] == ("coordinate_representation_of" if measured_group == 0 else "co_encodes") + assert labels["twin"][0] == ("coordinate_representation_of" if folded else "co_encodes") diff --git a/tests/routers/test_allele.py b/tests/routers/test_allele.py index 728700aab..bf995552c 100644 --- a/tests/routers/test_allele.py +++ b/tests/routers/test_allele.py @@ -22,7 +22,7 @@ from mavedb.models.variant import Variant from mavedb.view_models.allele_detail import AlleleDetail from tests.helpers.dependency_overrider import DependencyOverrider -from tests.helpers.util.annotation import AlleleSpec, seed_mapping_record +from tests.helpers.util.annotation import AlleleSpec, GroupSpec, seed_mapping_record from tests.helpers.util.experiment import create_experiment from tests.helpers.util.score_set import create_seq_score_set_with_variants from tests.helpers.util.user import change_ownership @@ -65,6 +65,9 @@ def _post_mapped() -> dict: def _seed(session, variant_urn): """Coding-measured record: authoritative cdna (+ClinGen CA1 +VEP), its genomic partner (also CA1), the protein apex, and a synonymous cousin in its own projection group.""" + measured = GroupSpec( + hgvs_c="NM_000546.6:c.1216G>A", hgvs_g="NC_000017.11:g.7676154C>T", hgvs_p="NP_000537.3:p.Ala406Thr" + ) seed_mapping_record( session, variant_urn, @@ -77,7 +80,7 @@ def _seed(session, variant_urn): hgvs_c="NM_000546.6:c.1216G>A", post_mapped=_post_mapped(), vep_consequence="missense_variant", - projection_group=0, + group=measured, ), AlleleSpec( digest=GEN, @@ -85,7 +88,7 @@ def _seed(session, variant_urn): clingen_allele_id="CA1", hgvs_g="NC_000017.11:g.7676154C>T", post_mapped=_post_mapped(), - projection_group=0, + group=measured, ), AlleleSpec( digest=PROT, @@ -99,7 +102,7 @@ def _seed(session, variant_urn): level="cdna", hgvs_c="NM_000546.6:c.1218C>T", post_mapped=_post_mapped(), - projection_group=1, + group=GroupSpec(hgvs_c="NM_000546.6:c.1218C>T"), ), ], assay_level="cdna", diff --git a/tests/routers/test_query_scaling.py b/tests/routers/test_query_scaling.py index 9c31b9f17..3715ffca3 100644 --- a/tests/routers/test_query_scaling.py +++ b/tests/routers/test_query_scaling.py @@ -35,7 +35,7 @@ TEST_PUBMED_IDENTIFIER, TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X, ) -from tests.helpers.util.annotation import AlleleSpec, seed_mapping_record +from tests.helpers.util.annotation import AlleleSpec, GroupSpec, seed_mapping_record from tests.helpers.util.common import deepcamelize from tests.helpers.util.experiment import create_experiment from tests.helpers.util.query_plan import captured_statements @@ -158,6 +158,7 @@ def _seed_world(client, session, data_provider, scores_csv, label: str) -> _Worl ).all() vrs = TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X for index, variant in enumerate(variants): + measured, convergent = GroupSpec(), GroupSpec() seed_mapping_record( session, variant, @@ -167,16 +168,16 @@ def _seed_world(client, session, data_provider, scores_csv, label: str) -> _Worl digest=_digest(f"{label}-measured-{index}"), is_authoritative=True, clingen_allele_id=f"CA{label}{index}", - projection_group=0, + group=measured, vep_consequence="missense_variant", clinvar_control_ids=(1,), gnomad_variant_ids=(1,), post_mapped=vrs, ), AlleleSpec( - digest=_digest(f"{label}-genomic-{index}"), level="genomic", projection_group=0, post_mapped=vrs + digest=_digest(f"{label}-genomic-{index}"), level="genomic", group=measured, post_mapped=vrs ), - AlleleSpec(digest=_digest(f"{label}-convergent-{index}"), projection_group=1, post_mapped=vrs), + AlleleSpec(digest=_digest(f"{label}-convergent-{index}"), group=convergent, post_mapped=vrs), AlleleSpec( digest=_digest(f"{label}-protein"), level="protein", clingen_allele_id=f"PA{label}", post_mapped=vrs ), diff --git a/tests/routers/test_score_set.py b/tests/routers/test_score_set.py index b29d2ef4f..fae869f33 100644 --- a/tests/routers/test_score_set.py +++ b/tests/routers/test_score_set.py @@ -81,7 +81,7 @@ VALID_VARIANT_URN, ) from tests.helpers.dependency_overrider import DependencyOverrider -from tests.helpers.util.annotation import AlleleSpec, seed_mapping_record +from tests.helpers.util.annotation import AlleleSpec, GroupSpec, seed_mapping_record from tests.helpers.util.common import ( create_failing_side_effect, deepcamelize, @@ -6088,6 +6088,9 @@ def _seed_lean_mapping(session, variant_urn): """Give one variant a live coding-measured mapping record (with an assay-level HGVS) whose authoritative allele carries a digest + ClinGen id + a live VEP consequence, plus its canonical genomic projection sibling (shared ``projection_group``) and the protein apex — the full triple.""" + measured = GroupSpec( + hgvs_c="NM_000546.6:c.1216G>A", hgvs_g="NC_000017.11:g.7676154C>T", hgvs_p="NP_000537.3:p.Ala406Thr" + ) seed_mapping_record( session, variant_urn, @@ -6100,13 +6103,13 @@ def _seed_lean_mapping(session, variant_urn): is_authoritative=True, clingen_allele_id="CA123", vep_consequence="missense_variant", - projection_group=0, + group=measured, ), AlleleSpec( digest="gen-digest", level="genomic", hgvs_g="NC_000017.11:g.7676154C>T", - projection_group=0, + group=measured, ), AlleleSpec(digest="prot-digest", level="protein", hgvs_p="NP_000537.3:p.Ala406Thr"), ], diff --git a/tests/routers/test_variant.py b/tests/routers/test_variant.py index f9a928066..0847cce0d 100644 --- a/tests/routers/test_variant.py +++ b/tests/routers/test_variant.py @@ -25,6 +25,7 @@ from mavedb.models.allele import Allele from mavedb.models.mapping_record import MappingRecord from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.projection_group import ProjectionGroup from mavedb.models.score_set import ScoreSet as ScoreSetDbModel from mavedb.models.variant import Variant as VariantDbModel from mavedb.models.vep_allele_consequence import VepAlleleConsequence @@ -96,39 +97,48 @@ def _seed_mapping(session, variant_urn): cousin = Allele( vrs_digest="cousin-digest", level="cdna", post_mapped=_post_mapped(), hgvs_c="NM_000546.6:c.1218C>T" ) - session.add_all([measured, genomic, protein, cousin]) + measured_group = ProjectionGroup( + mapping_record_id=record.id, + score_set_id=record.score_set_id, + hgvs_c=measured.hgvs_c, + hgvs_g=genomic.hgvs_g, + hgvs_p=protein.hgvs_p, + ) + cousin_group = ProjectionGroup(mapping_record_id=record.id, score_set_id=record.score_set_id, hgvs_c=cousin.hgvs_c) + session.add_all([measured, genomic, protein, cousin, measured_group, cousin_group]) session.commit() session.add_all( [ - # The measured cdna link and its genomic projection share a projection_group; the protein - # apex is in no pair (group None); the cousin sits in its own projection group. + # The measured group holds the measured cdna, its genomic projection, and the protein apex; the + # cousin sits in its own projection group. MappingRecordAllele( mapping_record_id=record.id, score_set_id=record.score_set_id, allele_id=measured.id, is_authoritative=True, - projection_group=0, + projection_group=measured_group.id, ), MappingRecordAllele( mapping_record_id=record.id, score_set_id=record.score_set_id, allele_id=genomic.id, is_authoritative=False, - projection_group=0, + projection_group=measured_group.id, ), MappingRecordAllele( mapping_record_id=record.id, score_set_id=record.score_set_id, allele_id=protein.id, is_authoritative=False, + projection_group=measured_group.id, ), MappingRecordAllele( mapping_record_id=record.id, score_set_id=record.score_set_id, allele_id=cousin.id, is_authoritative=False, - projection_group=1, + projection_group=cousin_group.id, ), VepAlleleConsequence( allele_id=measured.id, @@ -161,6 +171,11 @@ def test_get_variant_detail_envelope(client, session, data_provider, data_files, assert body["assayLevelDigest"] == "cdna-digest" assert body["clingenAlleleId"] == "CA123" assert body["mode"] == "projection" + assert {level: slot["hgvs"] for level, slot in body["mapped"].items() if slot is not None} == { + "cdna": "NM_000546.6:c.1216G>A", + "genomic": "NC_000017.11:g.7676154C>T", + "protein": "NP_000537.3:p.Ala406Thr", + } # Spec-pure Cat-VRS carries its own field names (no camelization of the nested GA4GH object). assert body["molecularRepresentation"]["type"] == "CategoricalVariant" # State A: the full-closure Cat-VRS member set agrees with the sidecar (every linked allele is a @@ -186,9 +201,9 @@ def test_get_variant_detail_envelope(client, session, data_provider, data_files, assert body["alleles"]["prot-digest"]["level"] == "protein" assert body["alleles"]["prot-digest"]["hgvs"] == "NP_000537.3:p.Ala406Thr" assert body["alleles"]["prot-digest"]["relation"] == "translation_of" - # The apex is a deterministic projection here but pairs with nothing (projectionOf is null). + # The apex is a deterministic projection that pairs with the measured allele it shares the measured group with. assert body["alleles"]["prot-digest"]["derivation"] == "projection" - assert body["alleles"]["prot-digest"]["projectionOf"] is None + assert body["alleles"]["prot-digest"]["projectionOf"] == "cdna-digest" # The synonymous cousin (different projection group) surfaces as a member wearing co_encodes and is # labelled `convergent` (a distinct change sharing the consequence, not an ambiguous candidate). assert body["alleles"]["cousin-digest"]["relation"] == "co_encodes" diff --git a/tests/routers/test_variant_annotation.py b/tests/routers/test_variant_annotation.py index 77d2b3393..60c9bebad 100644 --- a/tests/routers/test_variant_annotation.py +++ b/tests/routers/test_variant_annotation.py @@ -26,6 +26,7 @@ from sqlalchemy import select +from mavedb.models.mapping_record import MappingRecord as MappingRecordDbModel from mavedb.models.score_set import ScoreSet as ScoreSetDbModel from mavedb.models.variant import Variant as VariantDbModel from tests.helpers.constants import ( @@ -230,6 +231,118 @@ def test_lookup_variants_by_vrs_identifier(client, session, data_provider, data_ assert response_data[0]["vrsId"] == TEST_GA4GH_IDENTIFIER +PRE_MAPPED_IDENTIFIER = "ga4gh:VA.PPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPP" + + +def _set_pre_mapped_digest(session, variant, digest): + record = session.scalar(select(MappingRecordDbModel).where(MappingRecordDbModel.variant_id == variant.id)) + record.vrs_digest = digest + session.commit() + + +def test_lookup_variants_by_pre_mapped_vrs_identifier(client, session, data_provider, data_files, setup_router_db): + """A pre-mapped identifier resolves to its variant, reported with the variant's measured allele.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + variant = session.scalar(select(VariantDbModel).where(VariantDbModel.urn == f"{score_set['urn']}#1")) + seed_mapping_record( + session, + variant, + assay_level="cdna", + alleles=[ + AlleleSpec( + digest=TEST_GA4GH_IDENTIFIER, + level="cdna", + is_authoritative=True, + clingen_allele_id="CA1", + post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X, + ) + ], + ) + _set_pre_mapped_digest(session, variant, PRE_MAPPED_IDENTIFIER) + + response = client.get(f"/api/v1/variants/vrs/{quote_plus(PRE_MAPPED_IDENTIFIER)}") + + assert response.status_code == 200 + assert response.json() == [ + { + "variantUrn": f"{score_set['urn']}#1", + "clingenAlleleId": "CA1", + "vrsId": TEST_GA4GH_IDENTIFIER, + "level": "cdna", + } + ] + + +def test_lookup_variants_by_pre_mapped_vrs_identifier_of_an_unmapped_variant( + client, session, data_provider, data_files, setup_router_db +): + """A variant that did not map still resolves by its pre-mapped identifier, with no allele fields.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + variant = session.scalar(select(VariantDbModel).where(VariantDbModel.urn == f"{score_set['urn']}#1")) + seed_mapping_record(session, variant, assay_level="cdna", alleles=[]) + _set_pre_mapped_digest(session, variant, PRE_MAPPED_IDENTIFIER) + + response = client.get(f"/api/v1/variants/vrs/{quote_plus(PRE_MAPPED_IDENTIFIER)}") + + assert response.status_code == 200 + assert response.json() == [ + {"variantUrn": f"{score_set['urn']}#1", "clingenAlleleId": None, "vrsId": None, "level": None} + ] + + +def test_lookup_variants_by_vrs_identifier_reports_a_variant_once_when_pre_and_post_mapped_agree( + client, session, data_provider, data_files, setup_router_db +): + """An accession-based target's pre- and post-mapped variations can share an identifier.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + variant = session.scalar(select(VariantDbModel).where(VariantDbModel.urn == f"{score_set['urn']}#1")) + seed_mapping_record( + session, + variant, + assay_level="cdna", + alleles=[ + AlleleSpec(digest=TEST_GA4GH_IDENTIFIER, level="cdna", is_authoritative=True, clingen_allele_id="CA1") + ], + ) + _set_pre_mapped_digest(session, variant, TEST_GA4GH_IDENTIFIER) + + response = client.get(f"/api/v1/variants/vrs/{quote_plus(TEST_GA4GH_IDENTIFIER)}") + + assert response.status_code == 200 + assert [match["variantUrn"] for match in response.json()] == [f"{score_set['urn']}#1"] + + +def test_lookup_variants_by_vrs_identifier_orders_matches_by_variant_number( + client, session, data_provider, data_files, setup_router_db +): + """A shared allele resolves to every variant linking it, in variant-number order whatever the seeding order.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + for number in (3, 1, 2): + variant = session.scalar(select(VariantDbModel).where(VariantDbModel.urn == f"{score_set['urn']}#{number}")) + seed_mapping_record( + session, + variant, + assay_level="cdna", + alleles=[AlleleSpec(digest=TEST_GA4GH_IDENTIFIER, level="cdna", is_authoritative=True)], + ) + + response = client.get(f"/api/v1/variants/vrs/{quote_plus(TEST_GA4GH_IDENTIFIER)}") + + assert [match["variantUrn"] for match in response.json()] == [f"{score_set['urn']}#{n}" for n in (1, 2, 3)] + + def test_lookup_variants_by_vrs_identifier_empty_when_none_match( client, session, data_provider, data_files, setup_router_db ): diff --git a/tests/scripts/test_audit_reverse_translation.py b/tests/scripts/test_audit_reverse_translation.py index e4c76b520..82c7f136d 100644 --- a/tests/scripts/test_audit_reverse_translation.py +++ b/tests/scripts/test_audit_reverse_translation.py @@ -28,7 +28,7 @@ round_trip_pair, run_sql_check, ) -from tests.helpers.util.annotation import AlleleSpec, seed_mapping_record +from tests.helpers.util.annotation import AlleleSpec, GroupSpec, seed_mapping_record CHECKS = {check.name: check for check in SQL_CHECKS} @@ -54,7 +54,10 @@ def _score_set_with_variants(session, make_score_set, urn, n): def _healthy_record(session, variant, n): - """A measured cdna allele folded into its projection pair, with the genomic twin and a protein apex.""" + """A measured cdna allele in its measured projection group, with the genomic twin and the protein apex.""" + measured = GroupSpec( + hgvs_c=f"NM_000001.1:c.{n}A>G", hgvs_g=f"NC_000001.11:g.{1000 + n}A>G", hgvs_p=f"NP_000001.1:p.Lys{n}Glu" + ) return seed_mapping_record( session, variant, @@ -65,7 +68,7 @@ def _healthy_record(session, variant, n): is_authoritative=True, hgvs_c=f"NM_000001.1:c.{n}A>G", clingen_allele_id=f"CA{n}", - projection_group=1, + group=measured, post_mapped=_post_mapped("SQ.cdna"), ), AlleleSpec( @@ -73,13 +76,14 @@ def _healthy_record(session, variant, n): level="genomic", hgvs_g=f"NC_000001.11:g.{1000 + n}A>G", clingen_allele_id=f"CA{n}", - projection_group=1, + group=measured, post_mapped=_post_mapped("SQ.genome"), ), AlleleSpec( digest=f"p-{n}", level="protein", hgvs_p=f"NP_000001.1:p.Lys{n}Glu", + group=measured, post_mapped=_post_mapped("SQ.protein"), ), ], @@ -107,13 +111,14 @@ def test_a_healthy_graph_passes_every_error_check(self, session, make_score_set) def test_flags_a_measured_allele_left_outside_its_projection_group(self, session, make_score_set): score_set, (variant,) = _score_set_with_variants(session, make_score_set, "urn:mavedb:00000002-a-1", 1) + candidate = GroupSpec(hgvs_c="NM_000002.2:c.5A>G", hgvs_g="NC_000002.12:g.50A>G") seed_mapping_record( session, variant, alleles=[ AlleleSpec(digest="measured", level="cdna", is_authoritative=True, hgvs_c="NM_000002.1:c.5A>G"), - AlleleSpec(digest="rt-c", level="cdna", hgvs_c="NM_000002.2:c.5A>G", projection_group=1), - AlleleSpec(digest="rt-g", level="genomic", hgvs_g="NC_000002.12:g.50A>G", projection_group=1), + AlleleSpec(digest="rt-c", level="cdna", hgvs_c="NM_000002.2:c.5A>G", group=candidate), + AlleleSpec(digest="rt-g", level="genomic", hgvs_g="NC_000002.12:g.50A>G", group=candidate), ], ) @@ -131,6 +136,7 @@ def test_reports_a_measured_indel_outside_the_groups_as_info_not_error(self, ses "location": {"type": "SequenceLocation", "start": 67611951, "end": 67611954}, "state": {"type": "LiteralSequenceExpression", "sequence": ""}, } + candidate = GroupSpec(hgvs_c="NM_006565.4:c.784_786del", hgvs_g="NC_000016.10:g.67611953_67611955del") seed_mapping_record( session, variant, @@ -142,9 +148,9 @@ def test_reports_a_measured_indel_outside_the_groups_as_info_not_error(self, ses hgvs_g="NC_000016.10:g.67611952_67611954del", post_mapped=deletion, ), - AlleleSpec(digest="rt-c", level="cdna", hgvs_c="NM_006565.4:c.784_786del", projection_group=0), + AlleleSpec(digest="rt-c", level="cdna", hgvs_c="NM_006565.4:c.784_786del", group=candidate), AlleleSpec( - digest="rt-g", level="genomic", hgvs_g="NC_000016.10:g.67611953_67611955del", projection_group=0 + digest="rt-g", level="genomic", hgvs_g="NC_000016.10:g.67611953_67611955del", group=candidate ), ], ) @@ -169,6 +175,7 @@ def located(start, end, sequence): "state": {"type": "LiteralSequenceExpression", "sequence": sequence}, } + candidate = GroupSpec(hgvs_c="NM_152416.4:c.137T>C", hgvs_g="NC_000008.11:g.95025145T>C") seed_mapping_record( session, variant, @@ -180,19 +187,19 @@ def located(start, end, sequence): hgvs_g="NC_000008.11:g.95025143_95025145delinsTGC", post_mapped=located(95025142, 95025145, "TGC"), ), - AlleleSpec(digest="rt-c", level="cdna", hgvs_c="NM_152416.4:c.137T>C", projection_group=0), + AlleleSpec(digest="rt-c", level="cdna", hgvs_c="NM_152416.4:c.137T>C", group=candidate), AlleleSpec( digest="rt-g", level="genomic", hgvs_g="NC_000008.11:g.95025145T>C", - projection_group=0, + group=candidate, post_mapped=located(95025144, 95025145, "C"), ), AlleleSpec( digest="rt-g-2", level="genomic", hgvs_g="NC_000008.11:g.95025145_95025146delinsCA", - projection_group=1, + group=GroupSpec(hgvs_g="NC_000008.11:g.95025145_95025146delinsCA"), post_mapped=located(95025144, 95025146, "CA"), ), ], @@ -203,6 +210,7 @@ def located(start, end, sequence): def test_reports_a_reference_identical_allele_as_info_not_error(self, session, make_score_set): score_set, (variant,) = _score_set_with_variants(session, make_score_set, "urn:mavedb:00000014-a-1", 1) + candidate = GroupSpec(hgvs_c="NM_001130145.3:c.519G>A", hgvs_g="NC_000011.10:g.102114338T>A") seed_mapping_record( session, variant, @@ -210,8 +218,8 @@ def test_reports_a_reference_identical_allele_as_info_not_error(self, session, m AlleleSpec( digest="measured", level="genomic", is_authoritative=True, hgvs_g="NC_000011.10:g.102114338T=" ), - AlleleSpec(digest="rt-c", level="cdna", hgvs_c="NM_001130145.3:c.519G>A", projection_group=0), - AlleleSpec(digest="rt-g", level="genomic", hgvs_g="NC_000011.10:g.102114338T>A", projection_group=0), + AlleleSpec(digest="rt-c", level="cdna", hgvs_c="NM_001130145.3:c.519G>A", group=candidate), + AlleleSpec(digest="rt-g", level="genomic", hgvs_g="NC_000011.10:g.102114338T>A", group=candidate), ], ) @@ -299,19 +307,27 @@ def test_reports_a_twin_on_a_retired_allele_as_info_not_error(self, session, mak assert _count(session, "clingen_allele_id_on_twin_alleles", [score_set.id]).count == 0 assert _count(session, "clingen_allele_id_on_retired_twin", [score_set.id]).count == 1 - def test_flags_a_projection_group_with_one_member(self, session, make_score_set): + def test_flags_a_candidate_projection_group_with_one_member(self, session, make_score_set): score_set, (variant,) = _score_set_with_variants(session, make_score_set, "urn:mavedb:00000005-a-1", 1) + measured = GroupSpec(hgvs_c="NM_000005.1:c.1A>G", hgvs_g="NC_000005.10:g.10A>G") seed_mapping_record( session, variant, alleles=[ AlleleSpec( - digest="lonely", + digest="measured", level="cdna", is_authoritative=True, hgvs_c="NM_000005.1:c.1A>G", - projection_group=1, - ) + group=measured, + ), + AlleleSpec(digest="twin", level="genomic", hgvs_g="NC_000005.10:g.10A>G", group=measured), + AlleleSpec( + digest="lonely", + level="cdna", + hgvs_c="NM_000005.1:c.3A>G", + group=GroupSpec(hgvs_c="NM_000005.1:c.3A>G"), + ), ], ) @@ -320,6 +336,26 @@ def test_flags_a_projection_group_with_one_member(self, session, make_score_set) assert finding.count == 1 assert finding.samples[0]["levels"] == "cdna" + def test_does_not_flag_a_measured_group_with_one_member(self, session, make_score_set): + """An intronic genomic change has no cdna allele, so its measured group holds the genomic allele alone.""" + score_set, (variant,) = _score_set_with_variants(session, make_score_set, "urn:mavedb:00000015-a-1", 1) + seed_mapping_record( + session, + variant, + assay_level="genomic", + alleles=[ + AlleleSpec( + digest="intronic", + level="genomic", + is_authoritative=True, + hgvs_g="NC_000005.10:g.10A>G", + group=GroupSpec(hgvs_c="NM_000465.4:c.123+5A>G", hgvs_g="NC_000005.10:g.10A>G"), + ) + ], + ) + + assert _count(session, "malformed_projection_group", [score_set.id]).count == 0 + def test_flags_a_level_that_disagrees_with_the_hgvs(self, session, make_score_set): score_set, (variant,) = _score_set_with_variants(session, make_score_set, "urn:mavedb:00000006-a-1", 1) seed_mapping_record( @@ -378,13 +414,13 @@ def test_scoping_excludes_other_score_sets(self, session, make_score_set): session, broken_variant, alleles=[ + AlleleSpec(digest="b-measured", level="cdna", is_authoritative=True, hgvs_c="NM_000011.1:c.1A>G"), AlleleSpec( digest="b-lonely", level="cdna", - is_authoritative=True, - hgvs_c="NM_000011.1:c.1A>G", - projection_group=1, - ) + hgvs_c="NM_000011.1:c.3A>G", + group=GroupSpec(hgvs_c="NM_000011.1:c.3A>G"), + ), ], ) diff --git a/tests/scripts/test_backfill_campaign.py b/tests/scripts/test_backfill_campaign.py index 0b28d6e23..85bdcb851 100644 --- a/tests/scripts/test_backfill_campaign.py +++ b/tests/scripts/test_backfill_campaign.py @@ -23,7 +23,7 @@ scan_legacy_readers, ) from tests.helpers.constants import TEST_MINIMAL_MAPPED_VARIANT -from tests.helpers.util.annotation import AlleleSpec, seed_mapping_record +from tests.helpers.util.annotation import AlleleSpec, GroupSpec, seed_mapping_record # Pure: state classification @@ -240,7 +240,7 @@ def test_rt_and_enriched_score_set(self, session, make_score_set): digest="sib", level="cdna", is_authoritative=False, - projection_group=1, + group=GroupSpec(), vep_consequence="missense_variant", clinvar_control_ids=(control.id,), gnomad_variant_ids=(gnomad.id,), diff --git a/tests/worker/jobs/variant_processing/test_reverse_translation.py b/tests/worker/jobs/variant_processing/test_reverse_translation.py index 22f904f63..faae726a8 100644 --- a/tests/worker/jobs/variant_processing/test_reverse_translation.py +++ b/tests/worker/jobs/variant_processing/test_reverse_translation.py @@ -25,15 +25,19 @@ from mavedb.models.job_run import JobRun from mavedb.models.mapping_record import MappingRecord from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.projection_group import ProjectionGroup +from mavedb.models.target_accession import TargetAccession from mavedb.models.target_gene_mapping import TargetGeneMapping from mavedb.models.variant import Variant from mavedb.models.annotation_event import AnnotationEvent from mavedb.worker.jobs.variant_processing.mapping import map_variants_for_score_set from mavedb.worker.jobs.variant_processing.reverse_translation import ( _build_translation_config, + cdna_transcript_resolver, reverse_translate_variants_for_score_set, ) from mavedb.worker.lib.managers.job_manager import JobManager +from mavedb.worker.lib.projection import project_measured_variants from mavedb.lib.vrs_utils import canonical_variation_document from tests.helpers.constants import TEST_VALID_POST_MAPPED_VRS_ALLELE from tests.helpers.util.setup.worker import construct_mock_mapping_output @@ -156,6 +160,44 @@ def _translate(hgvs: str, translator=None) -> FakeVrsVariation: return _translate +class FakeCoordinates: + """Deterministic stand-in for ``WorkerCoordinateTranslator``, used by the measured projection step. + + The worker context's ``hdp`` is a bare mock, so the real translator would produce projection errors + rather than strings. Tables pin a projection per input HGVS; anything else projects to a fixed + placeholder so a test that doesn't care still gets a well-formed measured group. + """ + + def __init__( + self, + c_by_g: dict | None = None, + g_by_c: dict | None = None, + p_by_c: dict | None = None, + mane: str | None = None, + ): + self.c_by_g = c_by_g or {} + self.g_by_c = g_by_c or {} + self.p_by_c = p_by_c or {} + self.mane = mane + self.mane_lookups: list[str] = [] + + def g_to_c(self, g_hgvs: str, transcript: str) -> str: + return self.c_by_g.get(g_hgvs, f"{transcript}:c.1A>G") + + def normalize(self, hgvs_string: str) -> str: + return hgvs_string + + def c_to_g(self, c_hgvs: str) -> str: + return self.g_by_c.get(c_hgvs, "NC_000001.11:g.1000A>G") + + def c_to_p(self, c_hgvs: str) -> str: + return self.p_by_c.get(c_hgvs, "NP_999999.1:p.(Met1Val)") + + def mane_select_transcript(self, g_hgvs: str) -> str | None: + self.mane_lookups.append(g_hgvs) + return self.mane + + # Mapping recomputes an incoming allele's identity from its own content rather than trusting the # producer's `id`, so a stored authoritative allele carries this rather than the fixture's placeholder. # The fold-in tests below depend on the reverse-translated coding allele identifying to the same value. @@ -164,6 +206,13 @@ def _translate(hgvs: str, translator=None) -> FakeVrsVariation: )[1] +# Candidates for the rerun tests: the measured coding allele with its genomic partner, and an unrelated pair. +MEASURED_C = "NM_000001.1:c.5A>G" +MEASURED_PARTNER_G = "NC_000001.11:g.1000A>G" +OTHER_C = "NM_000001.1:c.9C>T" +OTHER_G = "NC_000001.11:g.1004C>T" + + async def _map_variants(session, mock_worker_ctx, mapping_run, score_set, with_layers=frozenset({"g", "c", "p"})): """Run the (mocked) mapping job so the score set's variants gain authoritative MappingRecords/Alleles — the real inputs the reverse translation job reads.""" @@ -182,11 +231,16 @@ async def dummy_mapping_job(): session.commit() -async def _reverse_translate(session, mock_worker_ctx, rt_run): +async def _reverse_translate(session, mock_worker_ctx, rt_run, coordinates=None): # The job opens a UTA-backed TranscriptSource to back codon_at (WtCodonMode.ALL). # construct_equivalent_variants is mocked in these tests, so the source is never - # queried -- stub the factory with a no-op context yielding a dummy client. - with patch(f"{RT_MODULE}.uta_transcript_source", lambda: contextlib.nullcontext(MagicMock())): + # queried -- stub the factory with a no-op context yielding a dummy client. The measured + # projection step reads the coordinate translator, so it gets a deterministic fake. + coordinates = coordinates or FakeCoordinates() + with ( + patch(f"{RT_MODULE}.uta_transcript_source", lambda: contextlib.nullcontext(MagicMock())), + patch(f"{RT_MODULE}.WorkerCoordinateTranslator", lambda hdp: coordinates), + ): return await reverse_translate_variants_for_score_set( mock_worker_ctx, rt_run.id, @@ -242,6 +296,36 @@ def _record_for(session, variant_id): return session.query(MappingRecord).filter(MappingRecord.variant_id == variant_id).one() +def _group(session, group_id): + return session.get(ProjectionGroup, group_id) + + +def _measured_group(session, record_id): + """The projection group the record's live authoritative link points at.""" + return _group(session, _authoritative_link(session, record_id).projection_group) + + +def _live_groups(session, record_id): + return ( + session.query(ProjectionGroup) + .filter(ProjectionGroup.mapping_record_id == record_id, ProjectionGroup.valid_to.is_(None)) + .all() + ) + + +def _authoritative_versions(session, record_id): + """Every version of the record's authoritative link, oldest first.""" + return ( + session.query(MappingRecordAllele) + .filter( + MappingRecordAllele.mapping_record_id == record_id, + MappingRecordAllele.is_authoritative.is_(True), + ) + .order_by(MappingRecordAllele.id) + .all() + ) + + @pytest.mark.unit class TestBuildTranslationConfig: """Unit tests for _build_translation_config — the optional translation_config job param.""" @@ -466,6 +550,267 @@ async def test_falls_back_to_mapped_date_when_the_record_has_no_job_run_id( assert result.status == JobStatus.SUCCEEDED assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 2, "fold_in_missed": 1} + @pytest.mark.parametrize("record_has_job_run_id", [False, True]) + async def test_legacy_record_falls_back_to_the_target_post_mapped_cdna_accession( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + record_has_job_run_id, + ): + """Legacy mappings wrote no cdna TargetGeneMapping; their coding accession lives in the target's + post_mapped_metadata. A record with no job_run_id falls back to it, but a record anchored to a run + resolves only through that run's own cdna row, so the legacy accession never leaks into it.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn=f"{sample_score_set.urn}#11", + hgvs_nt="NC_000001.11:g.1000A>G", + data={}, + ) + session.add(variant) + session.commit() + # Genomic only: no cdna TargetGeneMapping for either key to find. + await _map_variants( + session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set, with_layers={"g"} + ) + target_gene = sample_score_set.target_genes[0] + target_gene.post_mapped_metadata = {"cdna": {"sequence_accessions": ["NM_777777.1", "NM_777778.1"]}} + if not record_has_job_run_id: + session.query(TargetGeneMapping).filter(TargetGeneMapping.target_gene_id == target_gene.id).update( + {TargetGeneMapping.job_run_id: None}, synchronize_session=False + ) + session.commit() + + record = _record_for(session, variant.id) + tgm = session.get(TargetGeneMapping, record.target_gene_mapping_id) + resolve = cdna_transcript_resolver(session, sample_score_set.id, FakeCoordinates()) + expected = None if record_has_job_run_id else "NM_777777.1" + assert resolve(tgm.job_run_id, target_gene.id, tgm.mapped_date, "NC_000001.11:g.1000A>G") == expected + + assay_hgvs = "NC_000001.11:g.1000A>G" + delegate = fake_construct({assay_hgvs: [("NM_777777.1:c.5A>G", None)]}) + captured: dict = {} + + def capturing_construct(inputs, *, transcripts, coordinates, config): + captured["transcripts"] = [inp.transcript for inp in inputs] + return delegate(inputs, transcripts=transcripts, coordinates=coordinates, config=config) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", capturing_construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", fake_translate({})), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + if record_has_job_run_id: + assert captured["transcripts"] == [] + assert result.data["skipped"] == 1 + assert len(_cross_level_events(session, sample_score_set.id, reason="transcript_unresolved")) == 1 + else: + assert captured["transcripts"] == ["NM_777777.1"] + assert result.data["translated"] == 1 + + @pytest.mark.parametrize("record_has_job_run_id", [True, False]) + async def test_genomic_only_target_falls_back_to_its_transcript_accession_then_mane_select( + self, + session, + with_independent_processing_runs, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_score_set, + record_has_job_run_id, + ): + """A genomic-only target has no cdna row and no legacy cdna accession. A transcript target + accession is used as is; otherwise a protein-coding target takes the MANE Select transcript at the + record's locus, looked up once per target gene.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn=f"{sample_score_set.urn}#11", + hgvs_nt="NC_000001.11:g.1000A>G", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants( + session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set, with_layers={"g"} + ) + target_gene = sample_score_set.target_genes[0] + target_gene.category = TargetCategory.protein_coding + target_gene.post_mapped_metadata = {"genomic": {"sequence_accessions": ["NC_000001.11"]}} + if not record_has_job_run_id: + session.query(TargetGeneMapping).filter(TargetGeneMapping.target_gene_id == target_gene.id).update( + {TargetGeneMapping.job_run_id: None}, synchronize_session=False + ) + session.commit() + record = _record_for(session, variant.id) + tgm = session.get(TargetGeneMapping, record.target_gene_mapping_id) + key = (tgm.job_run_id, target_gene.id, tgm.mapped_date) + + coordinates = FakeCoordinates(mane="NM_888888.2") + resolve = cdna_transcript_resolver(session, sample_score_set.id, coordinates) + assert resolve(*key, "NC_000001.11:g.1000A>G") == "NM_888888.2" + assert resolve(*key, "NC_000001.11:g.1001A>G") == "NM_888888.2" + assert coordinates.mane_lookups == ["NC_000001.11:g.1000A>G"] + + # Protein records pass no genomic HGVS and keep their NP_→NM_ lookup. + assert resolve(*key, None) is None + + target_gene.target_accession = TargetAccession(accession="NM_999999.3", assembly="GRCh38", gene="GENE") + session.commit() + coordinates = FakeCoordinates(mane="NM_888888.2") + resolve = cdna_transcript_resolver(session, sample_score_set.id, coordinates) + assert resolve(*key, "NC_000001.11:g.1000A>G") == "NM_999999.3" + assert coordinates.mane_lookups == [] + + async def test_mane_select_is_only_looked_up_for_protein_coding_targets_and_a_bounded_number_of_times( + self, + session, + with_independent_processing_runs, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_score_set, + ): + variant = Variant( + score_set_id=sample_score_set.id, + urn=f"{sample_score_set.urn}#11", + hgvs_nt="NC_000001.11:g.1000A>G", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants( + session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set, with_layers={"g"} + ) + target_gene = sample_score_set.target_genes[0] + target_gene.post_mapped_metadata = {"genomic": {"sequence_accessions": ["NC_000001.11"]}} + target_gene.category = TargetCategory.regulatory + session.commit() + record = _record_for(session, variant.id) + tgm = session.get(TargetGeneMapping, record.target_gene_mapping_id) + key = (tgm.job_run_id, target_gene.id, tgm.mapped_date) + + coordinates = FakeCoordinates(mane="NM_888888.2") + resolve = cdna_transcript_resolver(session, sample_score_set.id, coordinates) + assert resolve(*key, "NC_000001.11:g.1000A>G") is None + assert coordinates.mane_lookups == [] + + target_gene.category = TargetCategory.protein_coding + session.commit() + coordinates = FakeCoordinates(mane=None) + resolve = cdna_transcript_resolver(session, sample_score_set.id, coordinates) + for position in range(1000, 1010): + assert resolve(*key, f"NC_000001.11:g.{position}A>G") is None + assert len(coordinates.mane_lookups) == 5 + + @pytest.mark.parametrize("failure", ["projection", "transcript"]) + async def test_a_failed_projection_keeps_the_group_an_earlier_run_wrote( + self, + session, + with_independent_processing_runs, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_score_set, + failure, + ): + """A data provider outage must not erase coding and protein projections: the record keeps its group.""" + assay_hgvs = "NC_000001.11:g.1000A>G" + variant = Variant( + score_set_id=sample_score_set.id, urn=f"{sample_score_set.urn}#11", hgvs_nt=assay_hgvs, data={} + ) + session.add(variant) + session.commit() + await _map_variants( + session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set, with_layers={"g"} + ) + record = _record_for(session, variant.id) + + def resolve(*_): + return "NM_000001.1" + + project_measured_variants( + session, + sample_score_set.id, + coordinates=FakeCoordinates(c_by_g={assay_hgvs: MEASURED_C}), + resolve_transcript=resolve, + ) + session.commit() + written = _measured_group(session, record.id) + assert written.hgvs_c == MEASURED_C + + class Unreachable(FakeCoordinates): + def g_to_c(self, g_hgvs, transcript): + raise ConnectionError("cdot-rest unreachable") + + def unreachable_resolve(*_): + raise ConnectionError("cdot-rest unreachable") + + counts = project_measured_variants( + session, + sample_score_set.id, + coordinates=Unreachable() if failure == "projection" else FakeCoordinates(), + resolve_transcript=resolve if failure == "projection" else unreachable_resolve, + ) + session.commit() + + assert counts["kept_on_error"] == 1 + assert _measured_group(session, record.id).id == written.id + assert [group.id for group in _live_groups(session, record.id)] == [written.id] + + async def test_untranslatable_delins_still_gets_a_measured_projection_group( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """A multi-codon delins has no codon-level equivalence class, so the library returns NOT_TRANSLATABLE + and RT links nothing. The measured group comes from the projection step, not from RT, so the record + still describes its variant at the coding and protein levels.""" + assay_hgvs = "NC_000001.11:g.1000_1005delinsTTT" + variant = Variant( + score_set_id=sample_score_set.id, urn=f"{sample_score_set.urn}#11", hgvs_nt=assay_hgvs, data={} + ) + session.add(variant) + session.commit() + await _map_variants( + session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set, with_layers={"g", "c"} + ) + + hgvs_c = "NM_999999.1:c.4_9delinsTTT" + coordinates = FakeCoordinates( + c_by_g={assay_hgvs: hgvs_c}, p_by_c={hgvs_c: "NP_999999.1:p.(Lys2_Ala3delinsPhe)"} + ) + construct = fake_construct({}, not_translatable_by_hgvs={assay_hgvs: "multi-codon delins"}) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", fake_translate({})), + ): + result = await _reverse_translate( + session, mock_worker_ctx, sample_independent_reverse_translation_run, coordinates=coordinates + ) + + assert result.status == JobStatus.SUCCEEDED + assert result.data["skipped"] == 1 + assert _non_authoritative_links(session) == [] + + record = _record_for(session, variant.id) + measured = _measured_group(session, record.id) + assert measured.current + assert (measured.hgvs_c, measured.hgvs_g, measured.hgvs_p) == ( + hgvs_c, + assay_hgvs, + "NP_999999.1:p.Lys2_Ala3delinsPhe", + ) + assert [group.id for group in _live_groups(session, record.id)] == [measured.id] + async def test_creates_genomic_and_coding_candidate_alleles( self, session, @@ -526,16 +871,20 @@ async def test_creates_genomic_and_coding_candidate_alleles( mapping_record = _record_for(session, variant.id) assert {link.mapping_record_id for link in non_auth_links} == {mapping_record.id} - # Both members of the single projection pair carry the same (first, index 0) group id — the - # coding↔genomic pairing survives the flatten that Phase B exists to stop. - assert {link.projection_group for link in non_auth_links} == {0} + # Both members of the single projection pair share one candidate group, distinct from the measured + # group, whose strings are the pair's own. + (group_id,) = {link.projection_group for link in non_auth_links} + assert group_id != _authoritative_link(session, mapping_record.id).projection_group + group = _group(session, group_id) + assert group.current + assert (group.hgvs_c, group.hgvs_g, group.hgvs_p) == (c_candidate, g_candidate, None) events = _cross_level_events(session, sample_score_set.id) assert len(events) == 1 assert events[0].disposition == "present" # The event metadata carries the paired projection pair shape, not the removed flat lists. assert events[0].event_metadata["candidates"] == [ - {"hgvs_c": c_candidate, "hgvs_g": g_candidate, "variant_type": None, "projection_group": 0} + {"hgvs_c": c_candidate, "hgvs_g": g_candidate, "variant_type": None, "measured": False} ] async def test_transcript_is_queryable_via_derived_expression( @@ -591,8 +940,8 @@ async def test_protein_consequence_is_persisted_as_a_protein_allele( ): """The deterministic protein consequence (``result.hgvs_p``) is emitted as the protein-level member of the equivalence set -- a non-authoritative ``level=protein`` - allele linked to the record, not dropped. As the apex shared across all projection pairs it - belongs to no projection pair, so its link's projection_group is NULL.""" + allele linked to the record, not dropped. It is the measured variant's own consequence, so its + link joins the record's measured group; each candidate group records it as its hgvs_p.""" variant = Variant( score_set_id=sample_score_set.id, urn=f"{sample_score_set.urn}#11", @@ -629,13 +978,17 @@ async def test_protein_consequence_is_persisted_as_a_protein_allele( assert protein_allele.level == SequenceLevel.protein.value assert protein_allele.hgvs_p == p_stripped # stored without the prediction parens - # Linked to the record as a non-authoritative member of the equivalence set, but grouped - # with no pair (the apex is shared across every projection pair). + # Linked to the record as a non-authoritative member of the measured group. + record = _record_for(session, variant.id) + measured_group_id = _authoritative_link(session, record.id).projection_group protein_link = next(link for link in _non_authoritative_links(session) if link.allele_id == protein_allele.id) - assert protein_link.projection_group is None - # The coding/genomic pair is still grouped together (group 0). + assert protein_link.projection_group == measured_group_id + # The coding/genomic pair shares one candidate group, which carries the stripped consequence. pair_links = [link for link in _non_authoritative_links(session) if link.allele_id != protein_allele.id] - assert {link.projection_group for link in pair_links} == {0} + (pair_group_id,) = {link.projection_group for link in pair_links} + assert pair_group_id != measured_group_id + pair_group = _group(session, pair_group_id) + assert (pair_group.hgvs_c, pair_group.hgvs_g, pair_group.hgvs_p) == (c_candidate, g_candidate, p_stripped) async def test_cis_phased_multivariant_candidate_is_persisted_as_a_block( self, @@ -687,7 +1040,9 @@ async def test_cis_phased_multivariant_candidate_is_persisted_as_a_block( assert block_allele.post_mapped == {"type": "CisPhasedBlock", "id": "ga4gh:CPB.block"} # The block (genomic projection) and its coding partner are one projection pair -> one group. - assert {link.projection_group for link in _non_authoritative_links(session)} == {0} + (group_id,) = {link.projection_group for link in _non_authoritative_links(session)} + group = _group(session, group_id) + assert (group.hgvs_c, group.hgvs_g) == (c_candidate, block_candidate) async def test_duplicate_candidate_digests_are_deduped_per_record( self, @@ -730,7 +1085,8 @@ async def test_duplicate_candidate_digests_are_deduped_per_record( assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 1, "fold_in_missed": 1} non_auth_links = _non_authoritative_links(session) assert len(non_auth_links) == 1 - assert non_auth_links[0].projection_group == 0 + group = _group(session, non_auth_links[0].projection_group) + assert (group.hgvs_c, group.hgvs_g) == (c_candidate, g_candidate) assert session.query(Allele).filter(Allele.vrs_digest == "ga4gh:VA.shared").count() == 1 async def test_authoritative_allele_is_folded_into_its_projection_group( @@ -744,10 +1100,10 @@ async def test_authoritative_allele_is_folded_into_its_projection_group( sample_score_set, ): """Nucleotide assay: the measured allele appears among the RT candidates (the library makes - no exclusions) and already has a live authoritative link (group NULL). Rather than insert a - forbidden derived duplicate, the job folds the pair's group onto the existing - authoritative link. That group then resolves to {canonical coding, canonical genomic}: the - authoritative allele plus its sibling projection derived in the same projection pair.""" + no exclusions). That pair is the measured variant itself, so rather than forming a candidate + group its other member joins the record's measured group, which then resolves to {canonical + coding, canonical genomic}. The authoritative link already points at that group and is not + re-versioned by RT.""" variant = Variant( score_set_id=sample_score_set.id, urn=f"{sample_score_set.urn}#11", @@ -786,19 +1142,32 @@ async def test_authoritative_allele_is_folded_into_its_projection_group( derived_links = _non_authoritative_links(session) assert len(derived_links) == 1 - # The authoritative link was folded into the pair's group, and the sibling derived - # link shares it -- the pairing is resolvable from the authoritative allele's group. - assert auth_link.projection_group == 0 - assert derived_links[0].projection_group == 0 + # The sibling derived link joins the measured group -- the pairing is resolvable from the + # authoritative allele's group -- and no candidate group was created for the measured pair. + measured_group_id = auth_link.projection_group + assert measured_group_id is not None + assert derived_links[0].projection_group == measured_group_id + assert [group.id for group in _live_groups(session, record.id)] == [measured_group_id] - # group 0 resolves to {canonical c (authoritative), canonical g (projection)}: two live + # The measured group resolves to {canonical c (authoritative), canonical g (projection)}: two live # links, one authoritative and one derived. - group_links = [link for link in _live_links(session, record.id) if link.projection_group == 0] + group_links = [link for link in _live_links(session, record.id) if link.projection_group == measured_group_id] assert len(group_links) == 2 assert {link.is_authoritative for link in group_links} == {True, False} assert auth_link.allele.vrs_digest == CANONICAL_AUTHORITATIVE_DIGEST assert derived_links[0].allele.vrs_digest == "ga4gh:VA.genomic" - assert _cross_level_events(session, sample_score_set.id)[0].event_metadata["fold_in_missed"] is False + + # Only the mapping job's link and the projection step's copy exist: RT did not regroup it. + assert [link.projection_group for link in _authoritative_versions(session, record.id)] == [ + None, + measured_group_id, + ] + + metadata = _cross_level_events(session, sample_score_set.id)[0].event_metadata + assert metadata["fold_in_missed"] is False + assert metadata["candidates"] == [ + {"hgvs_c": c_candidate, "hgvs_g": g_sibling, "variant_type": None, "measured": True} + ] async def test_authoritative_fold_in_when_measured_at_genomic_level( self, @@ -811,8 +1180,8 @@ async def test_authoritative_fold_in_when_measured_at_genomic_level( sample_score_set, ): """The fold-in is symmetric in the measured level: when the genomic member of a projection pair - equals the record's authoritative allele, its group folds onto the authoritative link and - the coding sibling becomes the derived member of the same group.""" + equals the record's authoritative allele, the coding sibling joins the measured group and no + candidate group is created for the pair.""" variant = Variant( score_set_id=sample_score_set.id, urn=f"{sample_score_set.urn}#11", @@ -846,8 +1215,10 @@ async def test_authoritative_fold_in_when_measured_at_genomic_level( record = _record_for(session, variant.id) auth_link = _authoritative_link(session, record.id) derived_links = _non_authoritative_links(session) - assert auth_link.projection_group == 0 - assert [link.projection_group for link in derived_links] == [0] + measured_group_id = auth_link.projection_group + assert measured_group_id is not None + assert [link.projection_group for link in derived_links] == [measured_group_id] + assert [group.id for group in _live_groups(session, record.id)] == [measured_group_id] assert auth_link.allele.vrs_digest == CANONICAL_AUTHORITATIVE_DIGEST assert derived_links[0].allele.vrs_digest == "ga4gh:VA.coding" @@ -862,7 +1233,8 @@ async def test_measured_allele_matching_no_candidate_is_flagged_as_a_missed_fold sample_score_set, ): """Nucleotide assay whose measured allele has a different digest from every RT candidate: the - authoritative link stays ungrouped, and the run counts the miss and flags it on the variant's event.""" + measured group holds only the authoritative link, both candidates form a candidate group, and the run + counts the miss and flags it on the variant's event.""" variant = Variant( score_set_id=sample_score_set.id, urn=f"{sample_score_set.urn}#11", @@ -889,7 +1261,15 @@ async def test_measured_allele_matching_no_candidate_is_flagged_as_a_missed_fold assert result.status == JobStatus.SUCCEEDED assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 2, "fold_in_missed": 1} - assert _authoritative_link(session, _record_for(session, variant.id).id).projection_group is None + record = _record_for(session, variant.id) + measured_group_id = _authoritative_link(session, record.id).projection_group + assert [ + link.is_authoritative + for link in _live_links(session, record.id) + if link.projection_group == measured_group_id + ] == [True] + (candidate_group_id,) = {link.projection_group for link in _non_authoritative_links(session)} + assert candidate_group_id != measured_group_id assert _cross_level_events(session, sample_score_set.id)[0].event_metadata["fold_in_missed"] is True async def test_projection_failed_candidate_is_a_one_member_group( @@ -939,17 +1319,18 @@ async def test_projection_failed_candidate_is_a_one_member_group( assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 3, "fold_in_missed": 1} non_auth_links = _non_authoritative_links(session) - groups: dict[int, set[str]] = {} + groups: dict[tuple[str, str | None], set[str]] = {} for link in non_auth_links: - groups.setdefault(link.projection_group, set()).add(link.allele.vrs_digest) + group = _group(session, link.projection_group) + groups.setdefault((group.hgvs_c, group.hgvs_g), set()).add(link.allele.vrs_digest) - # Group 0 is the full projection pair; group 1 is the projection-failed coding-only member. + # One group is the full projection pair; the other is the projection-failed coding-only member. assert groups == { - 0: {"ga4gh:VA.coding0", "ga4gh:VA.genomic0"}, - 1: {"ga4gh:VA.coding1"}, + (c_paired, g_paired): {"ga4gh:VA.coding0", "ga4gh:VA.genomic0"}, + (c_orphan, None): {"ga4gh:VA.coding1"}, } - async def test_protein_assay_leaves_authoritative_group_null_over_a_fan_out( + async def test_protein_assay_measured_group_holds_only_the_protein_over_a_fan_out( self, session, with_independent_processing_runs, @@ -960,8 +1341,8 @@ async def test_protein_assay_leaves_authoritative_group_null_over_a_fan_out( sample_score_set, ): """Protein assay: the authoritative allele is the protein itself, which appears in no - coding/genomic projection pair, so nothing folds in and its link's group stays NULL. The - ambiguous fan-out becomes N distinct groups, each a coding/genomic pair.""" + coding/genomic projection pair, so nothing folds in and its measured group describes only + the protein. The ambiguous fan-out becomes N distinct candidate groups, each a coding/genomic pair.""" variant = Variant( score_set_id=sample_score_set.id, urn=f"{sample_score_set.urn}#11", @@ -999,18 +1380,24 @@ async def test_protein_assay_leaves_authoritative_group_null_over_a_fan_out( assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 6, "fold_in_missed": 0} record = _record_for(session, variant.id) - # The protein authoritative allele is not part of any pair: its group stays NULL. + # The protein authoritative allele is not part of any pair: its measured group holds only hgvs_p and + # no derived link joins it. auth_link = _authoritative_link(session, record.id) - assert auth_link.projection_group is None assert auth_link.allele.vrs_digest == CANONICAL_AUTHORITATIVE_DIGEST + measured = _group(session, auth_link.projection_group) + assert (measured.hgvs_c, measured.hgvs_g, measured.hgvs_p) == (None, None, assay_hgvs) - # Three distinct groups, each a coding/genomic pair (two members). - groups: dict[int, int] = {} + # Three distinct candidate groups, each a coding/genomic pair (two members), with no protein + # consequence re-emitted. + groups: dict[tuple[str, str, str | None], int] = {} for link in _non_authoritative_links(session): - groups[link.projection_group] = groups.get(link.projection_group, 0) + 1 - assert groups == {0: 2, 1: 2, 2: 2} + assert link.projection_group != measured.id + group = _group(session, link.projection_group) + key = (group.hgvs_c, group.hgvs_g, group.hgvs_p) + groups[key] = groups.get(key, 0) + 1 + assert groups == {(hgvs_c, hgvs_g, None): 2 for hgvs_c, hgvs_g in pairs} - async def test_independent_rerun_retires_prior_links_and_regroups( + async def test_independent_rerun_supersedes_prior_links_and_candidate_groups( self, session, with_independent_processing_runs, @@ -1020,12 +1407,11 @@ async def test_independent_rerun_retires_prior_links_and_regroups( sample_independent_reverse_translation_run, sample_score_set, ): - """An independent second run is idempotent: it retires the prior derived links (closing - valid_to) and writes fresh live ones (re-grouped from scratch), so the set of *current* - non-authoritative links is stable while the superseded links are retained as history -- - the partial unique index on live links is never violated and no alleles are duplicated. - projection_group is regenerated per run; it is a within-record grouping key, not a stable - identity across re-maps.""" + """An independent second run is idempotent: it retires the prior derived links and candidate + groups (closing valid_to) and writes fresh live ones, so the set of *current* non-authoritative + links is stable while the superseded rows are retained as history -- the partial unique index on + live links is never violated and no alleles are duplicated. The measured group, whose strings are + unchanged, is left alone.""" variant = Variant( score_set_id=sample_score_set.id, urn=f"{sample_score_set.urn}#11", @@ -1043,7 +1429,7 @@ async def test_independent_rerun_retires_prior_links_and_regroups( construct = fake_construct({assay_hgvs: [(c_candidate, g_candidate)]}) translate = fake_translate({g_candidate: "ga4gh:VA.genomic", c_candidate: "ga4gh:VA.coding"}) - # First run: two live derived links, both in group 0. + # First run: two live derived links sharing one candidate group. with ( patch(f"{RT_MODULE}.construct_equivalent_variants", construct), patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), @@ -1054,7 +1440,9 @@ async def test_independent_rerun_retires_prior_links_and_regroups( first_links = _non_authoritative_links(session) assert len(first_links) == 2 assert all(link.valid_to is None for link in first_links) - assert {link.projection_group for link in first_links} == {0} + (first_group_id,) = {link.projection_group for link in first_links} + record = _record_for(session, variant.id) + measured_group_id = _authoritative_link(session, record.id).projection_group # Second, independent run with identical inputs. rerun = JobRun( @@ -1080,11 +1468,20 @@ async def test_independent_rerun_retires_prior_links_and_regroups( live = [link for link in links if link.valid_to is None] retired = [link for link in links if link.valid_to is not None] - # The live set is stable (still exactly the two candidates, re-grouped into group 0) and the - # prior run's links are retained as closed history rather than deleted. + # The live set is stable (still exactly the two candidates, sharing a fresh candidate group) and + # the prior run's links and group are retained as closed history rather than deleted. assert len(live) == 2 assert len(retired) == 2 - assert {link.projection_group for link in live} == {0} + (second_group_id,) = {link.projection_group for link in live} + assert second_group_id != first_group_id + first_group, second_group = _group(session, first_group_id), _group(session, second_group_id) + assert first_group.valid_to == second_group.valid_from + assert (second_group.hgvs_c, second_group.hgvs_g) == (first_group.hgvs_c, first_group.hgvs_g) + + # The measured group and the authoritative link pointing at it are untouched by the rerun. + assert _authoritative_link(session, record.id).projection_group == measured_group_id + assert _group(session, measured_group_id).current + assert len(_authoritative_versions(session, record.id)) == 2 # Gap-free handoff: the prior links closed at exactly the instant the new links opened, under # one supersession timestamp -- so a point-in-time query never lands in a hole between runs, @@ -1102,25 +1499,51 @@ async def test_independent_rerun_retires_prior_links_and_regroups( assert max(events, key=lambda e: e.id).disposition == "present" @pytest.mark.parametrize( - "rerun_pairs, rerun_unavailable, expected_group, regrouped, derived_groups", + "rerun_pairs, rerun_unavailable, expected_live, candidate_group_kept", [ - # The measured allele's pair moves to a new position: the ids are swapped so its link keeps group 0. + # The measured pair moves position: its partner still joins the measured group. + ( + [(OTHER_C, OTHER_G), (MEASURED_C, MEASURED_PARTNER_G)], + False, + { + ("ga4gh:VA.partner", "measured"), + ("ga4gh:VA.other_c", "candidate"), + ("ga4gh:VA.other_g", "candidate"), + }, + False, + ), + # The measured allele no longer appears among the candidates: its partner leaves the measured group. + ( + [(OTHER_C, OTHER_G)], + False, + {("ga4gh:VA.other_c", "candidate"), ("ga4gh:VA.other_g", "candidate")}, + False, + ), + # Identical candidates: the candidate group is still re-minted (TODO#765), the measured group is not. ( - [("NM_000001.1:c.9C>T", "NC_000001.11:g.1004C>T"), ("NM_000001.1:c.5A>G", None)], + [(MEASURED_C, MEASURED_PARTNER_G), (OTHER_C, OTHER_G)], False, - 0, + { + ("ga4gh:VA.partner", "measured"), + ("ga4gh:VA.other_c", "candidate"), + ("ga4gh:VA.other_g", "candidate"), + }, False, - {1}, ), - # The measured allele no longer folds in: its prior group would name an unrelated pair. - ([("NM_000001.1:c.9C>T", "NC_000001.11:g.1004C>T")], False, None, True, {0}), - # Same fold-in as before: the link is left alone rather than re-versioned. - ([("NM_000001.1:c.5A>G", None)], False, 0, False, set()), - # UTA unavailable: the record's prior links, its group included, are still this run's answer. - ([], True, 0, False, set()), + # UTA unavailable: the record's prior links and candidate group are still this run's answer. + ( + [], + True, + { + ("ga4gh:VA.partner", "measured"), + ("ga4gh:VA.other_c", "candidate"), + ("ga4gh:VA.other_g", "candidate"), + }, + True, + ), ], ) - async def test_rerun_supersedes_the_authoritative_link_when_its_group_changes( + async def test_rerun_never_regroups_the_measured_allele( self, session, with_independent_processing_runs, @@ -1131,14 +1554,13 @@ async def test_rerun_supersedes_the_authoritative_link_when_its_group_changes( sample_score_set, rerun_pairs, rerun_unavailable, - expected_group, - regrouped, - derived_groups, + expected_live, + candidate_group_kept, ): - """Group ids only pair links within one run, so a rerun that skips re-mapping must not leave the measured - allele holding a group that now names another pair. A run renumbers to keep the link's group where the - measured allele still folds in, and otherwise supersedes the link under the run's timestamp, keeping the - old group readable through ``as_of``.""" + """The measured group belongs to the projection step, so whatever a rerun's candidates look like, the + authoritative link keeps pointing at the same measured group and is never re-versioned by RT. Only the + derived links and candidate groups are superseded, under the run's timestamp, unless the record's input + could not be asked about (UTA unavailable).""" variant = Variant( score_set_id=sample_score_set.id, urn=f"{sample_score_set.urn}#11", @@ -1151,19 +1573,30 @@ async def test_rerun_supersedes_the_authoritative_link_when_its_group_changes( await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) assay_hgvs = "NM_000000.1:c.1A>G" - measured_candidate = "NM_000001.1:c.5A>G" - translate = fake_translate({measured_candidate: CANONICAL_AUTHORITATIVE_DIGEST}) + translate = fake_translate( + { + MEASURED_C: CANONICAL_AUTHORITATIVE_DIGEST, + MEASURED_PARTNER_G: "ga4gh:VA.partner", + OTHER_C: "ga4gh:VA.other_c", + OTHER_G: "ga4gh:VA.other_g", + } + ) + first_construct = fake_construct({assay_hgvs: [(MEASURED_C, MEASURED_PARTNER_G), (OTHER_C, OTHER_G)]}) with ( - patch( - f"{RT_MODULE}.construct_equivalent_variants", fake_construct({assay_hgvs: [(measured_candidate, None)]}) - ), + patch(f"{RT_MODULE}.construct_equivalent_variants", first_construct), patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), ): first = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) assert first.status == JobStatus.SUCCEEDED record = _record_for(session, variant.id) - assert _authoritative_link(session, record.id).projection_group == 0 + measured_group_id = _authoritative_link(session, record.id).projection_group + (first_candidate_group_id,) = { + link.projection_group + for link in _non_authoritative_links(session) + if link.projection_group != measured_group_id + } + first_live_ids = {link.id for link in _non_authoritative_links(session)} rerun = JobRun( urn="test:reverse_translate_variants_for_score_set:rerun", @@ -1187,25 +1620,90 @@ async def test_rerun_supersedes_the_authoritative_link_when_its_group_changes( ): await _reverse_translate(session, mock_worker_ctx, rerun) - versions = ( - session.query(MappingRecordAllele) - .filter( - MappingRecordAllele.mapping_record_id == record.id, - MappingRecordAllele.is_authoritative.is_(True), - ) - .order_by(MappingRecordAllele.id) - .all() - ) - live = _authoritative_link(session, record.id) - assert live.projection_group == expected_group - # The mapping job's link (group NULL) plus the first run's version (group 0), and one more if regrouped. - assert len(versions) == (3 if regrouped else 2) - prior = versions[-2] - assert prior.projection_group == (0 if regrouped else None) - assert prior.valid_to == live.valid_from - assert live.allele_id == prior.allele_id + # The measured group and its authoritative link are exactly as the first run's projection left them. + assert _authoritative_link(session, record.id).projection_group == measured_group_id + assert _group(session, measured_group_id).current + assert len(_authoritative_versions(session, record.id)) == 2 + live_derived = [link for link in _non_authoritative_links(session) if link.valid_to is None] - assert {link.projection_group for link in live_derived} == derived_groups + assert { + (link.allele.vrs_digest, "measured" if link.projection_group == measured_group_id else "candidate") + for link in live_derived + } == expected_live + + first_candidate_group = _group(session, first_candidate_group_id) + assert first_candidate_group.current is candidate_group_kept + if candidate_group_kept: + assert {link.id for link in live_derived} == first_live_ids + else: + # Every live candidate link points at a group minted by this run, under the run's timestamp. + live_candidate_groups = { + link.projection_group for link in live_derived if link.projection_group != measured_group_id + } + assert first_candidate_group_id not in live_candidate_groups + assert {_group(session, group_id).valid_from for group_id in live_candidate_groups} == { + first_candidate_group.valid_to + } + + async def test_a_changed_standalone_projection_detaches_the_measured_groups_rt_alleles( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """Rewriting a measured group outside the reverse translation job retires the alleles RT added to it, + reports them as ``detached_rt_links``, and leaves candidate groups live.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn=f"{sample_score_set.urn}#11", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + translate = fake_translate( + { + MEASURED_C: CANONICAL_AUTHORITATIVE_DIGEST, + MEASURED_PARTNER_G: "ga4gh:VA.partner", + OTHER_C: "ga4gh:VA.other_c", + OTHER_G: "ga4gh:VA.other_g", + } + ) + construct = fake_construct({"NM_000000.1:c.1A>G": [(MEASURED_C, MEASURED_PARTNER_G), (OTHER_C, OTHER_G)]}) + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + record = _record_for(session, variant.id) + measured_group_id = _authoritative_link(session, record.id).projection_group + attached = [link for link in _non_authoritative_links(session) if link.projection_group == measured_group_id] + candidates = [link for link in _non_authoritative_links(session) if link.projection_group != measured_group_id] + assert attached and candidates + + class Shifted(FakeCoordinates): + def c_to_g(self, c_hgvs): + return "NC_000001.11:g.2000A>G" + + counts = project_measured_variants( + session, sample_score_set.id, coordinates=Shifted(), resolve_transcript=lambda *_: None + ) + session.commit() + + assert counts["detached_rt_links"] == len(attached) + for link in attached + candidates: + session.refresh(link) + assert all(link.valid_to is not None for link in attached) + assert all(link.valid_to is None for link in candidates) + assert not _group(session, measured_group_id).current @pytest.mark.parametrize( "rerun_outcome, keeps_prior_link", @@ -1289,6 +1787,8 @@ async def test_rerun_keeps_prior_links_only_when_upstream_unavailable( session.refresh(down_link) assert (down_link.valid_to is None) is keeps_prior_link + # The link's candidate group shares its fate. + assert _group(session, down_link.projection_group).current is keeps_prior_link live_digests = sorted( link.allele.vrs_digest for link in _non_authoritative_links(session) if link.valid_to is None ) diff --git a/tests/worker/lib/test_projection.py b/tests/worker/lib/test_projection.py new file mode 100644 index 000000000..f854ba041 --- /dev/null +++ b/tests/worker/lib/test_projection.py @@ -0,0 +1,209 @@ +# ruff: noqa: E402 +import pytest + +pytest.importorskip("arq") + +from mavedb.models.enums.sequence_level import SequenceLevel +from mavedb.worker.lib.projection import MeasuredHgvs, project_measured_hgvs + + +class FakeCoordinates: + """Answers the projections in ``answers``; anything else raises, like hgvs on unknown input.""" + + def __init__(self, answers: dict[tuple[str, str], str]): + self.answers = answers + + def _answer(self, op: str, hgvs: str) -> str: + if (op, hgvs) not in self.answers: + raise ValueError(f"no {op} for {hgvs}") + return self.answers[(op, hgvs)] + + def g_to_c(self, g_hgvs: str, transcript: str) -> str: + return self._answer("g_to_c", g_hgvs) + + def normalize(self, hgvs_string: str) -> str: + return hgvs_string + + def c_to_g(self, c_hgvs: str) -> str: + return self._answer("c_to_g", c_hgvs) + + def c_to_p(self, c_hgvs: str) -> str: + return self._answer("c_to_p", c_hgvs) + + +MULTI_CODON_G = "NC_000008.11:g.95025143_95025145delinsAAC" +MULTI_CODON_C = "NM_152416.4:c.135_137delinsAAC" +INTRONIC_G = "NC_000002.12:g.214809442A>G" +INTRONIC_C = "NM_000465.4:c.123+5A>G" + + +@pytest.mark.unit +class TestProjectMeasuredHgvs: + def test_genomic_measurement_projects_to_coding_and_protein(self): + coordinates = FakeCoordinates( + { + ("g_to_c", MULTI_CODON_G): MULTI_CODON_C, + ("c_to_p", MULTI_CODON_C): "NP_689629.2:p.(Ser45_Val46delinsArgThr)", + } + ) + + measured, errors = project_measured_hgvs( + SequenceLevel.genomic, MULTI_CODON_G, transcript="NM_152416.4", coordinates=coordinates + ) + + # A multi-codon change reverse translation cannot reach still gets its coding coordinate, and the + # prediction parens are stripped like reverse translation's. + assert measured == MeasuredHgvs( + hgvs_c=MULTI_CODON_C, hgvs_g=MULTI_CODON_G, hgvs_p="NP_689629.2:p.Ser45_Val46delinsArgThr" + ) + assert errors == [] + + def test_intronic_change_keeps_its_coding_coordinate_without_a_protein(self): + coordinates = FakeCoordinates({("g_to_c", INTRONIC_G): INTRONIC_C, ("c_to_p", INTRONIC_C): "NP_000456.2:p.?"}) + + measured, errors = project_measured_hgvs( + SequenceLevel.genomic, INTRONIC_G, transcript="NM_000465.4", coordinates=coordinates + ) + + assert measured == MeasuredHgvs(hgvs_c=INTRONIC_C, hgvs_g=INTRONIC_G, hgvs_p=None) + assert errors == [] + + def test_coding_measurement_projects_to_genomic_and_protein(self): + coordinates = FakeCoordinates( + { + ("c_to_g", MULTI_CODON_C): MULTI_CODON_G, + ("c_to_p", MULTI_CODON_C): "NP_689629.2:p.(Ser45_Val46delinsArgThr)", + } + ) + + measured, _ = project_measured_hgvs(SequenceLevel.cdna, MULTI_CODON_C, transcript=None, coordinates=coordinates) + + assert measured.hgvs_g == MULTI_CODON_G + assert measured.hgvs_c == MULTI_CODON_C + + def test_genomic_measurement_without_a_transcript_has_only_its_genomic_hgvs(self): + measured, errors = project_measured_hgvs( + SequenceLevel.genomic, MULTI_CODON_G, transcript=None, coordinates=FakeCoordinates({}) + ) + + assert measured == MeasuredHgvs(hgvs_g=MULTI_CODON_G) + assert errors == [] + + def test_failed_protein_keeps_the_coding_projection(self): + coordinates = FakeCoordinates({("g_to_c", MULTI_CODON_G): MULTI_CODON_C}) + + measured, errors = project_measured_hgvs( + SequenceLevel.genomic, MULTI_CODON_G, transcript="NM_152416.4", coordinates=coordinates + ) + + assert measured == MeasuredHgvs(hgvs_c=MULTI_CODON_C, hgvs_g=MULTI_CODON_G) + assert len(errors) == 1 and errors[0].startswith("c_to_p") + + def test_protein_measurement_is_its_own_description(self): + measured, errors = project_measured_hgvs( + SequenceLevel.protein, "NP_689629.2:p.Val46Ala", transcript=None, coordinates=FakeCoordinates({}) + ) + + assert measured == MeasuredHgvs(hgvs_p="NP_689629.2:p.Val46Ala") + assert errors == [] + + def test_genomic_measurement_is_normalized_before_it_is_projected(self): + """An untrimmed delins at an exon boundary projects to the coding deletion it is, not to an + intron-spanning delins with no protein consequence.""" + raw = "NC_000010.11:g.87965286_87965289delinsG" + trimmed = "NC_000010.11:g.87965287_87965289del" + + class Normalizing(FakeCoordinates): + def normalize(self, hgvs_string: str) -> str: + return trimmed if hgvs_string == raw else hgvs_string + + coordinates = Normalizing( + { + ("g_to_c", trimmed): "NM_000314.8:c.1027_1029del", + ("c_to_p", "NM_000314.8:c.1027_1029del"): "NP_000305.3:p.(Val343del)", + } + ) + + measured, errors = project_measured_hgvs( + SequenceLevel.genomic, raw, transcript="NM_000314.8", coordinates=coordinates + ) + + assert measured == MeasuredHgvs( + hgvs_c="NM_000314.8:c.1027_1029del", hgvs_g=trimmed, hgvs_p="NP_000305.3:p.Val343del" + ) + assert errors == [] + + def test_an_inversion_is_projected_as_the_equivalent_delins(self): + """VRS cannot represent ``inv``; hgvs's normalizing mapper writes one, so the coding projection is + taken from the unnormalized mapper instead.""" + genomic = "NC_000014.9:g.90403898_90403899delinsTG" + + class Literal(FakeCoordinates): + def g_to_c_literal(self, g_hgvs: str, transcript: str) -> str: + return "NM_006888.6:c.215_216delinsCA" + + coordinates = Literal( + { + ("g_to_c", genomic): "NM_006888.6:c.215_216inv", + ("c_to_p", "NM_006888.6:c.215_216delinsCA"): "NP_008819.1:p.(Met72Thr)", + } + ) + + measured, errors = project_measured_hgvs( + SequenceLevel.genomic, genomic, transcript="NM_006888.6", coordinates=coordinates + ) + + assert measured.hgvs_c == "NM_006888.6:c.215_216delinsCA" + assert measured.hgvs_p == "NP_008819.1:p.Met72Thr" + assert errors == [] + + def test_a_reference_identity_is_projected_as_written(self): + """Normalizing ``g.100CCT=`` would collapse its three-base span to one position.""" + identity = "NC_000007.14:g.144548593_144548595=" + + class Collapsing(FakeCoordinates): + def normalize(self, hgvs_string: str) -> str: + return "NC_000007.14:g.144548593=" + + coordinates = Collapsing({("g_to_c", identity): "NM_022445.4:c.610_612="}) + + measured, _ = project_measured_hgvs( + SequenceLevel.genomic, identity, transcript="NM_022445.4", coordinates=coordinates + ) + + assert measured.hgvs_g == identity + assert measured.hgvs_c == "NM_022445.4:c.610_612=" + + def test_a_spanless_multi_base_identity_is_given_its_span(self): + """Legacy ``g.100CCT=`` names one position with a three-base reference, which hgvs rejects.""" + coordinates = FakeCoordinates( + { + ("g_to_c", "NC_000007.14:g.144548593_144548595="): "NM_022445.4:c.509_511=", + ("c_to_p", "NM_022445.4:c.509_511="): "NP_071890.2:p.(His170=)", + } + ) + + measured, errors = project_measured_hgvs( + SequenceLevel.genomic, "NC_000007.14:g.144548593CCT=", transcript="NM_022445.4", coordinates=coordinates + ) + + assert measured == MeasuredHgvs( + hgvs_c="NM_022445.4:c.509_511=", + hgvs_g="NC_000007.14:g.144548593_144548595=", + hgvs_p="NP_071890.2:p.His170=", + ) + assert errors == [] + + def test_a_synonymous_stop_is_written_with_equals(self): + coordinates = FakeCoordinates( + { + ("g_to_c", "NC_000002.12:g.214728677C>T"): "NM_000465.4:c.2333G>A", + ("c_to_p", "NM_000465.4:c.2333G>A"): "NP_000456.2:p.(Ter778Ter)", + } + ) + + measured, _ = project_measured_hgvs( + SequenceLevel.genomic, "NC_000002.12:g.214728677C>T", transcript="NM_000465.4", coordinates=coordinates + ) + + assert measured.hgvs_p == "NP_000456.2:p.Ter778=" From 61e2b1838bc9f0ff562bcfca8e7b69790d417bc8 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 14 Oct 2026 23:14:52 -0700 Subject: [PATCH 244/252] fix(worker): run pipeline starts and variant creation on a fast-lane queue With the main worker capped at two jobs, hour-long mapping jobs could hold both slots and leave new score sets waiting for variant creation. start_pipeline and create_variants_for_score_set now go to a separate ARQ queue consumed by a second worker process (FastLaneWorkerSettings). - arq_queue_name() routes each JobRun; every enqueue and cleanup's ARQ status/abort checks resolve the queue through it. - The main worker still registers the fast-lane jobs so any already on the default queue at deploy time still run. - Requires the fast-worker container from the deployment repo. --- .github/instructions/worker.instructions.md | 3 +- src/mavedb/lib/workflow/kickoff.py | 8 +++-- src/mavedb/scripts/run_job.py | 6 ++-- src/mavedb/scripts/run_score_set_pipelines.py | 3 +- src/mavedb/worker/README.md | 4 +-- src/mavedb/worker/__init__.py | 5 +-- src/mavedb/worker/job_registry.md | 8 +++++ src/mavedb/worker/jobs/system/cleanup.py | 14 ++++---- .../worker/lib/managers/pipeline_manager.py | 7 +++- src/mavedb/worker/lib/managers/utils.py | 20 +++++++++++ src/mavedb/worker/settings/__init__.py | 5 +-- src/mavedb/worker/settings/worker.py | 28 +++++++++++++++- tests/lib/workflow/test_kickoff.py | 2 ++ tests/worker/jobs/system/test_cleanup.py | 33 ++++++++++++++++++- .../lib/managers/test_pipeline_manager.py | 3 +- tests/worker/lib/managers/test_utils.py | 20 +++++++++++ tests/worker/test_settings.py | 31 +++++++++++++++++ 17 files changed, 177 insertions(+), 23 deletions(-) create mode 100644 tests/worker/test_settings.py diff --git a/.github/instructions/worker.instructions.md b/.github/instructions/worker.instructions.md index 55a31abeb..8d654c2d5 100644 --- a/.github/instructions/worker.instructions.md +++ b/.github/instructions/worker.instructions.md @@ -124,6 +124,7 @@ Parameters with `None` values in pipeline definitions are filled at runtime from - **Slack safety**: `send_slack_job_failure()` and `send_slack_job_error()` catch their own exceptions internally (logging critical on failure), so Slack outages never interfere with job lifecycle management or error recovery in the decorators. - **Stale RUNNING recovery**: `start_job()` accepts RUNNING as a startable status (alongside QUEUED and PENDING). When ARQ re-delivers a job after a worker crash, `start_job()` logs a warning and resets the start timestamp rather than raising `JobTransitionError`. - **Concurrency limit**: `max_jobs = 2` in `ArqWorkerSettings` prevents event loop starvation from sync psycopg2 DB calls. With the default `max_jobs=10`, multiple concurrent jobs issuing blocking DB operations can starve the asyncio event loop. +- **Fast lane**: `start_pipeline` and `create_variants_for_score_set` run on a separate queue (`FAST_LANE_QUEUE_NAME`) consumed by a second worker process (`ArqFastLaneWorkerSettings`), so long mapping jobs can't block new pipelines. Every `enqueue_job` and every `arq.jobs.Job(...)` status/abort call must pass `_queue_name=arq_queue_name(job_run)`. ## Pipeline Lifecycle (Brief) @@ -181,7 +182,7 @@ Pipeline definitions live in `src/mavedb/lib/workflow/definitions.py`. The `Pipe | `lib/managers/pipeline_manager.py` | Pipeline coordination, dependency resolution, job enqueueing | | `lib/managers/constants.py` | Status groupings (`TERMINAL_JOB_STATUSES`, `STARTABLE_JOB_STATUSES`, etc.) | | `lib/managers/exceptions.py` | Exception hierarchy (`JobStateError`, `PipelineCoordinationError`, etc.) | -| `settings/worker.py` | `ArqWorkerSettings` class (ARQ worker configuration) | +| `settings/worker.py` | `ArqWorkerSettings` (main) and `ArqFastLaneWorkerSettings` classes (ARQ worker configuration) | | `settings/lifecycle.py` | Worker startup/shutdown hooks, `standalone_ctx()` | | `src/mavedb/lib/workflow/definitions.py` | Pipeline and job definitions (`PIPELINE_DEFINITIONS`) | | `src/mavedb/lib/workflow/pipeline_factory.py` | Creates Pipeline + JobRun + JobDependency records | diff --git a/src/mavedb/lib/workflow/kickoff.py b/src/mavedb/lib/workflow/kickoff.py index 94feec0bf..35f236bf2 100644 --- a/src/mavedb/lib/workflow/kickoff.py +++ b/src/mavedb/lib/workflow/kickoff.py @@ -3,7 +3,7 @@ The one shared kickoff seam: build the ``Pipeline`` / ``JobRun`` / ``JobDependency`` records via :class:`PipelineFactory`, then enqueue the ``start_pipeline`` entrypoint in ARQ. Both the ad-hoc ``run_pipeline`` script and the batch ``enrich_backfilled_score_sets`` driver go through here so the -enqueue contract (entrypoint id + :func:`arq_job_id`) lives in exactly one place. +enqueue contract (entrypoint id + :func:`arq_job_id` + :func:`arq_queue_name`) lives in exactly one place. """ from __future__ import annotations @@ -21,7 +21,7 @@ from mavedb.models.pipeline import Pipeline from mavedb.models.score_set import ScoreSet from mavedb.models.user import User -from mavedb.worker.lib.managers.utils import arq_job_id +from mavedb.worker.lib.managers.utils import arq_job_id, arq_queue_name logger = logging.getLogger(__name__) @@ -63,7 +63,9 @@ async def enqueue_pipeline_for_score_set( job_id = arq_job_id(entrypoint) try: - job = await redis.enqueue_job(entrypoint.job_function, entrypoint.id, _job_id=job_id) + job = await redis.enqueue_job( + entrypoint.job_function, entrypoint.id, _job_id=job_id, _queue_name=arq_queue_name(entrypoint) + ) except Exception: logger.error( "Failed to enqueue pipeline %s for score set %s; discarding pipeline %s", diff --git a/src/mavedb/scripts/run_job.py b/src/mavedb/scripts/run_job.py index 3204b14b4..55164235a 100644 --- a/src/mavedb/scripts/run_job.py +++ b/src/mavedb/scripts/run_job.py @@ -35,7 +35,7 @@ from mavedb.models.score_set import ScoreSet from mavedb.models.user import User from mavedb.worker.jobs.registry import STANDALONE_JOB_DEFINITIONS -from mavedb.worker.lib.managers.utils import arq_job_id +from mavedb.worker.lib.managers.utils import arq_job_id, arq_queue_name from mavedb.worker.settings import RedisWorkerSettings from mavedb.worker.settings.lifecycle import standalone_ctx @@ -221,7 +221,9 @@ async def _enqueue_jobs( db.flush() arq_id = arq_job_id(job_run) - job = await redis.enqueue_job(job_run.job_function, job_run.id, _job_id=arq_id) + job = await redis.enqueue_job( + job_run.job_function, job_run.id, _job_id=arq_id, _queue_name=arq_queue_name(job_run) + ) if job: click.echo(f"Enqueued {job_name} (job_run={job_run.id}, arq_id={arq_id})") else: diff --git a/src/mavedb/scripts/run_score_set_pipelines.py b/src/mavedb/scripts/run_score_set_pipelines.py index c5872dbf9..f27f38489 100644 --- a/src/mavedb/scripts/run_score_set_pipelines.py +++ b/src/mavedb/scripts/run_score_set_pipelines.py @@ -75,7 +75,7 @@ from mavedb.models.score_set import ScoreSet from mavedb.models.user import User from mavedb.scripts.run_pipeline import _print_available_pipelines -from mavedb.worker.lib.managers.utils import arq_job_id +from mavedb.worker.lib.managers.utils import arq_job_id, arq_queue_name from mavedb.worker.settings import RedisWorkerSettings logger = logging.getLogger(__name__) @@ -507,6 +507,7 @@ async def enqueue_pipeline( pipeline_entrypoint.job_function, pipeline_entrypoint.id, _job_id=arq_job_id(pipeline_entrypoint), + _queue_name=arq_queue_name(pipeline_entrypoint), ) except Exception as e: pipeline_factory.discard_pipeline(pipeline) diff --git a/src/mavedb/worker/README.md b/src/mavedb/worker/README.md index 55ed05eab..77f6328fe 100644 --- a/src/mavedb/worker/README.md +++ b/src/mavedb/worker/README.md @@ -96,10 +96,10 @@ worker/ │ ├── constants.py # Status grouping constants │ ├── exceptions.py # Exception hierarchy │ ├── types.py # TypedDicts (RetryHistoryEntry, PipelineProgress) -│ └── utils.py # Dependency checking helpers, classify_exception() +│ └── utils.py # Dependency checking helpers, classify_exception(), ARQ job id/queue routing │ └── settings/ # ARQ worker configuration - ├── worker.py # ArqWorkerSettings class + ├── worker.py # ArqWorkerSettings (main) and ArqFastLaneWorkerSettings classes ├── lifecycle.py # Startup/shutdown/job hooks, standalone_ctx() ├── redis.py # Redis connection settings └── constants.py # Environment variable handling diff --git a/src/mavedb/worker/__init__.py b/src/mavedb/worker/__init__.py index 1f1489b59..04d73a604 100644 --- a/src/mavedb/worker/__init__.py +++ b/src/mavedb/worker/__init__.py @@ -1,7 +1,7 @@ from sqlalchemy.orm import configure_mappers from mavedb.models import * # noqa: F403 -from mavedb.worker.settings import ArqWorkerSettings +from mavedb.worker.settings import ArqFastLaneWorkerSettings, ArqWorkerSettings # Scan all our model classes and create backref attributes. Otherwise, these attributes only get added to classes once # an instance of the related class has been created. Since the worker is a distinct service, we should make sure this @@ -9,5 +9,6 @@ configure_mappers() # Expose the worker settings to the arq CLI Worker initializer -global WorkerSettings +global WorkerSettings, FastLaneWorkerSettings WorkerSettings = ArqWorkerSettings +FastLaneWorkerSettings = ArqFastLaneWorkerSettings diff --git a/src/mavedb/worker/job_registry.md b/src/mavedb/worker/job_registry.md index 44021e333..3b7deb5d9 100644 --- a/src/mavedb/worker/job_registry.md +++ b/src/mavedb/worker/job_registry.md @@ -245,6 +245,14 @@ class ArqWorkerSettings: allow_abort_jobs = True # Lets cleanup_stalled_jobs abort a wedged RUNNING job ``` +### Fast-lane worker + +`ArqFastLaneWorkerSettings` runs as a second worker process (`arq mavedb.worker.FastLaneWorkerSettings`, its own container on the worker host). It consumes only `FAST_LANE_QUEUE_NAME` and registers only the jobs in `FAST_LANE_JOB_FUNCTIONS` (`start_pipeline`, `create_variants_for_score_set`), so new pipelines get their variants created even while hour-long mapping jobs hold every main-worker slot. It runs no cron jobs. + +Routing lives in `arq_queue_name()` (`lib/managers/utils.py`). Every ARQ call that takes a queue — `enqueue_job(_queue_name=...)`, `Job(..., _queue_name=...).status()` / `.abort()` — must resolve it from the `JobRun` there; a status check against the wrong queue reports a queued job as missing. To move a job into the fast lane, add its function name to `FAST_LANE_JOB_FUNCTIONS`; keep it short-running, since it shares two slots with variant creation. + +ARQ reads settings from the class `__dict__`, so the fast-lane class repeats its attributes rather than subclassing `ArqWorkerSettings`. + The lifecycle hooks (in `settings/lifecycle.py`) manage the ARQ context dict (`ctx`): - `startup`: Creates `ProcessPoolExecutor` for CPU-intensive tasks - `on_job_start`: Initializes `hdp` (CDOT REST data provider) and `state` dict diff --git a/src/mavedb/worker/jobs/system/cleanup.py b/src/mavedb/worker/jobs/system/cleanup.py index 40e636be6..f44ed9221 100644 --- a/src/mavedb/worker/jobs/system/cleanup.py +++ b/src/mavedb/worker/jobs/system/cleanup.py @@ -33,7 +33,7 @@ from mavedb.worker.lib.managers.constants import ACTIVE_JOB_STATUSES, TERMINAL_PIPELINE_STATUSES from mavedb.worker.lib.managers.job_manager import JobManager from mavedb.worker.lib.managers.pipeline_manager import PipelineManager -from mavedb.worker.lib.managers.utils import arq_job_id +from mavedb.worker.lib.managers.utils import arq_job_id, arq_queue_name logger = logging.getLogger(__name__) @@ -179,7 +179,9 @@ async def _handle_stalled_job_retry( try: manager.prepare_queue() # Transition to QUEUED db.flush() - result = await redis.enqueue_job(job.job_function, job.id, _job_id=arq_job_id(job)) + result = await redis.enqueue_job( + job.job_function, job.id, _job_id=arq_job_id(job), _queue_name=arq_queue_name(job) + ) if result is None: raise RuntimeError( @@ -210,7 +212,7 @@ async def _handle_stalled_job_retry( return False -async def _running_job_confirmed_dead(arq_id: str, redis: ArqRedis) -> bool: +async def _running_job_confirmed_dead(arq_id: str, queue_name: str, redis: ArqRedis) -> bool: """Abort a RUNNING job's current ARQ attempt and report whether it is safe to re-drive. RUNNING recovery differs from QUEUED/PENDING recovery: a RUNNING job may still have a live @@ -232,7 +234,7 @@ async def _running_job_confirmed_dead(arq_id: str, redis: ArqRedis) -> bool: Returns True when it is safe to re-drive the job, False when the job may still be running. """ try: - await ArqJob(arq_id, redis).abort(timeout=ABORT_CONFIRM_TIMEOUT_SECONDS) + await ArqJob(arq_id, redis, _queue_name=queue_name).abort(timeout=ABORT_CONFIRM_TIMEOUT_SECONDS) return True except asyncio.TimeoutError: logger.warning( @@ -265,7 +267,7 @@ async def _handle_stalled_running_job( # Could not confirm the coroutine is dead. Leave the job RUNNING; the next sweep will retry # the abort, and ARQ's job_timeout ceiling remains as a final backstop. - if not await _running_job_confirmed_dead(arq_id, redis): + if not await _running_job_confirmed_dead(arq_id, arq_queue_name(job), redis): logger.warning( f"Leaving stalled RUNNING job {job.urn} in place — abort unconfirmed, re-driving would risk a double-run", extra=manager.logging_context(), @@ -369,7 +371,7 @@ async def cleanup_stalled_jobs(ctx: dict, job_id: int, job_manager: JobManager) # A healthy job waiting for a worker slot appears QUEUED in the DB and is also # present in Redis; only a crashed-enqueue job has the DB state without the # corresponding Redis entry. - arq_status = await ArqJob(arq_job_id(job), job_manager.redis).status() + arq_status = await ArqJob(arq_job_id(job), job_manager.redis, _queue_name=arq_queue_name(job)).status() if arq_status in (ArqJobStatus.queued, ArqJobStatus.in_progress, ArqJobStatus.deferred): logger.debug( f"QUEUED job {job.urn} is present in ARQ Redis (status={arq_status.value}); skipping cleanup", diff --git a/src/mavedb/worker/lib/managers/pipeline_manager.py b/src/mavedb/worker/lib/managers/pipeline_manager.py index 68f56c9d4..8c0f6c418 100644 --- a/src/mavedb/worker/lib/managers/pipeline_manager.py +++ b/src/mavedb/worker/lib/managers/pipeline_manager.py @@ -64,6 +64,7 @@ ) from mavedb.worker.lib.managers.utils import ( arq_job_id, + arq_queue_name, construct_bulk_cancellation_result, job_dependency_is_met, job_should_be_skipped_due_to_unfulfillable_dependency, @@ -1278,7 +1279,11 @@ async def _enqueue_in_arq(self, job: JobRun, is_retry: bool) -> None: try: defer_by = timedelta(seconds=job.retry_delay_seconds if is_retry and job.retry_delay_seconds else 0) arq_success = await self.redis.enqueue_job( - job.job_function, job.id, _defer_by=defer_by, _job_id=arq_job_id(job) + job.job_function, + job.id, + _defer_by=defer_by, + _job_id=arq_job_id(job), + _queue_name=arq_queue_name(job), ) except Exception as e: logger.debug(f"ARQ enqueue operation failed for job {job.urn}: {e}") diff --git a/src/mavedb/worker/lib/managers/utils.py b/src/mavedb/worker/lib/managers/utils.py index d6a266647..49135599c 100644 --- a/src/mavedb/worker/lib/managers/utils.py +++ b/src/mavedb/worker/lib/managers/utils.py @@ -12,6 +12,7 @@ import redis.exceptions import requests.exceptions import sqlalchemy.exc +from arq.constants import default_queue_name from mavedb.lib.types.workflow import JobExecutionOutcome from mavedb.models.enums.job_pipeline import DependencyType, FailureCategory, JobStatus @@ -44,6 +45,16 @@ OSError: FailureCategory.NETWORK_ERROR, } +FAST_LANE_QUEUE_NAME = "arq:queue:fast-lane" +"""ARQ queue consumed by a dedicated worker process, independent of the main worker's job slots.""" + +FAST_LANE_JOB_FUNCTIONS = frozenset({"start_pipeline", "create_variants_for_score_set"}) +"""Short jobs that must not wait behind hour-long mapping jobs holding every main-worker slot. + +``start_pipeline`` is included because it enqueues ``create_variants_for_score_set``; routing only the +latter would leave a new pipeline stuck behind mapping before variant creation is ever enqueued. +""" + def classify_exception(exc: Exception) -> FailureCategory: """Map an exception to a FailureCategory. Uses isinstance to match parent classes.""" @@ -65,6 +76,15 @@ def arq_job_id(job: JobRun) -> str: return f"{job.urn}#{job.retry_count or 0}" +def arq_queue_name(job: JobRun) -> str: + """Return the ARQ queue a JobRun is enqueued on. + + Every ARQ call that takes a queue name (``enqueue_job``, ``Job.status``, ``Job.abort``) must resolve it + here; a status lookup against the wrong queue reports a queued job as ``not_found``. + """ + return FAST_LANE_QUEUE_NAME if job.job_function in FAST_LANE_JOB_FUNCTIONS else default_queue_name + + def construct_bulk_cancellation_result(reason: str) -> JobExecutionOutcome: """Construct a standardized JobExecutionOutcome for bulk job cancellations. diff --git a/src/mavedb/worker/settings/__init__.py b/src/mavedb/worker/settings/__init__.py index af2e6a275..9f0c43198 100644 --- a/src/mavedb/worker/settings/__init__.py +++ b/src/mavedb/worker/settings/__init__.py @@ -4,16 +4,17 @@ - constants: Environment variable configuration - redis: Redis connection settings - lifecycle: Worker startup/shutdown hooks -- worker: Main ARQ worker configuration class +- worker: Main and fast-lane ARQ worker configuration classes The settings are designed to be modular and easily testable, with clear separation between infrastructure and application concerns. """ from .redis import RedisWorkerSettings -from .worker import ArqWorkerSettings +from .worker import ArqFastLaneWorkerSettings, ArqWorkerSettings __all__ = [ + "ArqFastLaneWorkerSettings", "ArqWorkerSettings", "RedisWorkerSettings", ] diff --git a/src/mavedb/worker/settings/worker.py b/src/mavedb/worker/settings/worker.py index f205fae7e..645809c20 100644 --- a/src/mavedb/worker/settings/worker.py +++ b/src/mavedb/worker/settings/worker.py @@ -7,11 +7,13 @@ - Lifecycle hooks for startup/shutdown and job execution - Timeout and logging configuration -This is the main configuration class used to start the ARQ worker. +ArqWorkerSettings configures the main worker; ArqFastLaneWorkerSettings configures the +fast-lane worker that runs short jobs independently of the main worker's slots. """ from mavedb.lib.logging.canonical import log_job from mavedb.worker.jobs import BACKGROUND_CRONJOBS, BACKGROUND_FUNCTIONS +from mavedb.worker.lib.managers.utils import FAST_LANE_JOB_FUNCTIONS, FAST_LANE_QUEUE_NAME from mavedb.worker.settings.lifecycle import on_job_end, on_job_start, shutdown, startup from mavedb.worker.settings.redis import RedisWorkerSettings @@ -54,3 +56,27 @@ class ArqWorkerSettings: # before re-driving it. Without this, ARQ ignores abort requests and the sweeper cannot safely # confirm a stalled job is dead before retrying it. allow_abort_jobs = True + + +class ArqFastLaneWorkerSettings: + """ + Settings for the fast-lane ARQ worker, run as a separate process from the main worker. + + Consumes only FAST_LANE_QUEUE_NAME so variant creation never waits for a main-worker slot. The main + worker still registers these functions to drain any already on the default queue. ARQ reads settings + from the class ``__dict__``, so attributes are not inherited from ArqWorkerSettings. No cron jobs: + those run once, on the main worker. + """ + + on_startup = startup + on_shutdown = shutdown + on_job_start = on_job_start + on_job_end = on_job_end + after_job_end = log_job + redis_settings = RedisWorkerSettings + queue_name = FAST_LANE_QUEUE_NAME + functions: list = [f for f in BACKGROUND_FUNCTIONS if f.__name__ in FAST_LANE_JOB_FUNCTIONS] + + max_jobs = MAX_JOBS + job_timeout = JOB_TIMEOUT_SECONDS + allow_abort_jobs = True diff --git a/tests/lib/workflow/test_kickoff.py b/tests/lib/workflow/test_kickoff.py index 9b98eaf45..f81879b12 100644 --- a/tests/lib/workflow/test_kickoff.py +++ b/tests/lib/workflow/test_kickoff.py @@ -12,6 +12,7 @@ from mavedb.models.job_run import JobRun from mavedb.models.pipeline import Pipeline from mavedb.models.user import User +from mavedb.worker.lib.managers.utils import FAST_LANE_QUEUE_NAME from tests.helpers.constants import TEST_USER @@ -56,6 +57,7 @@ async def test_returns_pipeline_and_job_when_enqueued(self): self.entrypoint.job_function, self.entrypoint.id, _job_id=f"{self.entrypoint.urn}#{self.entrypoint.retry_count}", + _queue_name=FAST_LANE_QUEUE_NAME, ) async def test_returns_none_job_when_arq_deduplicates(self): diff --git a/tests/worker/jobs/system/test_cleanup.py b/tests/worker/jobs/system/test_cleanup.py index 084fe93d7..1a127326a 100644 --- a/tests/worker/jobs/system/test_cleanup.py +++ b/tests/worker/jobs/system/test_cleanup.py @@ -35,7 +35,7 @@ cleanup_stalled_jobs, ) from mavedb.worker.lib.managers.job_manager import JobManager -from mavedb.worker.lib.managers.utils import arq_job_id +from mavedb.worker.lib.managers.utils import FAST_LANE_QUEUE_NAME, arq_job_id pytestmark = pytest.mark.usefixtures("patch_db_session_ctxmgr") @@ -1714,6 +1714,37 @@ async def test_cleanup_integration_excludes_recent_jobs(self, standalone_worker_ assert recent_queued.retry_count == 0 assert recent_running.retry_count == 0 + async def test_cleanup_integration_leaves_fast_lane_queued_job_alone(self, standalone_worker_context, session): + """Integration test: the Redis presence check looks for a fast-lane job on the fast-lane queue.""" + queued_creation = JobRun( + job_type="test_job", + job_function="create_variants_for_score_set", + status=JobStatus.QUEUED, + created_at=datetime.now(timezone.utc) - timedelta(minutes=PENDING_TIMEOUT_MINUTES + 5), + started_at=None, + max_retries=3, + retry_count=0, + job_params={}, + ) + session.add(queued_creation) + session.commit() + + arq_redis = standalone_worker_context["redis"] + await arq_redis.enqueue_job( + "create_variants_for_score_set", + queued_creation.id, + _job_id=arq_job_id(queued_creation), + _queue_name=FAST_LANE_QUEUE_NAME, + ) + + result = await cleanup_stalled_jobs(standalone_worker_context) + + assert result.status == JobStatus.SUCCEEDED + assert result.data["total_cleaned"] == 0 + session.refresh(queued_creation) + assert queued_creation.status == JobStatus.QUEUED + assert queued_creation.retry_count == 0 + async def test_cleanup_integration_updates_progress_correctly(self, standalone_worker_context, session): """Integration test: cleanup job updates progress correctly and returns proper data.""" # Create stalled jobs to trigger progress updates across different states diff --git a/tests/worker/lib/managers/test_pipeline_manager.py b/tests/worker/lib/managers/test_pipeline_manager.py index 3028a31d2..d66a3a649 100644 --- a/tests/worker/lib/managers/test_pipeline_manager.py +++ b/tests/worker/lib/managers/test_pipeline_manager.py @@ -39,7 +39,7 @@ PipelineTransitionError, ) from mavedb.worker.lib.managers.pipeline_manager import PipelineManager -from mavedb.worker.lib.managers.utils import arq_job_id +from mavedb.worker.lib.managers.utils import arq_job_id, arq_queue_name from tests.helpers.transaction_spy import TransactionSpy HANDLED_EXCEPTIONS_DURING_OBJECT_MANIPULATION = ( @@ -3567,6 +3567,7 @@ async def test_enqueue_in_arq_success(self, mock_pipeline_manager, retry, enqueu mock_job.id, _defer_by=datetime.timedelta(seconds=mock_job.retry_delay_seconds if retry else 0), _job_id=arq_job_id(mock_job), + _queue_name=arq_queue_name(mock_job), ) @pytest.mark.asyncio diff --git a/tests/worker/lib/managers/test_utils.py b/tests/worker/lib/managers/test_utils.py index 70e3ca24b..c1d9f27e5 100644 --- a/tests/worker/lib/managers/test_utils.py +++ b/tests/worker/lib/managers/test_utils.py @@ -4,6 +4,10 @@ pytest.importorskip("arq") +from unittest.mock import Mock + +from arq.constants import default_queue_name + from mavedb.lib.types.workflow import JobExecutionOutcome from mavedb.models.enums.job_pipeline import DependencyType, FailureCategory, JobStatus from mavedb.worker.lib.managers.constants import ( @@ -13,7 +17,10 @@ STARTABLE_JOB_STATUSES, TERMINAL_JOB_STATUSES, ) +from mavedb.models.job_run import JobRun from mavedb.worker.lib.managers.utils import ( + FAST_LANE_QUEUE_NAME, + arq_queue_name, classify_exception, construct_bulk_cancellation_result, job_dependency_is_met, @@ -148,3 +155,16 @@ def test_runtime_error_returns_unknown(self): def test_generic_exception_returns_unknown(self): assert classify_exception(Exception("generic")) == FailureCategory.UNKNOWN + + +@pytest.mark.unit +class TestArqQueueNameUnit: + """Tests for routing JobRuns to the fast-lane or default ARQ queue.""" + + @pytest.mark.parametrize("job_function", ["start_pipeline", "create_variants_for_score_set"]) + def test_fast_lane_functions_route_to_fast_lane_queue(self, job_function): + assert arq_queue_name(Mock(spec=JobRun, job_function=job_function)) == FAST_LANE_QUEUE_NAME + + @pytest.mark.parametrize("job_function", ["map_variants_for_score_set", "cleanup_stalled_jobs"]) + def test_other_functions_route_to_default_queue(self, job_function): + assert arq_queue_name(Mock(spec=JobRun, job_function=job_function)) == default_queue_name diff --git a/tests/worker/test_settings.py b/tests/worker/test_settings.py new file mode 100644 index 000000000..b1209f6ce --- /dev/null +++ b/tests/worker/test_settings.py @@ -0,0 +1,31 @@ +# ruff: noqa: E402 + +import pytest + +pytest.importorskip("arq") + +from arq.worker import get_kwargs + +from mavedb.worker.lib.managers.utils import FAST_LANE_JOB_FUNCTIONS, FAST_LANE_QUEUE_NAME +from mavedb.worker.settings import ArqFastLaneWorkerSettings, ArqWorkerSettings + + +@pytest.mark.unit +class TestArqFastLaneWorkerSettingsUnit: + """ARQ builds a Worker from the settings class ``__dict__``, so assert on what it actually reads.""" + + def test_consumes_fast_lane_queue(self): + assert get_kwargs(ArqFastLaneWorkerSettings)["queue_name"] == FAST_LANE_QUEUE_NAME + + def test_registers_exactly_the_fast_lane_functions(self): + functions = get_kwargs(ArqFastLaneWorkerSettings)["functions"] + assert {f.__name__ for f in functions} == FAST_LANE_JOB_FUNCTIONS + + def test_runs_no_cron_jobs(self): + assert "cron_jobs" not in get_kwargs(ArqFastLaneWorkerSettings) + + def test_main_worker_still_registers_fast_lane_functions(self): + """Fast-lane jobs already on the default queue at deploy time must still be runnable.""" + kwargs = get_kwargs(ArqWorkerSettings) + assert "queue_name" not in kwargs + assert FAST_LANE_JOB_FUNCTIONS <= {f.__name__ for f in kwargs["functions"]} From cfbb65bc674938b4e8d022440714d24c51d2163f Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Thu, 15 Oct 2026 08:50:49 -0700 Subject: [PATCH 245/252] fix(rt): keep a coding candidate spelled two ways paired with its genomic projection An insertion at an exon junction can be written on either side of it. Both spellings identify to one coding allele, but only the in-exon one projects to the genome, and an allele links to a record once. Link pairs with a genomic projection first so the shared allele joins that pair's group rather than a coding-only group. --- .../variant_processing/reverse_translation.py | 6 ++- .../test_reverse_translation.py | 46 +++++++++++++++++++ 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/src/mavedb/worker/jobs/variant_processing/reverse_translation.py b/src/mavedb/worker/jobs/variant_processing/reverse_translation.py index 91b30b6a2..2e1dc4a30 100644 --- a/src/mavedb/worker/jobs/variant_processing/reverse_translation.py +++ b/src/mavedb/worker/jobs/variant_processing/reverse_translation.py @@ -561,8 +561,10 @@ def link(allele: AlleleDbModel) -> MappingRecordAllele: ) # Each ProjectionPair is a coding candidate and its genomic projection: the same change at two levels. - # hgvs_g is None when the c→g projection failed, leaving a one-member group. - for pair in result.projection_pairs: + # hgvs_g is None when the c→g projection failed, leaving a one-member group. Two spellings of one coding + # change (an insertion either side of an exon junction) identify to one allele, which links only once, + # so pairs with a genomic projection go first to keep that allele with its genomic partner. + for pair in sorted(result.projection_pairs, key=lambda pair: pair.hgvs_g is None): members = [(pair.hgvs_c, SequenceLevel.cdna, "hgvs_c")] if pair.hgvs_g is not None: members.append((pair.hgvs_g, SequenceLevel.genomic, "hgvs_g")) diff --git a/tests/worker/jobs/variant_processing/test_reverse_translation.py b/tests/worker/jobs/variant_processing/test_reverse_translation.py index faae726a8..cc86c881b 100644 --- a/tests/worker/jobs/variant_processing/test_reverse_translation.py +++ b/tests/worker/jobs/variant_processing/test_reverse_translation.py @@ -1645,6 +1645,52 @@ async def test_rerun_never_regroups_the_measured_allele( first_candidate_group.valid_to } + async def test_a_coding_change_spelled_two_ways_stays_paired_with_its_genomic_projection( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """An insertion at an exon junction can be written on either side of it. Both spellings are one coding + allele, but only the one inside an exon projects to the genome; the allele joins that pair's group + rather than a coding-only group, whichever spelling the library lists first.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn=f"{sample_score_set.urn}#11", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + junction_c = "NM_000001.1:c.10_11insGAG" + exon_c = "NM_000001.1:c.11_12insAGG" + exon_g = "NC_000001.11:g.2000_2001insCCT" + construct = fake_construct({"NM_000000.1:c.1A>G": [(junction_c, None), (exon_c, exon_g)]}) + translate = fake_translate({junction_c: "ga4gh:VA.shared_c", exon_c: "ga4gh:VA.shared_c", exon_g: "ga4gh:VA.g"}) + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + record = _record_for(session, variant.id) + measured_group_id = _authoritative_link(session, record.id).projection_group + candidate_groups: dict[int, set[str]] = {} + for link in _non_authoritative_links(session): + if link.valid_to is None and link.projection_group != measured_group_id: + candidate_groups.setdefault(link.projection_group, set()).add(link.allele.vrs_digest) + + assert list(candidate_groups.values()) == [{"ga4gh:VA.shared_c", "ga4gh:VA.g"}] + (group_id,) = candidate_groups + assert (_group(session, group_id).hgvs_c, _group(session, group_id).hgvs_g) == (exon_c, exon_g) + async def test_a_changed_standalone_projection_detaches_the_measured_groups_rt_alleles( self, session, From 39609ae5bfe33c8e489d5213e892c82f5808a386 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Thu, 15 Oct 2026 09:02:25 -0700 Subject: [PATCH 246/252] fix(tests): expect the unversioned accession in the UniProt network test --- .../worker/jobs/external_services/network/test_uniprot.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/worker/jobs/external_services/network/test_uniprot.py b/tests/worker/jobs/external_services/network/test_uniprot.py index 742344c62..ff5d7f746 100644 --- a/tests/worker/jobs/external_services/network/test_uniprot.py +++ b/tests/worker/jobs/external_services/network/test_uniprot.py @@ -8,8 +8,10 @@ pytestmark = pytest.mark.usefixtures("patch_db_session_ctxmgr") -# UniProt ID mapping returns no results for unversioned RefSeq accessions. +# Mapping records the versioned transcript; ID mapping is submitted the unversioned form, which UniProt matches +# reliably (see id_mapping_query_accession). VERSIONED_REFSEQ_ACCESSION = "NM_003345.5" +SUBMITTED_ACCESSION = "NM_003345" @pytest.mark.asyncio @@ -47,12 +49,12 @@ async def test_uniprot_mapping_jobs_e2e( submitted_jobs = sample_submit_uniprot_mapping_jobs_run_in_pipeline.metadata_["submitted_jobs"] assert "1" in submitted_jobs assert submitted_jobs["1"]["job_id"] is not None - assert submitted_jobs["1"]["accession"] == VERSIONED_REFSEQ_ACCESSION + assert submitted_jobs["1"]["accession"] == SUBMITTED_ACCESSION # Verify that polling job params have been updated correctly session.refresh(sample_polling_job_for_submission_run_in_pipeline) assert sample_polling_job_for_submission_run_in_pipeline.job_params["mapping_jobs"] == { - "1": {"job_id": submitted_jobs["1"]["job_id"], "accession": VERSIONED_REFSEQ_ACCESSION} + "1": {"job_id": submitted_jobs["1"]["job_id"], "accession": SUBMITTED_ACCESSION} } # Verify that the submission job was completed successfully From b1eedee121a88a453d11189a3ed65fe4d5c351a9 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Thu, 15 Oct 2026 09:31:25 -0700 Subject: [PATCH 247/252] chore(dev): run the fast-lane worker in local compose --- README.md | 5 +++-- docker-compose-dev.yml | 8 +++++++- docker-compose-local.yml | 8 +++++++- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index c03bc35d7..8155e470c 100644 --- a/README.md +++ b/README.md @@ -67,8 +67,9 @@ To stop the application when it is running as a daemon, run docker-compose -f docker-compose-local.yml down ``` -`docker-compose-local.yml` configures four containers: one for the API server, one for the PostgreSQL 15 database, one for the -worker node and one for the Redis cache which acts as the job queue for the worker node. The worker node stores data in a Docker +`docker-compose-local.yml` configures five containers: one for the API server, one for the PostgreSQL 15 database, two worker +nodes and one for the Redis cache which acts as the job queue for the workers. The fast worker runs only pipeline starts and +variant creation, on its own queue, so new score sets get variants while mapping jobs occupy the main worker. The worker node stores data in a Docker volume named `mavedb-redis` and the database stores data in a Docker volume named `mavedb-data-15`. Both these volumes will persist after running `docker-compose down`. diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 013f549bd..99ae52b7d 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -32,7 +32,7 @@ services: - ../variant-annotation:/variant-annotation - mavedb-seqrepo-dev:/usr/local/share/seqrepo - worker: + worker: &worker image: mavedb-api/mavedb-worker:dev build: context: . @@ -61,6 +61,12 @@ services: - redis - moto-init + # Runs only start_pipeline and create_variants_for_score_set (the fast-lane queue), as in production, so + # variant creation never waits behind mapping jobs on the main worker. + fast-worker: + <<: *worker + command: bash -c "arq mavedb.worker.FastLaneWorkerSettings --watch ./src" + dcd-mapping: build: ../dcd_mapping platform: linux/amd64 diff --git a/docker-compose-local.yml b/docker-compose-local.yml index a3bf81831..baa5e9bd0 100644 --- a/docker-compose-local.yml +++ b/docker-compose-local.yml @@ -23,7 +23,7 @@ services: - "8444:8000" restart: unless-stopped - worker: + worker: &worker image: mavedb-api-worker/mavedb-api-worker:latest build: context: . @@ -43,6 +43,12 @@ services: - redis restart: unless-stopped + # Runs only start_pipeline and create_variants_for_score_set (the fast-lane queue), as in production, so + # variant creation never waits behind mapping jobs on the main worker. + fast-worker: + <<: *worker + command: bash -c "arq src.mavedb.worker.FastLaneWorkerSettings" + db: image: postgres:15 env_file: From 7f49694c58dac6dfd6842128c379822f1f7ed19a Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Thu, 15 Oct 2026 11:09:43 -0700 Subject: [PATCH 248/252] fix(cdot): raise on cdot-rest outages instead of reporting transcripts missing cdot's REST client returns None for any non-OK response and caches it for the provider's lifetime, so a cdot-rest 5xx read as "no such transcript" (or no transcripts in a region) until the worker restarted. The MANE Select lookup then returned None, and the measured projection overwrote stored coding and protein HGVS with a group lacking them while reverse translation reported success. cdot_rest() now builds a provider that treats only a 404 as missing, raises on any other error status and sets a request timeout. A transcript a region lists but cdot no longer has is skipped by the MANE lookup. --- src/mavedb/data_providers/services.py | 24 +++++- src/mavedb/worker/lib/translation_ports.py | 9 ++- tests/data_providers/__init__.py | 0 tests/data_providers/test_services.py | 93 ++++++++++++++++++++++ 4 files changed, 122 insertions(+), 4 deletions(-) create mode 100644 tests/data_providers/__init__.py create mode 100644 tests/data_providers/test_services.py diff --git a/src/mavedb/data_providers/services.py b/src/mavedb/data_providers/services.py index 45b91b756..25d85eb2b 100644 --- a/src/mavedb/data_providers/services.py +++ b/src/mavedb/data_providers/services.py @@ -2,6 +2,7 @@ from typing import TYPE_CHECKING, Optional import boto3 +import requests from biocommons.seqrepo import SeqRepo from cdot.hgvs.dataproviders import ChainedSeqFetcher, FastaSeqFetcher, RESTDataProvider, SeqFetcher from ga4gh.vrs.dataproxy import SeqRepoDataProxy @@ -21,13 +22,34 @@ SEQREPO_DIR = os.environ.get("HGVS_SEQREPO_DIR", "/seqrepo") CSV_UPLOAD_S3_BUCKET_NAME = os.getenv("UPLOAD_S3_BUCKET_NAME", "score-set-csv-uploads-dev") +# (connect, read) seconds for each cdot-rest request. +CDOT_REQUEST_TIMEOUT = (10, 60) + + +class StrictRESTDataProvider(RESTDataProvider): + """cdot's REST data provider, raising on a cdot-rest outage instead of reporting the data missing. + + The base client returns ``None`` for any non-OK response, which reads as "no such transcript" (or "no + transcripts in this region") and is cached for the provider's lifetime. Callers that overwrite stored + projections with a missing transcript would then erase them during an outage. Only a 404, which + cdot-rest returns for an unknown transcript or gene, is reported as missing here. + """ + + def _get_from_url(self, url: str) -> Optional[dict]: + response = requests.get(url, timeout=CDOT_REQUEST_TIMEOUT) + if response.status_code == 404: + return None + + response.raise_for_status() + return response.json() + def seqfetcher() -> ChainedSeqFetcher: return ChainedSeqFetcher(SeqFetcher(), *[FastaSeqFetcher(file) for file in GENOMIC_FASTA_FILES]) def cdot_rest() -> RESTDataProvider: - return RESTDataProvider(url=CDOT_URL, seqfetcher=seqfetcher()) + return StrictRESTDataProvider(url=CDOT_URL, seqfetcher=seqfetcher()) def seqrepo() -> SeqRepo: diff --git a/src/mavedb/worker/lib/translation_ports.py b/src/mavedb/worker/lib/translation_ports.py index 2ea801f03..de1950087 100644 --- a/src/mavedb/worker/lib/translation_ports.py +++ b/src/mavedb/worker/lib/translation_ports.py @@ -119,8 +119,10 @@ def mane_select_transcript(self, g_hgvs: str) -> str | None: Reads cdot's per-build ``tag``, so it needs a cdot data provider. Where MANE Select transcripts of two genes overlap the variant, the lowest accession wins, so the choice is stable across runs. HGVS that - does not parse has no locus and gives ``None``; data provider failures raise, so callers can tell an - outage from a locus with no MANE transcript. + does not parse has no locus and gives ``None``. Data provider failures raise, so callers can tell an + outage from a locus with no MANE transcript; this relies on a provider that raises on one, as + :func:`~mavedb.data_providers.services.cdot_rest` does, since cdot's own REST client reports an outage + as no transcripts. """ self._ensure_initialized() try: @@ -136,7 +138,8 @@ def mane_select_transcript(self, g_hgvs: str) -> str | None: if tx["tx_ac"].startswith("NM_") ) for accession in candidates: - build = self._hdp._get_transcript(accession).get("genome_builds", {}).get("GRCh38", {}) + transcript = self._hdp._get_transcript(accession) or {} + build = transcript.get("genome_builds", {}).get("GRCh38", {}) if "MANE Select" in (build.get("tag") or ""): return accession diff --git a/tests/data_providers/__init__.py b/tests/data_providers/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/data_providers/test_services.py b/tests/data_providers/test_services.py new file mode 100644 index 000000000..ba1e96221 --- /dev/null +++ b/tests/data_providers/test_services.py @@ -0,0 +1,93 @@ +# ruff: noqa: E402 +from unittest.mock import patch + +import pytest +import requests +import requests_mock + +pytest.importorskip("boto3") +pytest.importorskip("cdot") +pytest.importorskip("hgvs") +pytest.importorskip("variant_annotation") + +from mavedb.data_providers.services import CDOT_REQUEST_TIMEOUT, StrictRESTDataProvider, cdot_rest +from mavedb.worker.lib.translation_ports import WorkerCoordinateTranslator + +CDOT_URL = "http://cdot.test" +TRANSCRIPT = "NM_007294.4" +TRANSCRIPT_URL = f"{CDOT_URL}/transcript/{TRANSCRIPT}" +GENOMIC_HGVS = "NC_000017.11:g.43045712T>C" +REGION_URL = f"{CDOT_URL}/transcripts/region/NC_000017.11/splign/43045711/43045712" +MANE_TRANSCRIPT_DATA = {"id": TRANSCRIPT, "genome_builds": {"GRCh38": {"tag": "MANE Select"}}} + + +@pytest.fixture +def provider() -> StrictRESTDataProvider: + return StrictRESTDataProvider(url=CDOT_URL) + + +@pytest.mark.unit +def test_cdot_rest_builds_the_strict_provider(): + # The genome FASTA files behind the default seqfetcher exist only in the container. + with patch("mavedb.data_providers.services.seqfetcher", return_value=None): + assert isinstance(cdot_rest(), StrictRESTDataProvider) + + +@pytest.mark.unit +def test_a_missing_transcript_is_none_and_cached(provider): + with requests_mock.Mocker() as m: + m.get(TRANSCRIPT_URL, status_code=404) + + assert provider._get_transcript(TRANSCRIPT) is None + assert provider._get_transcript(TRANSCRIPT) is None + assert m.call_count == 1 + + +@pytest.mark.unit +def test_an_outage_raises_and_is_not_cached(provider): + with requests_mock.Mocker() as m: + m.get(TRANSCRIPT_URL, status_code=503) + with pytest.raises(requests.HTTPError): + provider._get_transcript(TRANSCRIPT) + + m.get(TRANSCRIPT_URL, json=MANE_TRANSCRIPT_DATA) + assert provider._get_transcript(TRANSCRIPT) == MANE_TRANSCRIPT_DATA + + +@pytest.mark.unit +def test_requests_carry_a_timeout(provider): + with requests_mock.Mocker() as m: + m.get(TRANSCRIPT_URL, json=MANE_TRANSCRIPT_DATA) + provider._get_transcript(TRANSCRIPT) + + assert m.last_request.timeout == CDOT_REQUEST_TIMEOUT + + +@pytest.mark.unit +def test_mane_select_lookup_raises_during_an_outage(provider): + coordinates = WorkerCoordinateTranslator(provider) + with requests_mock.Mocker() as m: + m.get(REGION_URL, status_code=503) + + with pytest.raises(requests.HTTPError): + coordinates.mane_select_transcript(GENOMIC_HGVS) + + +@pytest.mark.unit +def test_mane_select_lookup_finds_the_tagged_transcript(provider): + coordinates = WorkerCoordinateTranslator(provider) + with requests_mock.Mocker() as m: + m.get(REGION_URL, json={"results": [{"tx_ac": TRANSCRIPT}]}) + m.get(TRANSCRIPT_URL, json=MANE_TRANSCRIPT_DATA) + + assert coordinates.mane_select_transcript(GENOMIC_HGVS) == TRANSCRIPT + + +@pytest.mark.unit +def test_mane_select_lookup_skips_a_listed_transcript_cdot_no_longer_has(provider): + coordinates = WorkerCoordinateTranslator(provider) + with requests_mock.Mocker() as m: + m.get(REGION_URL, json={"results": [{"tx_ac": TRANSCRIPT}]}) + m.get(TRANSCRIPT_URL, status_code=404) + + assert coordinates.mane_select_transcript(GENOMIC_HGVS) is None From 1fd39c7bafafb2d7a25e671c7a62f116d762fe9d Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Thu, 15 Oct 2026 11:09:56 -0700 Subject: [PATCH 249/252] fix(clingen): link CAR's answer when it lists a different spelling of the submitted HGVS CAR lists its own normalized spellings of an allele, not the submitted one, so an indel or identity written differently from CAR's form was rejected as a malformed response even though the positional answer was right. On staging that dropped CAIDs for about 0.4% of submissions, all codon-level delins and insertions from reverse translation, and a forced re-registration skipped those alleles instead of verifying them. The count check already ties each response to its submission, so the CAID is linked as usual. An unlisted spelling is recorded on the event as submitted_hgvs_unlisted with the submitted HGVS, counted in the job outcome as unlisted_hgvs_allele_count, and summarized in one warning per job. --- .../worker/jobs/external_services/clingen.py | 40 ++++++---- .../jobs/external_services/test_clingen.py | 77 +++++++++++++++++-- 2 files changed, 92 insertions(+), 25 deletions(-) diff --git a/src/mavedb/worker/jobs/external_services/clingen.py b/src/mavedb/worker/jobs/external_services/clingen.py index 1efd1a2ad..3b85f60c8 100644 --- a/src/mavedb/worker/jobs/external_services/clingen.py +++ b/src/mavedb/worker/jobs/external_services/clingen.py @@ -13,6 +13,7 @@ import logging from collections.abc import Sequence from dataclasses import dataclass +from typing import Any from sqlalchemy import select from sqlalchemy.orm import Session @@ -212,6 +213,8 @@ async def submit_score_set_mappings_to_car(ctx: dict, job_id: int, job_manager: linked_allele_ids: set[int] = set() preexisting_allele_ids: set[int] = set() failed_allele_ids: set[int] = set() + # Linked alleles whose submitted spelling CAR's response did not list; a subset of linked_allele_ids. + unlisted_allele_ids: set[int] = set() preexisting_alleles = [] pending_alleles = [] @@ -296,6 +299,7 @@ def _outcome_data() -> dict[str, int]: "registered_allele_count": len(linked_allele_ids), "already_registered_allele_count": len(preexisting_allele_ids), "failed_allele_count": len(failed_allele_ids), + "unlisted_hgvs_allele_count": len(unlisted_allele_ids), } # All pending alleles failed HGVS extraction; annotations already written above. @@ -402,25 +406,17 @@ def _outcome_data() -> dict[str, int]: caid = caid_iri.split("/")[-1] - # The positional zip is CAR's contract, but nothing else ties a response to its submission. - # An allele that does not list the submitted expression cannot be trusted for these alleles. + # CAR lists its own normalized spellings, not the submitted one, so an indel or identity written + # differently from CAR's form is expected to be missing from the list. The positional zip (with the + # count check above) ties the response to the submission; an unlisted spelling is recorded for audit. + spelling_metadata: dict[str, Any] = {} if hgvs_string not in _response_hgvs(response): - logger.error( - msg=f"CAR returned {caid!r} for HGVS '{hgvs_string}', but that allele does not list it.", + unlisted_allele_ids.update(allele_ids_for_hgvs) + spelling_metadata = {"submitted_hgvs": hgvs_string, "submitted_hgvs_unlisted": True} + logger.debug( + msg=f"CAR returned {caid!r} for HGVS '{hgvs_string}' without listing that spelling.", extra=job_manager.logging_context(), ) - for allele_id in allele_ids_for_hgvs: - failed_allele_ids.add(allele_id) - _annotate_caid( - annotation_manager, - allele_id, - Disposition.FAILED, - EventReason.MALFORMED_RESPONSE, - error_message="ClinGen Allele Registry returned an allele that does not include the submitted HGVS.", - metadata={"submitted_hgvs": hgvs_string, "returned_caid": caid}, - ) - - continue for allele_id in allele_ids_for_hgvs: entry = allele_data[allele_id] @@ -446,6 +442,7 @@ def _outcome_data() -> dict[str, int]: Disposition.PRESENT, EventReason.CORRECTED, metadata={ + **spelling_metadata, "clingen_allele_id": caid, "replaced_caid": prior_caid, "submitted_hgvs": hgvs_string, @@ -472,7 +469,7 @@ def _outcome_data() -> dict[str, int]: allele_id, Disposition.PRESENT, reason, - metadata={"clingen_allele_id": caid}, + metadata={**spelling_metadata, "clingen_allele_id": caid}, ) # Submitted HGVS with no trustworthy response: the truncated tail when the counts line up @@ -516,6 +513,15 @@ def _outcome_data() -> dict[str, int]: extra=job_manager.logging_context(), ) + if outcome_data["unlisted_hgvs_allele_count"] > 0: + logger.warning( + msg=( + f"CAR did not list the submitted spelling for {outcome_data['unlisted_hgvs_allele_count']} linked " + f"alleles in score set {score_set.urn}; their events carry submitted_hgvs_unlisted." + ), + extra=job_manager.logging_context(), + ) + logger.info(msg="Completed CAR mapped resource submission", extra=job_manager.logging_context()) job_manager.db.flush() return JobExecutionOutcome.succeeded(data=outcome_data) diff --git a/tests/worker/jobs/external_services/test_clingen.py b/tests/worker/jobs/external_services/test_clingen.py index 52205a325..d89b35aa2 100644 --- a/tests/worker/jobs/external_services/test_clingen.py +++ b/tests/worker/jobs/external_services/test_clingen.py @@ -968,7 +968,7 @@ async def test_submit_score_set_mappings_to_car_force_reregister_replaces_a_caid assert corrected.event_metadata["retired_gnomad_links"] == 1 assert corrected.event_metadata["retired_clinvar_links"] == 1 - async def test_submit_score_set_mappings_to_car_rejects_a_response_without_the_submitted_hgvs( + async def test_submit_score_set_mappings_to_car_links_a_caid_whose_response_does_not_list_the_submitted_hgvs( self, mock_worker_ctx, session, @@ -981,7 +981,7 @@ async def test_submit_score_set_mappings_to_car_rejects_a_response_without_the_s dummy_variant_creation_job_run, dummy_variant_mapping_job_run, ): - """A response for some other allele is not written to the submitted one, however the counts line up.""" + """CAR lists its own normalized spelling, so a positionally matched CAID is linked and the gap recorded.""" await create_mappings_in_score_set( session, mock_s3_client, @@ -992,12 +992,17 @@ async def test_submit_score_set_mappings_to_car_rejects_a_response_without_the_s dummy_variant_mapping_job_run, ) allele = session.scalars(select(Allele)).first() - other_allele = {"@id": "CA_OTHER", "type": "nucleotide", "genomicAlleles": [{"hgvs": ["NC_000001.11:g.1A>T"]}]} + submitted_hgvs = get_hgvs_from_post_mapped(allele.post_mapped) + normalized = { + "@id": "CA_NORMALIZED", + "type": "nucleotide", + "genomicAlleles": [{"hgvs": ["NC_000001.11:g.1A>T"]}], + } with ( patch( "mavedb.worker.jobs.external_services.clingen.ClinGenAlleleRegistryService.dispatch_submissions", - return_value=[other_allele], + return_value=[normalized], ), patch("mavedb.worker.jobs.external_services.clingen.CAR_SUBMISSION_ENDPOINT", "http://fake-endpoint"), patch("mavedb.worker.jobs.external_services.clingen.CLIN_GEN_SUBMISSION_ENABLED", True), @@ -1008,15 +1013,71 @@ async def test_submit_score_set_mappings_to_car_rejects_a_response_without_the_s JobManager(session, mock_worker_ctx["redis"], submit_score_set_mappings_to_car_sample_job_run.id), ) - assert result.status == JobStatus.FAILED + assert result.status == JobStatus.SUCCEEDED + assert result.data["registered_allele_count"] == 1 + assert result.data["unlisted_hgvs_allele_count"] == 1 session.refresh(allele) - assert allele.clingen_allele_id is None + assert allele.clingen_allele_id == "CA_NORMALIZED" + + event = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") + ).one() + assert (event.disposition, event.reason) == ("present", "created") + assert event.event_metadata["submitted_hgvs_unlisted"] is True + assert event.event_metadata["submitted_hgvs"] == submitted_hgvs + + async def test_submit_score_set_mappings_to_car_force_reregister_reconfirms_a_caid_whose_response_does_not_list_the_submitted_hgvs( + self, + mock_worker_ctx, + session, + with_submit_score_set_mappings_to_car_job, + submit_score_set_mappings_to_car_sample_job_run, + mock_s3_client, + sample_score_dataframe, + sample_count_dataframe, + with_dummy_setup_jobs, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ): + """A forced re-registration verifies a stored CAID even when CAR spells the allele differently.""" + await create_mappings_in_score_set( + session, + mock_s3_client, + mock_worker_ctx, + sample_score_dataframe, + sample_count_dataframe, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ) + allele = session.scalars(select(Allele)).first() + allele.clingen_allele_id = "CA_STORED" + submit_score_set_mappings_to_car_sample_job_run.job_params = { + **submit_score_set_mappings_to_car_sample_job_run.job_params, + "force_reregister": True, + } + session.flush() + normalized = {"@id": "CA_STORED", "type": "nucleotide", "genomicAlleles": [{"hgvs": ["NC_000001.11:g.1A>T"]}]} + with ( + patch( + "mavedb.worker.jobs.external_services.clingen.ClinGenAlleleRegistryService.dispatch_submissions", + return_value=[normalized], + ), + patch("mavedb.worker.jobs.external_services.clingen.CAR_SUBMISSION_ENDPOINT", "http://fake-endpoint"), + patch("mavedb.worker.jobs.external_services.clingen.CLIN_GEN_SUBMISSION_ENABLED", True), + ): + result = await submit_score_set_mappings_to_car( + mock_worker_ctx, + submit_score_set_mappings_to_car_sample_job_run.id, + JobManager(session, mock_worker_ctx["redis"], submit_score_set_mappings_to_car_sample_job_run.id), + ) + + assert result.status == JobStatus.SUCCEEDED event = session.scalars( select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).one() - assert (event.disposition, event.reason) == ("failed", "malformed_response") - assert event.event_metadata["returned_caid"] == "CA_OTHER" + assert (event.disposition, event.reason) == ("present", "reconfirmed") + assert event.event_metadata["submitted_hgvs_unlisted"] is True async def _add_derived_alleles_with_distinct_hgvs(self, session, count): """Attach ``count`` extra alleles to the score set, each carrying a distinct HGVS so the CAR From a22b553a1c11afc58914ad188fe723f0754e5991 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Thu, 15 Oct 2026 11:21:16 -0700 Subject: [PATCH 250/252] fix(projection): project measured variants through variant-annotation's shared projection A measured coding change across an exon junction (a codon split by an intron) projected to a genomic change that also deleted the intron, because hgvs maps the two ends of an interval separately. The measured projection and reverse translation's candidates now use variant-annotation's coding_to_genomic and genomic_to_coding, which give such a change no projection and write an inv as the delins on its trimmed span. - Pin variant-annotation to v1.2.0-beta.1, which brings variant-translation 1.2.0. - WorkerCoordinateTranslator gains c_to_n for the span check. - The dump README and CHANGELOG note that a split-codon change has NA for post_mapped_hgvs_g. Score sets that already ran reverse translation need it re-run to retire intron-deleting candidates and refresh measured groups at split codons. --- poetry.lock | 17 ++++++------ pyproject.toml | 2 +- src/mavedb/scripts/resources/CHANGELOG.md | 1 + src/mavedb/scripts/resources/README.md | 2 ++ src/mavedb/worker/lib/projection.py | 22 ++++++--------- src/mavedb/worker/lib/translation_ports.py | 14 +++++++--- .../test_reverse_translation.py | 10 +++++++ tests/worker/lib/test_projection.py | 27 +++++++++++++++++++ 8 files changed, 69 insertions(+), 26 deletions(-) diff --git a/poetry.lock b/poetry.lock index ae3d6a64e..8455104b2 100644 --- a/poetry.lock +++ b/poetry.lock @@ -4937,7 +4937,7 @@ test = ["aiohttp (>=3.10.5)", "flake8 (>=6.1,<7.0)", "mypy (>=0.800)", "psutil", [[package]] name = "variant-annotation" -version = "0.1.0-dev.6" +version = "1.2.0-beta.1" description = "Genetic variant mapping and annotation pipeline" optional = true python-versions = ">=3.11" @@ -4955,11 +4955,12 @@ pandas = ">=2.2.1,<3.0.0" python-dotenv = "*" redis = "*" requests = "*" -variant-translation = "1.1.0b1" +variant-translation = "1.2.0" [package.extras] dev = ["ruff"] gnomad = ["hail"] +liftover = ["pyliftover"] mapping = ["dcd-mapping @ git+https://github.com/VariantEffect/dcd_mapping2.git@v2026.1.2"] publish = ["build", "twine"] tests = ["pytest"] @@ -4967,20 +4968,20 @@ tests = ["pytest"] [package.source] type = "git" url = "https://github.com/bbi-lab/variant-annotation.git" -reference = "v0.1.0-dev.6" -resolved_reference = "dee39b5d03a89a2464b02fe42774e173594422cb" +reference = "v1.2.0-beta.1" +resolved_reference = "350b53bae5e99d30ecf6f33821c3df70f5c76a65" [[package]] name = "variant-translation" -version = "1.1.0b1" +version = "1.2.0" description = "Reverse-translate protein HGVS variants." optional = true python-versions = ">=3.8" groups = ["main"] markers = "extra == \"server\"" files = [ - {file = "variant_translation-1.1.0b1-py3-none-any.whl", hash = "sha256:cdf693ab564b767c89e7b463b1345e801d2ab4a5f312631584bf79d178ba6122"}, - {file = "variant_translation-1.1.0b1.tar.gz", hash = "sha256:5151d02ccb5c651c7ba47398d75355f2bd19a36717faa39dc2a390cfb4c7968f"}, + {file = "variant_translation-1.2.0-py3-none-any.whl", hash = "sha256:143f242ebcff7b1a2c07bc01283c032f73535c50ddaaaa65e0803a26592f8250"}, + {file = "variant_translation-1.2.0.tar.gz", hash = "sha256:e12a1d7d8ab78ba89999e72fc9203f071b6f799417e575bbf5753e8ca69356b4"}, ] [package.dependencies] @@ -5301,4 +5302,4 @@ server = ["aiocache", "alembic", "alembic-utils", "arq", "authlib", "biocommons" [metadata] lock-version = "2.1" python-versions = "^3.11" -content-hash = "9b7c249bd890a9e36b278d6f0c1c0441d57acded1d0f4ca30cdfc6df9d0b2a4d" +content-hash = "5a24b728c64c490a22d95d754e2ffaa6bf7615d01de828803a69f3bfb5e82030" diff --git a/pyproject.toml b/pyproject.toml index 78029d1f6..126b5606d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,7 +65,7 @@ starlette = { version = "^1.7.0", optional = true } starlette-context = { version = "^0.3.6", optional = true } slack-sdk = { version = "~3.21.3", optional = true } uvicorn = { extras = ["standard"], version = "*", optional = true } -variant-annotation = { git = "https://github.com/bbi-lab/variant-annotation.git", tag = "v0.1.0-dev.6", optional = true } +variant-annotation = { git = "https://github.com/bbi-lab/variant-annotation.git", tag = "v1.2.0-beta.1", optional = true } watchtower = { version = "~3.2.0", optional = true } asyncclick = "^8.3.0.7" filelock = "^3.29.0" diff --git a/src/mavedb/scripts/resources/CHANGELOG.md b/src/mavedb/scripts/resources/CHANGELOG.md index 7f3ab1858..2bac1a66f 100644 --- a/src/mavedb/scripts/resources/CHANGELOG.md +++ b/src/mavedb/scripts/resources/CHANGELOG.md @@ -49,6 +49,7 @@ given archive was generated. projections to the other levels, with the genomic string normalized. Score sets assayed at the protein level fill only `mavedb.post_mapped_hgvs_p`, and their `gnomad.*` and `clinvar.*` columns are `NA`: both are nucleotide-level resources. + A coding change across an exon junction has `NA` for `mavedb.post_mapped_hgvs_g`. - gnomAD frequencies are now reported from whichever release each allele's current record came from, rather than only from the single release the deployment serves. An allele not covered by a newer release previously reported `NA` for every gnomAD diff --git a/src/mavedb/scripts/resources/README.md b/src/mavedb/scripts/resources/README.md index 4a42b2958..e28e5541a 100644 --- a/src/mavedb/scripts/resources/README.md +++ b/src/mavedb/scripts/resources/README.md @@ -146,6 +146,8 @@ Columns are grouped by a namespace prefix. The groups below always appear: The `mavedb.post_mapped_hgvs_*` columns describe the measured variant at each level: the measured HGVS plus its projections to the other levels. The genomic string is normalized, so for a genomic assay it can differ from `mavedb.post_mapped_hgvs_at_assay_level`, and a projected inversion is written as a `delins`. +A coding change across an exon junction (a codon split by an intron) has no single genomic change, so its +`mavedb.post_mapped_hgvs_g` is `NA`. Score sets assayed at the protein level fill only `mavedb.post_mapped_hgvs_p`: a protein change has no single coding or genomic encoding. gnomAD and ClinVar are nucleotide-level resources, so the `gnomad.*` and `clinvar.*` columns are `NA` for protein-level variants. diff --git a/src/mavedb/worker/lib/projection.py b/src/mavedb/worker/lib/projection.py index 38bc92b02..a6395f3d9 100644 --- a/src/mavedb/worker/lib/projection.py +++ b/src/mavedb/worker/lib/projection.py @@ -1,8 +1,8 @@ """Measured projection groups: each mapping record's measured variant described at every level it has. A record's measured group holds its measured variant's HGVS at the measured level and its projections to -the other levels: the coding HGVS of a genomic measurement (``g_to_c`` on the record's transcript) or the -genomic HGVS of a coding one (``c_to_g``), plus the protein consequence (``c_to_p``). They are computed from +the other levels: the coding HGVS of a genomic measurement (on the record's transcript) or the genomic HGVS +of a coding one, plus the protein consequence (``c_to_p``). They are computed from the mapping alone, so every nucleotide record gets them, including variants reverse translation cannot reach: intronic and UTR changes, multi-codon indels, start and stop changes. @@ -19,6 +19,7 @@ from sqlalchemy import CursorResult, select, true from sqlalchemy.orm import Session +from variant_annotation.lib.translation import coding_to_genomic, genomic_to_coding from mavedb.lib.hgvs import extract_accession, is_cis_phased_hgvs, strip_protein_prediction_parens from mavedb.models.enums.sequence_level import SequenceLevel @@ -59,11 +60,6 @@ def _expand_reference_identity(hgvs: str) -> str: return f"{match['accession']}:g.{start}_{start + len(match['ref']) - 1}=" -def _is_inversion(hgvs: str) -> bool: - """Whether hgvs wrote a variant as ``inv``, which VRS cannot represent; projections keep the delins.""" - return hgvs.endswith("inv") - - def _protein_consequence(coordinates: WorkerCoordinateTranslator, hgvs_c: str) -> Optional[str]: """The protein consequence of a coding variant, or ``None`` when it is unknown (``p.?``: intronic, UTR).""" hgvs_p = strip_protein_prediction_parens(coordinates.c_to_p(hgvs_c)) @@ -85,7 +81,9 @@ def project_measured_hgvs( Each projection is attempted independently, so a failed protein consequence still keeps the coding projection. A genomic measurement is normalized before it is projected, and the group carries the - normalized form; one with no coding transcript has only its genomic HGVS. + normalized form; one with no coding transcript has only its genomic HGVS. Coding and genomic levels are + projected by variant-annotation's :func:`coding_to_genomic` and :func:`genomic_to_coding`, so a change no + single change describes at the other level (a codon split by an intron) has no projection there. """ errors: list[str] = [] if assay_level == SequenceLevel.protein: @@ -105,18 +103,14 @@ def project_measured_hgvs( if transcript is not None: try: - hgvs_c = coordinates.g_to_c(hgvs_g, transcript) - if _is_inversion(hgvs_c): - hgvs_c = coordinates.g_to_c_literal(hgvs_g, transcript) + hgvs_c = genomic_to_coding(hgvs_g, transcript, coordinates) except Exception as e: errors.append(f"g_to_c: {e}") else: hgvs_c = hgvs try: - hgvs_g = coordinates.c_to_g(hgvs) - if _is_inversion(hgvs_g): - hgvs_g = coordinates.c_to_g_literal(hgvs) + hgvs_g = coding_to_genomic(hgvs, coordinates) except Exception as e: errors.append(f"c_to_g: {e}") diff --git a/src/mavedb/worker/lib/translation_ports.py b/src/mavedb/worker/lib/translation_ports.py index de1950087..dd2e19d61 100644 --- a/src/mavedb/worker/lib/translation_ports.py +++ b/src/mavedb/worker/lib/translation_ports.py @@ -70,7 +70,8 @@ def _ensure_initialized(self) -> None: assembly_name="GRCh38", alt_aln_method="splign", ) - # Unnormalized, so a delins stays a delins rather than becoming ``inv`` (see c_to_g_literal). + # Unnormalized, so each end of a change maps where it is and a delins stays a delins rather than + # becoming ``inv`` (see c_to_g_literal). self._literal_mapper = hgvs.assemblymapper.AssemblyMapper( self._hdp, assembly_name="GRCh38", @@ -106,7 +107,8 @@ def g_to_c(self, g_hgvs: str, transcript: str) -> str: return str(self._mapper.g_to_t(self._parser.parse(g_hgvs), transcript)) def g_to_c_literal(self, g_hgvs: str, transcript: str) -> str: - """Project to coding without normalization, so a delins stays a delins rather than becoming ``inv``.""" + """Project to coding without normalization; variant-annotation uses it to check a projection's span and + to rewrite an ``inv`` as delins.""" self._ensure_initialized() return str(self._literal_mapper.g_to_t(self._parser.parse(g_hgvs), transcript)) @@ -146,6 +148,12 @@ def mane_select_transcript(self, g_hgvs: str) -> str | None: return None def c_to_g_literal(self, c_hgvs: str) -> str: - """Project to genomic without normalization; variant-annotation uses it to rewrite an ``inv`` as delins.""" + """Project to genomic without normalization; variant-annotation uses it to check a projection's span + and to rewrite an ``inv`` as delins.""" self._ensure_initialized() return str(self._literal_mapper.c_to_g(self._parser.parse(c_hgvs))) + + def c_to_n(self, c_hgvs: str) -> str: + """Project to transcript (``n.``) coordinates, which count exonic bases only.""" + self._ensure_initialized() + return str(self._literal_mapper.c_to_n(self._parser.parse(c_hgvs))) diff --git a/tests/worker/jobs/variant_processing/test_reverse_translation.py b/tests/worker/jobs/variant_processing/test_reverse_translation.py index cc86c881b..52b045788 100644 --- a/tests/worker/jobs/variant_processing/test_reverse_translation.py +++ b/tests/worker/jobs/variant_processing/test_reverse_translation.py @@ -193,6 +193,16 @@ def c_to_g(self, c_hgvs: str) -> str: def c_to_p(self, c_hgvs: str) -> str: return self.p_by_c.get(c_hgvs, "NP_999999.1:p.(Met1Val)") + # The tables hold changes whose spans match on both levels, so the unnormalized projections agree. + def c_to_g_literal(self, c_hgvs: str) -> str: + return self.c_to_g(c_hgvs) + + def g_to_c_literal(self, g_hgvs: str, transcript: str) -> str: + return self.g_to_c(g_hgvs, transcript) + + def c_to_n(self, c_hgvs: str) -> str: + return c_hgvs.replace(":c.", ":n.") + def mane_select_transcript(self, g_hgvs: str) -> str | None: self.mane_lookups.append(g_hgvs) return self.mane diff --git a/tests/worker/lib/test_projection.py b/tests/worker/lib/test_projection.py index f854ba041..8c9d3d187 100644 --- a/tests/worker/lib/test_projection.py +++ b/tests/worker/lib/test_projection.py @@ -30,6 +30,17 @@ def c_to_g(self, c_hgvs: str) -> str: def c_to_p(self, c_hgvs: str) -> str: return self._answer("c_to_p", c_hgvs) + # Unnormalized projections default to the normalized answer; a test that needs them to differ adds a + # ``*_literal`` answer. + def c_to_g_literal(self, c_hgvs: str) -> str: + return self.answers.get(("c_to_g_literal", c_hgvs)) or self.c_to_g(c_hgvs) + + def g_to_c_literal(self, g_hgvs: str, transcript: str) -> str: + return self.answers.get(("g_to_c_literal", g_hgvs)) or self.g_to_c(g_hgvs, transcript) + + def c_to_n(self, c_hgvs: str) -> str: + return self.answers.get(("c_to_n", c_hgvs)) or c_hgvs.replace(":c.", ":n.") + MULTI_CODON_G = "NC_000008.11:g.95025143_95025145delinsAAC" MULTI_CODON_C = "NM_152416.4:c.135_137delinsAAC" @@ -157,6 +168,22 @@ def g_to_c_literal(self, g_hgvs: str, transcript: str) -> str: assert measured.hgvs_p == "NP_008819.1:p.Met72Thr" assert errors == [] + def test_a_coding_change_at_a_split_codon_has_no_genomic_projection(self): + """hgvs maps each end of the change separately, so its genomic form would delete the intron between.""" + split = "NM_000546.6:c.73_75delinsAGG" + coordinates = FakeCoordinates( + { + ("c_to_g", split): "NC_000017.11:g.7676403_7676522delinsCCT", + ("c_to_n", split): "NM_000546.6:n.215_217delinsAGG", + ("c_to_p", split): "NP_000537.3:p.(Leu25Arg)", + } + ) + + measured, errors = project_measured_hgvs(SequenceLevel.cdna, split, transcript=None, coordinates=coordinates) + + assert measured == MeasuredHgvs(hgvs_c=split, hgvs_g=None, hgvs_p="NP_000537.3:p.Leu25Arg") + assert errors == [] + def test_a_reference_identity_is_projected_as_written(self): """Normalizing ``g.100CCT=`` would collapse its three-base span to one position.""" identity = "NC_000007.14:g.144548593_144548595=" From 104be2f819f647a0e66d3f4b00af6ecb34db1019 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Thu, 15 Oct 2026 11:32:03 -0700 Subject: [PATCH 251/252] fix(scripts): keep each score set's modification date in the score distribution backfill ScoreSet.modification_date has onupdate=date.today, so writing the summary through the ORM stamped every score set as modified on the day the backfill ran. The summary is now written with an UPDATE that sets modification_date to itself. --- src/mavedb/scripts/backfill_score_distributions.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/mavedb/scripts/backfill_score_distributions.py b/src/mavedb/scripts/backfill_score_distributions.py index 6d28221f1..09c5dd3cb 100644 --- a/src/mavedb/scripts/backfill_score_distributions.py +++ b/src/mavedb/scripts/backfill_score_distributions.py @@ -15,7 +15,7 @@ import logging import asyncclick as click -from sqlalchemy import select +from sqlalchemy import select, update from sqlalchemy.orm import Session from mavedb.lib.score_distribution import summarize_variant_data @@ -50,13 +50,16 @@ def backfill(db: Session, force: bool = False, commit_each: bool = False) -> tup ) summary = summarize_variant_data(variant_data) - score_set = db.get_one(ScoreSet, score_set_id) - score_set.score_distribution = summary + # Setting modification_date to itself keeps its onupdate from stamping every score set as modified today. + db.execute( + update(ScoreSet) + .where(ScoreSet.id == score_set_id) + .values(score_distribution=summary, modification_date=ScoreSet.modification_date) + ) if commit_each: db.commit() else: db.flush() - db.expunge(score_set) if summary is None: empty += 1 From 324cee2fd2ddad7a5f15f7e8c90b7bbbf08d4cbc Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Thu, 15 Oct 2026 11:45:15 -0700 Subject: [PATCH 252/252] fix(variants): keep projectionOf to a nucleotide allele's c<->g counterpart projectionOf names another spelling of the same nucleotide change. Since projection groups landed, the protein consequence in a measured group was also given the measured allele as its projectionOf, so the UI's allele grouping folded the protein into the measured nucleotide group: on a protein-change page with a nucleotide measurement the nucleotide row was marked as the page's own variant, and rows reordered with the selected measurement. The protein's relation (translation_of) already says what it is to the measured allele, so it no longer carries projectionOf in variant or allele detail. --- src/mavedb/lib/allele_detail.py | 19 ++++++------------- src/mavedb/lib/variant_detail.py | 4 +--- tests/lib/test_variant_detail.py | 17 +++++++++-------- tests/routers/test_variant.py | 4 ++-- 4 files changed, 18 insertions(+), 26 deletions(-) diff --git a/src/mavedb/lib/allele_detail.py b/src/mavedb/lib/allele_detail.py index 33a6d2429..b372970e2 100644 --- a/src/mavedb/lib/allele_detail.py +++ b/src/mavedb/lib/allele_detail.py @@ -76,10 +76,9 @@ def _focus_projection_pairs( """A ``digest -> digest`` map pairing each focus allele with its projection. A ``projection_group`` is one variant at each level, local to a single mapping record. A nucleotide - focus pairs with the ≤1 *other* nucleotide allele in its group (c↔g, symmetric). A protein focus in a - measured group pairs with that record's measured allele, one way only, so the measured allele keeps its - nucleotide pairing. Reading any one live link - is safe: the focus holds one live link per record it appears in, and each record re-encodes the same + focus pairs with the ≤1 *other* nucleotide allele in its group (c↔g, symmetric). A protein focus pairs + with nothing: it is what its group's nucleotide alleles encode, not another spelling of them. Reading any + one live link is safe: the focus holds one live link per record it appears in, and each record re-encodes the same change onto the same content-addressed digests, so the pairing is record-independent (assuming those records agree on genome assembly). @@ -88,9 +87,8 @@ def _focus_projection_pairs( """ pairs: dict[str, str] = {} for focus in focus_alleles: - if focus.vrs_digest is None: + if focus.vrs_digest is None or focus.level not in NUCLEOTIDE_LEVELS: continue - nucleotide = focus.level in NUCLEOTIDE_LEVELS # limit(1) rather than one_or_none(), which would raise for any allele measured more than once. link_query = ( @@ -113,18 +111,13 @@ def _focus_projection_pairs( .where(MappingRecordAllele.mapping_record_id == link.mapping_record_id) .where(MappingRecordAllele.projection_group == link.projection_group) .where(MappingRecordAllele.allele_id != focus.id) + .where(Allele.level.in_(NUCLEOTIDE_LEVELS)) .where(MappingRecordAllele.live_at(as_of)) ) - partners = ( - partners.where(Allele.level.in_(NUCLEOTIDE_LEVELS)) - if nucleotide - else partners.where(MappingRecordAllele.is_authoritative) - ) for projection in db.scalars(partners).all(): if projection is not None: pairs[focus.vrs_digest] = projection - if nucleotide: - pairs.setdefault(projection, focus.vrs_digest) + pairs.setdefault(projection, focus.vrs_digest) return pairs diff --git a/src/mavedb/lib/variant_detail.py b/src/mavedb/lib/variant_detail.py index e9b33e65a..22b6cd66e 100644 --- a/src/mavedb/lib/variant_detail.py +++ b/src/mavedb/lib/variant_detail.py @@ -238,7 +238,7 @@ def _assemble_variant_detail( # projection_group -> the nucleotide digests sharing it, for within-record c↔g pairing. A nucleotide link's # projection is the ≤1 *other* nucleotide digest in its group. The protein consequence in the measured group - # pairs with the measured allele: it is the part of the measurement the score describes. + # pairs with nothing: it is what the pair encodes (its relation says so), not another spelling of it. group_digests: dict[int, list[str]] = {} for link in links: if ( @@ -262,8 +262,6 @@ def _assemble_variant_detail( (digest for digest in group_digests.get(link.projection_group, []) if digest != allele.vrs_digest), None, ) - elif link.projection_group is not None and link.projection_group == focus_group and authoritative is not None: - projection_of = authoritative.allele.vrs_digest relation, derivation = ( (None, None) diff --git a/tests/lib/test_variant_detail.py b/tests/lib/test_variant_detail.py index edd380c8e..efb8f1d33 100644 --- a/tests/lib/test_variant_detail.py +++ b/tests/lib/test_variant_detail.py @@ -238,10 +238,10 @@ def test_full_envelope_for_a_coding_assay(session, setup_lib_db_with_score_set): assert protein_identity.hgvs == "NP_000537.3:p.Ala406Thr" # coalesced from hgvs_p # The protein member is a translation of the measured coding allele (member -> defining)... assert protein_identity.relation == "translation_of" - # ...and, on a nucleotide assay, is itself a deterministic projection (axes are independent) that pairs - # with the measured allele it shares the measured group with. + # ...and, on a nucleotide assay, is itself a deterministic projection (axes are independent). It is not + # a c<->g counterpart, so it pairs with nothing. assert protein_identity.derivation == "projection" - assert protein_identity.projection_of == "cdna-digest" + assert protein_identity.projection_of is None # The synonymous cousin (different projection group) is a member wearing co_encodes (structural), and # labelled `convergent` (provenance) — a distinct, precise change that converges on the measured # consequence, not an ambiguous candidate and not a projection of the measured change. It pairs with @@ -351,9 +351,9 @@ def test_mapped_triple_carries_a_coding_projection_with_no_allele(session, setup @pytest.mark.integration -def test_protein_link_in_the_measured_group_pairs_with_the_measured_allele(session, setup_lib_db_with_score_set): - """The protein consequence is the part of the measurement the score describes, so it pairs with the - measured allele; the nucleotide pair still points at each other rather than at it.""" +def test_protein_link_in_the_measured_group_pairs_with_nothing(session, setup_lib_db_with_score_set): + """The protein consequence shares the measured group but is what the pair encodes, not another spelling of + it, so it pairs with nothing; the nucleotide pair still points at each other.""" score_set = setup_lib_db_with_score_set variant = _variant(session, score_set, 1, hgvs_nt="c.1216G>A") record = _record(session, variant, assay_level="cdna", hgvs_assay_level="NM_000546.6:c.1216G>A") @@ -373,7 +373,7 @@ def test_protein_link_in_the_measured_group_pairs_with_the_measured_allele(sessi _link(session, record, genomic, group=measured_group) variant_alleles = get_variant_detail(session, variant).alleles - assert variant_alleles["prot-digest"].projection_of == "cdna-digest" + assert variant_alleles["prot-digest"].projection_of is None assert variant_alleles["cdna-digest"].projection_of == "gen-digest" assert variant_alleles["gen-digest"].projection_of == "cdna-digest" @@ -382,7 +382,8 @@ def test_protein_link_in_the_measured_group_pairs_with_the_measured_allele(sessi assert nucleotide_focus["gen-digest"].projection_of == "cdna-digest" protein_focus = get_allele_detail(session, protein, focus_digests={"prot-digest"}).alleles - assert protein_focus["prot-digest"].projection_of == "cdna-digest" + assert protein_focus["prot-digest"].projection_of is None + assert protein_focus["cdna-digest"].projection_of is None T0 = datetime(2020, 1, 1, tzinfo=timezone.utc) diff --git a/tests/routers/test_variant.py b/tests/routers/test_variant.py index 0847cce0d..944c4b1a1 100644 --- a/tests/routers/test_variant.py +++ b/tests/routers/test_variant.py @@ -201,9 +201,9 @@ def test_get_variant_detail_envelope(client, session, data_provider, data_files, assert body["alleles"]["prot-digest"]["level"] == "protein" assert body["alleles"]["prot-digest"]["hgvs"] == "NP_000537.3:p.Ala406Thr" assert body["alleles"]["prot-digest"]["relation"] == "translation_of" - # The apex is a deterministic projection that pairs with the measured allele it shares the measured group with. + # The apex is a deterministic projection, but not a c<->g counterpart, so it pairs with nothing. assert body["alleles"]["prot-digest"]["derivation"] == "projection" - assert body["alleles"]["prot-digest"]["projectionOf"] == "cdna-digest" + assert body["alleles"]["prot-digest"]["projectionOf"] is None # The synonymous cousin (different projection group) surfaces as a member wearing co_encodes and is # labelled `convergent` (a distinct change sharing the consequence, not an ambiguous candidate). assert body["alleles"]["cousin-digest"]["relation"] == "co_encodes"